From 4afc0a0e2aeadc3bcafd4979ad5625ca7c48a01d Mon Sep 17 00:00:00 2001 From: Ra4ster Date: Sat, 5 Sep 2026 06:54:15 +0000 Subject: [PATCH 01/11] Now implementing CUDA backend. WIP. --- deepity_build/__pycache__/cli.cpython-312.pyc | Bin 15076 -> 15195 bytes deepity_build/__pycache__/cli.cpython-314.pyc | Bin 16315 -> 0 bytes .../__pycache__/cmake_runner.cpython-312.pyc | Bin 4111 -> 3971 bytes .../__pycache__/cmake_runner.cpython-314.pyc | Bin 4830 -> 0 bytes .../__pycache__/config.cpython-312.pyc | Bin 3624 -> 3677 bytes .../__pycache__/config.cpython-314.pyc | Bin 4431 -> 0 bytes .../__pycache__/git_info.cpython-312.pyc | Bin 1718 -> 1701 bytes .../__pycache__/git_info.cpython-314.pyc | Bin 1929 -> 0 bytes .../__pycache__/process.cpython-312.pyc | Bin 2540 -> 2521 bytes .../__pycache__/process.cpython-314.pyc | Bin 3071 -> 0 bytes deepity_build/cli.py | 15 +- deepity_build/cmake_runner.py | 3 + deepity_build/config.py | 1 + .../__pycache__/__init__.cpython-312.pyc | Bin 2314 -> 2297 bytes .../__pycache__/__init__.cpython-314.pyc | Bin 2763 -> 0 bytes .../__pycache__/base.cpython-312.pyc | Bin 3583 -> 3566 bytes .../__pycache__/base.cpython-314.pyc | Bin 5945 -> 0 bytes .../plain_reporter.cpython-312.pyc | Bin 10618 -> 0 bytes .../__pycache__/rich_reporter.cpython-312.pyc | Bin 16552 -> 16541 bytes .../__pycache__/rich_reporter.cpython-314.pyc | Bin 19742 -> 0 bytes .../__pycache__/__init__.cpython-314.pyc | Bin 9747 -> 0 bytes include/deepity/backend/Backend.h | 10 + include/deepity/backend/CPUBackend.h | 42 ++ include/deepity/backend/CUDABackend.h | 42 ++ include/deepity/backend/GPUBackend.h | 0 logs/build.log | 496 ++++++++---------- .../ConvolutionalPCN.cpython-312.pyc | Bin 5728 -> 5707 bytes .../ConvolutionalPCN.cpython-314.pyc | Bin 7518 -> 0 bytes pydeepity/__pycache__/DKPPCN.cpython-312.pyc | Bin 7712 -> 7695 bytes pydeepity/__pycache__/DKPPCN.cpython-314.pyc | Bin 9288 -> 0 bytes .../GaussSeidelPCN.cpython-312.pyc | Bin 6265 -> 6243 bytes .../GaussSeidelPCN.cpython-314.pyc | Bin 8083 -> 0 bytes .../__pycache__/SequentialPCN.cpython-312.pyc | Bin 11495 -> 11587 bytes .../__pycache__/SequentialPCN.cpython-314.pyc | Bin 13800 -> 0 bytes .../SimpleConvolutionalPCN.cpython-312.pyc | Bin 7839 -> 7085 bytes .../SimpleConvolutionalPCN.cpython-314.pyc | Bin 10046 -> 0 bytes .../__pycache__/SimplePCN.cpython-312.pyc | Bin 11800 -> 11703 bytes .../__pycache__/SimplePCN.cpython-314.pyc | Bin 13758 -> 0 bytes .../__pycache__/__init__.cpython-312.pyc | Bin 1117 -> 1100 bytes .../__pycache__/__init__.cpython-314.pyc | Bin 1112 -> 0 bytes .../__pycache__/_backend.cpython-312.pyc | Bin 439 -> 422 bytes .../__pycache__/_backend.cpython-314.pyc | Bin 443 -> 0 bytes pydeepity/__pycache__/utils.cpython-312.pyc | Bin 3578 -> 3543 bytes pydeepity/__pycache__/utils.cpython-314.pyc | Bin 3932 -> 0 bytes pydeepity/_backend.py | 8 +- src/backend/Backend.cpp | 28 + src/backend/CPUBackend.cpp | 159 ++++++ src/backend/CUDABackend.cu | 9 + src/backend/Tensor.cpp | 1 + 49 files changed, 514 insertions(+), 300 deletions(-) delete mode 100644 deepity_build/__pycache__/cli.cpython-314.pyc delete mode 100644 deepity_build/__pycache__/cmake_runner.cpython-314.pyc delete mode 100644 deepity_build/__pycache__/config.cpython-314.pyc delete mode 100644 deepity_build/__pycache__/git_info.cpython-314.pyc delete mode 100644 deepity_build/__pycache__/process.cpython-314.pyc delete mode 100644 deepity_build/reporting/__pycache__/__init__.cpython-314.pyc delete mode 100644 deepity_build/reporting/__pycache__/base.cpython-314.pyc delete mode 100644 deepity_build/reporting/__pycache__/plain_reporter.cpython-312.pyc delete mode 100644 deepity_build/reporting/__pycache__/rich_reporter.cpython-314.pyc delete mode 100644 experiments/step-tuning/pydeepity/__pycache__/__init__.cpython-314.pyc create mode 100644 include/deepity/backend/CUDABackend.h delete mode 100644 include/deepity/backend/GPUBackend.h delete mode 100644 pydeepity/__pycache__/ConvolutionalPCN.cpython-314.pyc delete mode 100644 pydeepity/__pycache__/DKPPCN.cpython-314.pyc delete mode 100644 pydeepity/__pycache__/GaussSeidelPCN.cpython-314.pyc delete mode 100644 pydeepity/__pycache__/SequentialPCN.cpython-314.pyc delete mode 100644 pydeepity/__pycache__/SimpleConvolutionalPCN.cpython-314.pyc delete mode 100644 pydeepity/__pycache__/SimplePCN.cpython-314.pyc delete mode 100644 pydeepity/__pycache__/__init__.cpython-314.pyc delete mode 100644 pydeepity/__pycache__/_backend.cpython-314.pyc delete mode 100644 pydeepity/__pycache__/utils.cpython-314.pyc create mode 100644 src/backend/Backend.cpp create mode 100644 src/backend/CPUBackend.cpp create mode 100644 src/backend/CUDABackend.cu create mode 100644 src/backend/Tensor.cpp diff --git a/deepity_build/__pycache__/cli.cpython-312.pyc b/deepity_build/__pycache__/cli.cpython-312.pyc index 93f7c10bc66d9c5d3e2fe93d6f3aa912495bf21f..3d89c8d02942878fcb63ddbbe269a3e04b9465ba 100644 GIT binary patch delta 1664 zcmZWpT}%{L6rQ{Dw|}#{m4(>_78U_VwgR_?O(&q9!eEqM?B%+I?tilWN;&A51J;tdK1arfFl+w22LUFg8tk?<^}#dz1O* zn=|L$d(OGvxjzs8_?F|2AXq6pzJ34FK&0)OV}|yv4Z?U_3vO4GM87r~lymR!{^9

MXkNs1rENM>Zq_TO3XiaOg{?hm)IicAyNooNlz+>805Lzrj;ZFJRG!&Lu`? z`b_AEs|#e)cU-qPyDJfasuCWI#Sh2Oc~J&D`b(_nv=7G&Q#b+-s98P%<9+>6CCd#( z;GhCk9`EbV4d}%@Yt1b~ykQg@cH&31rMo;8CaKuJ;BCx!8?U@Uk>+XvIbiR0$raTR3O-Wt*qT93JuFkltr+pcBYl=B#d0=y%E1k3X z<`^F=!Rzo}%>UBC49M|WFk{cLS1HNVE!ojT>2814rHp7W z67C&}3@S#xk{uF1xP?FrJt_?~Ktfdl8ip*Z4JV?CWP%6Dmo>KlNnRx}`O#)yxpfS) z`}k20A=$Uzwd5_E9Gmyn{^=F$+#^az-}R+IGAzk4yJL?(uq_@>BGVI6L^+n_lPQA7d=YP`jE~Txc zi={#3Q#V8ek|quN#c=#vuAVp~ua^8p_eMAcQhP z=!uHULA}$~QAB@L#CDYPk_oq11S&C2g;&wemQL;{S^W}vucg&|1XGEFpQELg%5u_M zkSGfpiabGBfIu~Ylc=nFq zmxRix?bEgMLJL}K9i;DB0;p?akfy8+=;+4FB7V?I9 zv%LjOpdIbwn^sTII1b|wNcFPja5#q33WqYnDnavzwEc}5VGZ_4cR+lKoIE+Ta55t-a0{kHA4}bWhh{+6z#&hna;Q8^K=RWUy z-+SKoyzgCq?YF_AC6mcO;dQQUD)vvyr$zJh5np?2@QfrvDR?s6I}m&hreb1mMhZqm zF&)he74}fiU^E^HW*^XhuwWE@r{6Bk8z`&uf!=xZ^!3`Y@H>y+M@xnoz+~fm8Lb(i z(bb}-X+~zy&qXc=o3WAak|i^-gDU$!PE#_As*0_ELC1>S7zY%bF5WFomxc;CW0WZB{&3tr3g| zR`|+gzH;it9A7iqv8m-(v=#TY6&sevRxGY%iz{bo_>|TJOuuPcH6a}_RvNx<+Q2HU5=T+Mal%NR+JX)wc zTO{z%h*fx;KpgpO-u5)17YINM0)Xf7CXvu~C7-oNlzRyErjmWp{!kxGB|>4?FTp|V zvPyiTXu;N_`^#c9a(T*_ZFqD{3zFHpju8Nc(b2j(kVL_{JBC^8`#bTUK)lm!*wmM9 zQ31fVf$USx+rX&MysDse;r%j2c&G{m&#}X+3_FK|oJ68~Ch!u@_^R#}mD#I=OwI4| z5s<-Fb;)2))ySF{Q_%N!ctX~oS3IX_O^id!MP+EwQ%}<)4BGOPv#iX<%F!%uMpa%D z&qml`T-s~8aUHtHnNgRwl;&g(g}fH4CU$5)W`Rvp5d)6?uD6-j^?``-E#rt5Kj*yF z7L82HjI5pCJrSUWD15#e)a`Mj)DMRT-1JIyLjpwwxZ=m%?t3^}D3Q!rL z@HHIS*+u4&_`{tyZK1u;wzxQjv3V^GJ{Zf{dNoJoYEN%##mgh)l=7;r^4 z`0|^P9B&=%*f7=3bmU9|5}F6}IekN(0&E1JTg`4pAwMc7`?<1ezoy5I`CY zNknGW^lY-GJg$-KY?V2c-CAuWsnyhOs$4axij%5TWoEZFlggtA5*lFIoLZ&UmC632 zN8VE6U-{1MZUCeyOPOg(x4Z8>eeb#FzRvm1!BD-&$>I7p>9uQ1Cpqr#=*76q+JouQ zfQjQ)xOX_wG{sGt6jL`n*G<(ab?k0d%UFpj*E=SWpK?!n6whS6Qa|Zcypuk~hjNo>nQEA9R2nCnl%~mMrJ3bfr&=akl~(p_ zn`)c%EB?uLrJcRor}j;DC>`CL=xF1VPSJ_9OKcTgZJg+CgN-wcw|9J zNpe(4%c@(EvWig}Fsq(1aeOi|BVL-Ax^VWg+I;5h#Mm1bu0(3HRoAQ3%s8tYt-BW_ zC9;rEB8k*|I#8!J%ehn}7R@MVxXMfmphPY@0LfuhldqzmSs{hQ&)McJoIp3(=2_#t{JQGRK1VtUv+a~j;Kd5_2`0#O}$*8 zF8?>##hj8zhU6viDGBq@L^3O+Q^HjUEqya9%%^1m9f$|hspPUSpGZnsVNeP!goH#Y zn}|z7)Hxw3u|m6qU1rvM4#LjYJhiPR!*L3G`@Q5^||zB6S@i3<=}(0ez3bMoWsk3}$0eXc|+Y zE}c~^vBjlyT;;QvDysc*Wa_Fob>%EZnV-+4Vk)1(tWoXKtr#1*EJ=P~U_d=GvY1|y zM&xu>Iy^EXr!}tQl9Wj(%OmQ5z427Ni#%K4*BIr~YPVEs=fk|gb^HSRw!x?~%o|iQO-5ez zZfqbk&DA{PNwf~TjAxF^Si+W>w(7g_WYkb;AN_;WXmr^Xu5w8-&`PT^wHe$kLb{@T zM~l|5b*96pr+WWJi|A`72U?m)yDozktJy3%L}v|OR-@$*AFlIpZ`g8mAud-pfl6Hah!wO52}pqk||@K?KYYdU9&yLbGS0SHZ895oyL#C)yt+o~DpegS- z6w1iyc{zGBKQwdhl5jIEUr(l^aRIsUTuh1!skC5_=sR5^CFDKH?jRMx1@ZEYL}&doJ+{qEO(R-3)t6a`??v;qB#ZIYc3TJ2~#O- zSjnVt;X?Q#_D*Unfjw_QP||7Ik7XemkMFDsMZ(P3mB~=v7nkOvxun8)CmvPHk|Zxk z`MO{_-w+H21u%3lb<-JWgdi4>~J^8zTZ;B3PdD#fnnJ9jWH;DF5}B?VHV z9^`$M9>lP5BRlV}G^BAav14QTln`iA-I;}SggA@D6SB%D6In&&7n12Y)h*^yN@7Vm zE6X^Vt=eG(5icfUi>f)hoK@|aWK_X1uq4w#r`jOXbn=FzS~F1@2aak>WFyoMP%EfG zx`oQKs#QBPR7*xi_f&IIN~z9lZVtS~q-<6-<7f^zRI8>gs4ksn#-E2?FilvsS(S}{ zgz~5dGA&Eh5ySbDz&rRz=afuNQB4`u99xQK9Vj0V^h>5AL2Zh#@raNV^oFwXDU{Fz z&VI|Z&*)$CvED76O)?ScD4o9)Lpn@_A<{I$!!Rll!X|NLh|pLMOf z&pk5Nd7ax_ox_QvrlsSP3m;#&*S*<%WaZ*RTl0pk^ItXndFP*Xe%e;-Il1L){lxvT zd+q9`uYc9@w=Un7zjN)4f8i?mhYJ3oO@FZDA1(Mt*H66m!2jCnRN3nm4K8oawNy>}y@UbywLon;czVnavjKS5D62 zeSiA<(|3=p#UJp!4_ys+Ub_8K$R=)}bpQDM!S$oBuDi!~NX_D0clX^3-kK(fL;=EmzxzAst2-qOFVJscU= za^!s=+M#mLVaz%+3Ps}SYNiNv@q{jj434%v)eqKrkryqYI?;N9M+>-cRP9Y|6YaAV zh4pEwhjH-Hp?5-8eV$me3xnXspXVEVcr>1BQ=?{5mG62O z7P54nN()N~_4>v39kiwysA;<|ai7>>NdJkw?i9Pk{h;g?g_^eamVFOs4~V^D-&04a zA9W6h2Tx$G(DlUD2SFEj2D(F_8+r!1VbBGifi48Pk!PSg47ww`=qgI3);CJS6&0FP z+?mE|iSblX41dZLlx8D;rqy_=-m4a^kDK@iiWn5z89EVmU9+@2*~QdlwxNheFPg+- z8Ov-%4cz-%Hy5_ls_{_BDqo1B_-4ORclG{6`sY9|ROyVTC(<7WeMS8$f`D7wNq`e z#dHFiOWqm`UQ5qqq1#={cTMM(=3qsq=LMW0&>y8Fv@_ro0+Eo>OAW~9!L`aSO394M z&n2T!sV-%t)Tu8__?M^AZ4%c!n1cC_H2?bs^I8kSSoqDOBNG?LggFQw`nDtt-a2_acziT?<5(ah z%mAK=&Y`c-7>%&7l*WMS`~n$~(@TV02oR|v{Qi-{!ZLvwA>oXsj98;qDQWY%Y4b9w2EMZ8O1pAVG+?Me{YAKXUncBBQMxk`x1%uyC2wFv9vmrv(kL3M*4m0(Bbetg2>KRR%LQoeE;k z$MIn{KfJS2nkt!P9ZM3VTY$^5*b6YL;<==hw*`ZQhNbd{L}@8~gGLkUhplE9O+eAo zSV%e`#6c z=9RJhN3^wqTg|*QY{e`S3al$7ZwUrr$wXg=J~+b+RVjEb2PiNuT*B_Ql*lub_LRm3 zeMe9srVB=i*^g{b7^Y1Cg_wpkX#g6+umb_xh^n@EbRkL~lgW(f6H8-{ixD*~Dmj{p zEskJ)C4=)c;iOb-Ss)v2m}Hj%6%#~O$Yr&Gj4G@+B%B)`9~md}oo!2)MCshZB9>Y_ zox^g+5Cds8#7NeuFsR&2YqBLfn8XM$6Lv6-Q3Xe!T`i$tBblcSPwN+q@2D^@-9*)S zurwD1R4QeMneC4P8ZAXe7l}c9AIdw0L4D5;s9e5I5b_QI0BGXYDeZA7dm~1Vje|NT zvS82$T2;GYPN?-`@&c3i3|SXIy#XLZP|EUDGa1F+C4dGA7^Gwog8^8znH)BMSmbIa zkuT+x9Bn}n0Gvq(@rD#xkkh%0>dpfG)|<_$PG-PRD%!1m78Dr%3N1(U73Kz zYH^kLwgTT)wl|mS8_K@+a>Ks0bL+moZ3k!Zf5llW)@^qUm5;1jk8jJ@UGfDse1T2h zP}wuQ?PA1M5ZBib*Rv}3l^Z(N-dOkb)>K)~0SIUqEi{a7HXL7ht<2Y#_>KbKQMNah zy`5!GN4dUro3Hb9Z+kdP=U1E)HMhb7HmH0CRr5A)@?0|MWDbxu*O0XtWF1sl|IW(v zp5ME&&bJl$Ll1eUAqrpF(^U3$lzq)*&*T&O*2HMQ>ZKjss+A{d@Vv5ZW1``~8BF+! zwD(i}o)?}_+Q3Sme?lqa4ZYdB=63e&^b<#F7UOFb2pY;ods9t+LcY$ zft524ZLX579skyC?OSbqrM97swxM6N1y|3N`-lI0_RnUE&rKEkr#_nfxx6~DY1M9;VD!@7^r69V)aPT5la%ZyYYVgOoe9wouxCxUm25de@Ql z#-l~|u^qXk{gdnXH#)iAHMQROTG4$T#XmgvrHAu}zN+UO?mPAm>~|YJbTJ4T51n7zW$qKhe=QVrk?u737fZ1XSxa0|xsXW;T)xA}| zM42vjVn&!!q&jdQ>5pDbw}}|K4WZxARaBd3 zdb#XrxbxQSx5}9CEdzy?0pnfI_ILiN=SQAmbAP#Y-zQ5SEfpJk%bh~0^H`ztSkZs1 zT;F)-+U;xD>ppRRAsxW)t!-$#QJ03|0inqbx%o z%e&^FdF$V@<*4!mK0b9l{%_Qsy}~;2n2VT9ub95nZe@4dwwYx-bhT_-*b_DsU(>ct zdv+`A=4k<{P z<=euRd0={Qm+iT@FPg4cGWILB7fcaTt}bVe)I=EIi&`GE3b`3lSaw|o7sirYxXN0z z&FG7&mfWBPGO_2f-wWL{zA@^L;{mR(%A((Ep~J1VXZ^Fy1~+0|)#WCdGLA4m-=j-v z52R|Q;@~vC=PI%?_*(eajIW9d^QtM#i)I5E%s9igZ+BVFR*Xa)euq1G6YxFoBg`M$Oux})^^SJ*wGC8l zz{uYVal&}GtF)+%mglZ3?5I*)^noV0ivX|2T)_OX&5V3o`Lp&L(mZ0+GkeTevl+9Ijr$AV z{Nn1LiZ)}9>E=4%`=GyS4u#G@Ze)w>x(rcbzC8PECq;=})ltbftD_dCmGz2FQghom zMj3a;$41E>_WjPI)bNc_8n0)tQ0A$QQdwiQjNlatc&MdkZO4t6L%k)L)OXoJ zpX;zbla5>Kiq?c$uf0_&wPkZL7Kq?sx;CN&8P_uDLaKxEG#RsCMFco7M%8+a*r+AY zC^L2OU1DH`F_26zL^R-!jiLN~$~7YtPqiDfO?V9t=MQiQ2`u)(f;QP|aDgQxIc^%ZQ4gyQb2&M@Vh0vZ6_eDw6*KpD{)B zyI9ExsN2W6+I|pP8Q=1Du7&RhHoc=OljXkS8+|X`wU&IshEG^O6JDA6wWHw!f7#n| z=iS@ymc0E1Z-2>qaKn4B>~1c(2MX?iqI(de9a|a&xWsO)7QId`J(UnEqD9cz^1#W+}v4e zK3Hfz_?5-n?=0ARR$X7(IY-mo)Axpp_K`1`y|mFXu-P(r|G5V(hgY2sU9D?9MOO!t zVo$-{148%6mfL%0@`K5{-zmC#9@>2+dw0R!ea~665B=KfXQZ74cjwwWpEVWTr_1if z$7vJiY`pE?zGeCrKI`|b9p9`!uxkFI_hi}aWy}r~ng_saAD9(ZU3b5uv3p|ee9=Dm zMUABI_djSJSp}f6n_WOJp}pC46zzw~#Js;?_pj~$^sS=(<;QQ=fzdnuJN^&+UuWq^ z`N^2?!hXxo_%Ro5f8{$nfZNYcG$H*j&Ep(y|82kP!ieK{2^iUB|7rNiAE520+H0@j zFw2s@LHJ3d=?cs>O4qJ$v-|zH_FlDELCP-LJ8_~~9=(YgjEMBwKvS`<=2SNA1)%hB z)hLd=Koij%s++A?O0#q*u*<0T+bp7)iVb9Z1&_lXc_MFUt1OCxHk@6TuIuAWfn}xZ z5TxZATtHu|Sc7{(DA4cg_OiojzOU9=3B}ZOj({!tHxRE#?heuim}6iVsC_&UqsTeK z!xGfJ8{`wJ_!JEP2R?LO%fXF9J0#p8-*l*u-!YkhI~a9B`UF9q7BV)^91KrjmJv&OECWv1zP641l<_JjG=*I zWscp9=f@0fkA&_km4!U&%dGa1WK-M+@$wI6RnN$IjX7 zOEyBs*T#yrt}VM~z5ey0T|`_W!_Qkw4F?Jh2a5LIE!Na8_{-I86RVhAJW?$8|36@y ztVQ`HN*FNy2lO^4?3Av1ySLfH{ZTEW$|7jlMK~5?@h_1^ z?X{>{jg`6MCk4h{#`aL^;;zfEtm5aG< z3+ck%-Jm^H@e_txjLtS0`LN#g!DE5{l}H|=9NROZ7i(hdEFxbUyq~sTw(k|Tt|7Sw zC*Q;wGZgf{2eu~Kah_D-ld4=GxZZ)1&T5YFgc4_M*uChYzZ!cOzsCyOkD2J#SV?=* zm$9Lq3I<`+!d2~KSY|j2dyJB+y;skT>Z6Wx%LRm^Zm-y`XF*8gs15sMgjAog5`-d*w4R3F8og&n`o8)85%^8e0m{E;rRHm61mjV*Ylcj972I=y7{#wr_2XW z!f3Z6?jcQkdA^M{ej_ODRTesOA-y0B=506xDNZYI3$VT2IlV(sB_Uv`m>%`v;aePi zbrjq?LcwoVrmP~mo{$RWi;xM&d$z4j-M55@htP{GHY)p4Xh&SL2|FGMl6E}a()lGN zYrWfayO(9R`wGj{_B?$Lzz5pTs8~1t7hYa8t#FskGpKI&MX}6PbbOCBOOe3XHFsa4 zIm}@d^3|W9usOO1W$n-Go={hf<_TXA{%Te-yg}bSS42?gv&E*GqVm57tMY$9lIKsK zJ}u-e0&dQHozppu-zM^DMEl_<6>>t65Owt*iI@gUHO;Gb!VB21Kxpa1LSNCsB<@kU zf9Eg}sSYjBAc-LPKPz0>({ktS+i$NOyWh6@cFA+3=sEJ~>t(m+&biy??#9=iTRV8K zb-j78=nfc}6CYlxd3W|_({M+yx`8rFc!k?TR>$d!~+ckFB0sd`>_JA&#mH!=z z^2ex$q$3eJCmb0$dTOUKJb3Qx^jUEXzxmU2Hxh+vH!>#>w)GyWVYo?*<7bpi zP;!|P@^&(`Ll)>MM9EJmIfW$PU?DG>Tb=1pRj|iKBSN+Q6;a=#ifkNxRUoJhczxYL4|7P340^huH7C*)CHr+k`gXGG&GVgl- z;(Hg@MprHtd70)%Ec=*WZk;`{s)F*kd#23_Z4DH(+}KB5>S|cTU|o zwR&pH-g9r@)ApkM+%N6*2(}?TZ(qT-uVm{f*m_E~-VIxC4gY~cYk(|&Betc)cNO@q zd+|>Z^wL%2PccpNitfB&tYUt3?2AMn{P_?El#MZRl`Z(Fk#`EEvcqR5{# z=%f;Va=b;q<{6|;b2hDS1NRGWl^D{4A8 zX@*)57A-Z^Zv2Xw;k7J)if~B=l`>=#VE_zQ#sQGZOuym|{2k~0oNN7@^L)-ZKj$2ub1wY* z73cq)Yk16COm&YqB#+&k>8z>1oxulx%f0d|uJ1P}^Zus(i0Rnlc@zB(<{vKbh#Yw= pFo*L$oG~{74S9TsROjvAGjY6oHD5Hhhu5r diff --git a/deepity_build/__pycache__/cmake_runner.cpython-312.pyc b/deepity_build/__pycache__/cmake_runner.cpython-312.pyc index 4f35c5ff47410b04be53bf91af5dd2cfe888fe5c..19af17885c58dc0ba77ac34694005c3aabe60961 100644 GIT binary patch delta 1081 zcmYLHOH3O_7@o0ry$^dI_JZyBvFijBf_a!g3l~U=LTZUbkKWLRmx|C{gs z=bM>-W;^wnAwQQS5s>fOufMXS!1wZZFgcJx#iE^=$>-OW?OPf1FwQTmFBi-V{u8cv z0?dcb?pw{_5-jfID#2zGtm*Pvt*T+52=KX5gLa)ZJrS|Pna%l zbH4nBonJNaheA(EB^>4E7gusO^7A>gRBCxkw@NL3t&p=nDxwu!6&j&QVi+QSiS+<* z7r1g4{#5)j`X&l^$>sgq;(f6^^F?3z{2t%4@+fB()tCp9_!C8u`^#+zEYuBG?* zfj>C$bLXB@{)A*7ynP^oaH4v>a=q5q=uU5muYy`NR*6;hie6)C3yolZGnlCdGmYTc zEuR}sJ!E#ngD=C=JKPWByCc(E{@4Dn8;)&de^XApGJ0yW4I|Su?u^%s@uo3VH>Q4K z>&C@$z)hTaI9X3jlm%ByG_~Hk*1M;jcJ;odK2q058v1BCyB|=tXYbFtp|l$scGUq_ z9dgyQs}BAxaZ&jIaFX&y1!}}qwW?SVcax(#6FdF8@yR`P%Js{4XOTkMv`Ah%wgSJv zF+42k>@<-sJ-I4%!NfU2`x%rZ0qGPB3g;=jgYU^nc81Vv_#3&D9iyU;0#D#yv1Fr^ zle$t|&7-%d`YwfwG$j<4OZEpOKS!*Or_mhB;yESt-XNvq(E*eu(Bjr|_Q!?g1;kM4 zc!o@e^sW)zGNOBV1llJ;65%a=a;0dl7xL%OW&A=J>=GxS@(+Rn^c_S+=xbO05Bt6k AsQ>@~ delta 1452 zcmZ`(O-vg{6y7zq*IxVsVQqu?@n9nXiE$F3kxD3q8iQgg5~aZzAXRV%ePrt-$(kV-xEL{$z|PMtB@IFZU6-n@Bl zzHh$yd2>JX5O{uZyJ-To`#*o2`^o*S=W8-H8XrmBQh2DOl5Bb@bpy&dKCLP#j^_)4 zde7|V7nTK)OX(FdZ~4CA@sy>VoUw4O6#@=w)?5oFeYJ6J&GH%Xx#cVJG)@v{%=2hO zj?vFsyN4Qb{#QBh2zH%;^J-{5!hw|9YQl0s)2sJf(^4OzInqd2MO2;jjOlUB+_}t4 zLe-F&$pQUWYbX?cG43(FdY|jnUA7R}uSad64gWH^N-9>8;94~M3b9UU6kO%}>kiEZ zr_~E4O0${Al8c(LdjEehF0j@}5bI9O$+gxzTBCCp>zGqdAGOhT*o0D-k(~A!#~B-L zKMXGH0B2NFTTRHo)4yg^WC3$g&DQ0%m222Yr$ZZcJbAkAm=Y zVm5Je;@w0z3S^ad-60EHH17^@LY00l9NC{>#jL*l@KEQ>rVfix@i zs(@cDKpw4Aqmm^csA$ci6veea^n8 zH&P3MHOs&>bfPmyL6Ma*RZBcxC1&+C>Nm1YPde`C-#EJc4az;G>L`e;iUjAYc79dm zB~Cwfl$omti;G723zf(F#4$~HefsatPi@Dogrix1K)<)~Cq+NdKG3#`PpGb^jZKEj z`!M-oa@V?>+1faC^`5whAo!x=la5E3a$9UOdE^W1%x%x@+}*zWXmT%H_6<~gLnYr( z+4tJ!lo9ONn>+|!dJ;_RPk*;^Fr3(&`NJE07VO%hp0;!zMSAz>a%89y87W0ZDv^m& zWFqrJq7=Ed^|k?`ds+!x-MVH3Kqb&u3iKTY&KuzimGE#WJX{W6*`=N}wd~N_wBe5# z{y~GeXfT%zCT1{~j@?wpxf6nFZuz^3VA>5Pu#?dk z9C4Jrtp_{`IjDc+2}Xx7B!SH&GS8hNdZo7&72ZMYSPrzCIXI=4Jm4lSUWjUtfap6R zs#Jkhv!bv7tvFOic<>NkZ%sAC%YI2fFAm_XJ$H=eWo}vI$KZ?}_eL4l>!kOeRnke? LPg-fxUYq$JfURFh diff --git a/deepity_build/__pycache__/cmake_runner.cpython-314.pyc b/deepity_build/__pycache__/cmake_runner.cpython-314.pyc deleted file mode 100644 index 6775dbae4bf63c2e03a6008c6ddeb181f66454ac..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4830 zcmb7IO>7&-72f6FE|=85Wa`h>T8b^x5=q5Ug2b)@Qxqjp)(^Xui8xlgS&=Jg9cp*6 zyR>Y=KvkeX?Obdh3adS|&Z+V#I_8*Tj;l%r*-eTVNP;3j4@#B5KrZc@A(xa`xoHQ~ z%$u3FGw;3c&Ac}nsIOxPD9;MNyOpUUh_6w{F3u7`9^`@8AUMJxZW0`sAg3ILqX*HB zgwt^1)@8VG>o(lbIuo8L%AkOULg2+kcKI8T7cHX009$5GIuQ7=mxKF$k0 z2G999AJBeG*Kv+MB3PflxRBFiofX)zoRrPP6nR-%VI|p6SYuUWg|v|qvg`-xHQ__? z12(O!tqF35%}TPUhl8Z$6J%L21Vd6}eGdArq+qO~@k6VnNzf7fq2ho)P@X*K24aIq z5jP!g{e~biFxE}{l3XP@l5<2z5Y71=;r3=Mbyqj?DsmL5JnhS96AxwcP8gQW|j5>S0H0HdaHnU` z2w=M)lVPyuKs@BkNqgo{t2&WOFIK(-E~HFNvE6`k!aUNvruc~q4$cL0NUxewaU0kJ z+{Qgiavsc20YAQjj(+e;ej4}@d^LshzV)tsL=Uab05|fVn)1rkeJN+Mv4$DCNYb!Z zb)_6!$J@})3?nU4t|UUJrf~k01M%C|>UkAk9ms|bZB429)|os}!vNii{5}6gj(X7L zm~W4+zu0cCp6s-#YRfestH#L;h4&#V`~IuE@5w5!O>vD}Q}~efi>K_FBx_UE7Ot0O z+fu4~vDcnSzGhR^Hfe8P9U{0Eu63k*Z6(oBsVO7nzUoQ2xwdKYQbmUq@ZfF&;!<l0udx}2lHz{=g zb>;t2x!f+|fCm@9!X_u@gM)1EWPC2pMHk||gRG*lz1ObBd)ZKkg(zT1@=8w9S4Ew{ zmprQqy3Q)gIGh{nN(lF18FpRKK3vW!>o6nf5ZY1X&uJo?7P46uMj)t$nJBBnf>{{F z^I?#QT~d^+D9FGl_ZzHuN7Y2w0oSCANluH*l47i48AP4cR|Qxf(;(YAdcyu)1T>TKt#Lr%t8MT~H=h^B+X`|%zg{k>Dek?IJo|v1w zI+|~*azwe<6hAQ&MZ+MUEehC?pPe5E`2t4=$D<2TemudA<~yqsvsY6w{?JMghb4LV z0w_JHHGr3a5we23VmV?livYE<(o&v*N?B9Ghz5D)RZ*Tz@-V{BB*wTXcYPG%I&J%d zaPyQ*mU z%Cx8(y5&meGJ@q=$_l#Gpx!lB6`5axqy@ZGx0swP-QkzB0?-~o1;I0xA04BNU_}^H>L3(0NtSr8_~N1_!>BuV^StXz0eZoawcU#b6k4_6EsKnup~rUk)8I zz63snum!X&e>Qu24N+KdFPyfF1bT=%y6DcHJt}5yJ89BWc>I&DPrA0cK5qT=jLBX$>1dJZewzC8jgN0^$M?>>{riPa zb7ptkq$i5Z`Rz9g!wLAA!-;1jpXbc;*GxK9WO|-b1@^)Yd%^4(HtCW57RKk;+X{pgQ1Z zQXLhx$Tn3N8rvBfGb1sxWqgm0e?u|9z4z;TkJg`4r@rzv7X1wmryopjx;9rH{c6wO z|AU|CWdGFthwi5C(_gj)p1k+?y(c#x-+Vg0)w|mkDzpvnv<>gJy>Wk{*!kMl zxY;>W=!~14@$Jb^Z|{u6?_c?QL+6XmQ;(>xT26l1*T2Q=_6-;MF7EVQEc8Wp`l6r3 zpZ6sm%@otY%SaaCVm*rUev=nr>n&p&BW86v}O8;U_$>g%)I~sbZB6reJO1$qR wN%F76b61ATZlX-=wLk<;W<-ZkPXj5;3}fW&8J1}2G* LOh8tVFwhVHK@e=? delta 438 zcmcaBvqFaVG%qg~0}#aCoRal&Bkx%zJu_#kn9$^YOit#=7t#URrO8yJ1*D2(K!on(CmaThA(KTpGuaFw0xg_+j1iMJbLOy_Km=5| zj2L4!M{o%-vVk?~PA=d!V@#O5oZHyX3Z#S;NNDmF@qyUGT9LS^ye82wLr?)=N5b_sn?0)?rqwM@07=vt(86wDWq7k9oB$_fG zxfm7EaRPi~UcDF-G08{g;};VmF;13ZFA`1FVy`LTI(Y)^_$bj5vyo6T8~%t$YRRP} z^d_}FEv2Qk{@2v7OUp=kMjHTmwnsiF<%8OgHmr?w<)x_T({f8GE&nnspk+k*nOYnG zEk^fg(XWjiBZY&_-@(+>*{1DS)b!k{z1-x6us!z;%5F2hCTh03OhwHQ)MA41y6rNb zzOo8}S7`Z)vBIeBQ^Q|j78PEqf(sStRgX5fXENWXHN&+WTm$~p^(bczj|&ty-m+~{ z$G!~<));5>&l{h@POx0vvziV|`8BTzYE>BZjVgoXsa?m(a;c_ki)q1pK|Pn+!l&M< zTTB&{AZ@s=C!`HNH2W+=7^Y+R*iaV?QCkEiUgh38bItad$X_&UC_86>*&s`B@^Jc* zn|TohKTwubEh2G7F52PL4i}SLT*R~lAe#)*bKI;gaIb1REQtDo2dZn-S&#zN@90&> zSoVWd-M?+N`LxA+liLkM|30}|hzChs#~HdFq;x$51^9kl|6bE@+C5oauiD%ffH8AD z=mtR#0wBoix{r96x*-I&D@{O6*Ew7OvV$fEv!$9>XC?0WY^DSX-(sflOBQ1dTdeK= zb$PU<&_{}mHJ*T}lH`H$M}ho+JW6Hm&TkKn+`Sf(q1@+cj^}nq25}ekxXsox({%&D zdAr?Am9X{3-1{G#YW@}mFE(9pr>e*4;>G#vv~HN-h3te+%L_M(^t|DR=Pr?&qF@lX z4Fk{en~wCXitQSFjhco_O>X!#x@wCWcn!53hbFLw6)3;xx({%ynzP-#`#%*)`mxuzLr01{9(Td?SJXbo=8D1#TvIqP7(qzZr z%M6sh<#9MF>QzBwe?@o=zvMG6hk>Kl+hgIbSj;pmR-_j^&w@tP0K_EFj7l2j{UALI(j!kvGulxx zpuGT5Z#+n!#Y<3@se~ud@x!E`i?s%XK{&KPz2#N>pdatN4rxNhN44U4PN14i%Lvp8 zgzF$JPg=*T?gxFm>9*Ny1Co;G>Tt;|NF#m_GvR#QAOY@Pfe>~+h&GnJASS1COs%^b zaL*GaV%5frU}lD>uk0KFW`n#>Zbn}Jm=FtON633hP0qwn zzHQ+GOlqt8*X)ta#m|oxe}A<2Fk9M6mb$7wH?H@id>VBsB2kfqpz;##*F}`0&HN;g z8`#ms>?DG9a&uPxx{AOc`E@XR3z~SqXvf~~OYpFWO6Ao8!HK3@ggw{ZI| za1)5w#O87N>uwH%k|(+f{*TaE!mYdLwB736uX3yBLuY+J7R+TRXIN3(5&R77 zx(}q3iTzO2LDhJm7pATp@4i0_By=SJYKzYNpZv(cy?2pPV^fa_kWEYey2k{Adc9_FsBtP#UO)|zYSge^z;X)f%7qKp z=&Hw8s*bl>Tu(xBnZ`FoK?$S5-aya~19Ti99sQ}`-`n$Dz^9F4D@6u#G9Vm!n4R28 zPD&i*jlNpQ@&Ophvq%P!;057$_`HBbMS@`|NCE^6#>E;xjWT%y=a4&xWF85gn_M^C z0izMe+$RZ+^?wBf>Udeg%VK^w|3J-erv~m`+LrAh+0J5#pFj9OJ-D6D-u=dQX6Wvf z?ZNy;`G+|e{>Rb7n-_li{f$eXs1MZRk7Ai85y)&iQ4$|$A=ionhq99$*~yMV~-DzNcxlGe^5rhfS!Dd00CPg zV=dVnB(coBmF-M^d*r3YxN^zE*_{@TWIQZOxv;t;;9G%->lKng7?AwUEQ=?c zWdYZ*sjsg8rUAT?M0X`)MA@n$(gqVatbnP^D;HF*j!0ac*SLneHlMhV+>39RKn`z`{iqPoiDN`MTa(4an8N&Dksnr6?ORF9)H3_#} zo7*Milz5$#C~2XL&{Cb)E}@g91#$~1YI}^Z$+9AHw(ker=3d~l1*9|bxYMv1yMmNn zqrp1yosJXHhGEO4Y#m3~hHJnT3{_ppaP1(5do5wAhfxw?T7)H#my9r-C(RVPo))R| z9-sGXfzTMILa$QWcj`iSf@ag>B2f*3hDf>|hV8@Q|^`vX~6kv24z%R}DE~mPHNpcMCrd$1C-qNh&m8 z zNC&$*j;wa4;vnmcDOI+H5NX2_tf(hH2XK)$1Rp7N#Z*kSqF~arU!|_lDd8dY<13}- zQJn(Xv^kST574)z8m6WbJ0^<0gRFl1m~^5IJFrigel5Yh)o6>H`&*f1M3G-F(P$g? z!!k1Bwb*%^U0Uv=Fp01BGJ4yim*~}qaZ^2kR^?hu#d|O8zV_Ce+?M{|ZtL|q-Rn<` zN3mrt_iFu1Wd9Mj){VQRE~zVUauuMZE$6zoU4a?pNQYRBYCnRL;p4&SPPWFN)SDAo z_kXPAzAj@*)xpI#I0Pf`*9~S8e|pzms1Cyzcrc?4gQt|ULc`~LrL2ksC!cVkGmhhe zA}z5XDWI)q~p;PDm;7c{vfu)-#mc znrPXSk@ZxhL9xXz%zb#ab66x?uU3_#JM1~D*;C~>iE<+(XZ11bvVyarha__-kT-H3%wNnfe#R2iMk zLq=sCm@H}uSEqqi3AegEPRCIeHP8=;v|&>%CZ}H%_HPu9J}Dd(M&)^7&x^s~jln~Y z2M=wh5(C+tH01ifIP~R_%?xDn+Xf5_b(N>yMETp1Z$^Z1c-QDI5?Q~iZyHc2emDAc z?r!o$e($|o8^!TQ#qs;^K3IC1pLvv?`Tb{O?}kx+Y?PmkpL&#ge;X9^7&X`Kmmgex znxA@-p4!xq_XiumkUl+gdRB#p>e1N=?csY0LOG-@%MF}(=GH=-I&zfA=BHx_L~i%L zf-oxIX*`o;s-lq^PrPUvrBB5(`vY1*sT@;G_BuMCC_lp7FL2-$6n=uy*E*=Vdk3E> Z#T`{s`Zoc=mIms+O-b$;II$wDe*#w%n}h%W diff --git a/deepity_build/__pycache__/process.cpython-312.pyc b/deepity_build/__pycache__/process.cpython-312.pyc index 2a14851554002bdca4e395de54004aae8ed672f1..a6f0ce1661d00a52f82faed772fd52e1f5efa8c1 100644 GIT binary patch delta 125 zcmaDOd{dbBG%qg~0}vRznUN*Gk=K<;R!zSsKfgrZB{j7mv!qf#1wzLsm1gFo=ud87 zdd_%ja|SaPGh@T#Vm2F{3oNP^xK$rWNHw@U5D=fvH-)c*W+1)!6x$po#$S`uIg0?lA}YoJ diff --git a/deepity_build/__pycache__/process.cpython-314.pyc b/deepity_build/__pycache__/process.cpython-314.pyc deleted file mode 100644 index f786f413b2e2fde6dbc25f11c2fefe66bde66894..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 3071 zcmcIm-ESMk7N7O5*I%)nq)F4#Hkl-CsGDMkQfQDukSJ{lx3p#(%tOU)I{0f|_Egb@hx7+&tv{X<-OrKU@fDn1@~bCgy{JaJ~$PC`_~17f5-b7tr3 zeEiNiGoDJu5wwY?R<94q2>r+gohAm1u=O4=HKZVl=8+=I2rq?bXo&Hl8Ig*-m#752 zI1_$Jrt%QFCgxL}V|Mysq)4ZQU`(S*cmOH#0J6GiOo;#=<$PR;0UzglLP-GM#rZBJ zG>S6G>cuN%&9ZQXI7P#Cb0vqAHA+o;5qs3MOlle~c9%3_=y>4xodOo`0?zC zOp$5O)&fbugPnve3``9b(0pj}U4(S_cNo1YED4IBgcMPca!mU0FCkdwM_i-m*t%Qo zDp`(3@jLjEV;jCq4C)cv7hOueK_H48DCe@vM*OF9OHSFy5yv$qa(UuhH;UBF>4s4; z>5WdWE_kM;=i1iER&MxRs@m4jP*u_eN3qX+3`h>!+7XTJXfWj$Ic&=k6zUDMj9@;YWq`M%rckN@l z=HRrT6@(5%x&SV4T9}_E?LO^MtLLZwaQ<*11pDQJ05oz6c4wlOJa)+swNI;J_Q{ej zxYbY@13yu!hL*4|lrte;*3Gxdj_!-5O*0~akN|g;P1~b}>xY*dkGL*7Wej7IV3X;< zN2=}-jv>+yf4PMzFg3IhN&V;M-E`mmNUGlZ?mHX32OFdHBMq&Y#?2`HY2Zj>agE#_ zIB`qrt6gX>y4TxZU;N}`eXv2+9$z0_JF*VTCz{cTpZX7dd3G)G#l`iJ+x<`c`%>-P zMrz;tuh(CwFW1e+o6W>YkYf1-p zUOri6<`RMbIt0 zl?82UXpa5oE#HpN&Y1PXVD()qq}_7?ziXCRZBJ8#c}Rh07^a|ALWNMC-M^;=+2mEU zBxF%VeDs=bbRHa7kqTm-h2EZ4*e)$Ys33sUyT<<-L@0&^5uD3{V^2HX9xGx#y=MxB z1+jy2)WEQ);!$k6ShHLQDiqa%{SK5cC=n&iv^)Z(t*F^h-kfDt+2Rf{7foBU zux@ILw&PN>xP6k1OP*zAPB7rR;O2{zfBx#O2?XsWA_g{Ry%*cd*HLwr>Xa)jc95$xst5w z|BoKJ8%@`xkCK~V)H4Jz0{f;`raqi*ocj9A=V#WQ`QrIz;zUzAu@f|%gusB$69VSq z{dqzND*c{h5Twr`*!q|0a~c+Hr_ZQxauWe*m@U?=U*@A7>0m8yEIV`Qz5?bS4{3M> zo@pn7ick<970~dPzbpgsI+TY{Xcj6fgA*BHWE4n7CS%a~B0z(%mp^{FIW&2@@7X5m Xc__on?jeS{`!EUx;nyiZ5 argparse.Namespace: +def parse_args(argv: list[str] | None = None) -> tuple[argparse.Namespace, list[str]]: parser = argparse.ArgumentParser( description="Deepity Cross-Platform Build & Test Runner" ) @@ -190,7 +190,7 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: ), ) - args = parser.parse_args(argv) + args, unknown_args = parser.parse_known_args(argv) if args.list_profiles: for profile in ARCH_PROFILES.values(): @@ -206,14 +206,12 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: args.arch_profile = DEFAULT_ARCH_PROFILE if args.cuda is None: - # Distributed builds default to CPU-only, since a CUDA-linked binary - # isn't portable either. Everything else keeps the old default (ON). args.cuda = args.arch_profile != "distributed" - return args + return args, unknown_args -def build_config_from_args(args: argparse.Namespace) -> BuildConfig: +def build_config_from_args(args: argparse.Namespace, extra_args: list[str]) -> BuildConfig: return BuildConfig( build_type=args.build_type, jobs=args.jobs, @@ -226,6 +224,7 @@ def build_config_from_args(args: argparse.Namespace) -> BuildConfig: clean=args.clean, verbose=args.verbose, pgo=args.pgo, + extra_cmake_args=tuple(extra_args) ) @@ -349,8 +348,8 @@ def _run_pgo_workload(config: BuildConfig) -> None: def main(argv: list[str] | None = None) -> None: - args = parse_args(argv) - config = build_config_from_args(args) + args, extra_args = parse_args(argv) + config = build_config_from_args(args, extra_args) if config.clean and config.build_dir.exists(): shutil.rmtree(config.build_dir, onexc=_rmtree_onexc) diff --git a/deepity_build/cmake_runner.py b/deepity_build/cmake_runner.py index f1bdcd7..84d0c72 100644 --- a/deepity_build/cmake_runner.py +++ b/deepity_build/cmake_runner.py @@ -40,6 +40,9 @@ def configure_command(config: BuildConfig, ninja: str | None, pgo_phase: str | N if profile.msvc_flags: cmd.append(f"-DDEEPITY_MSVC_ARCH_FLAGS={profile.msvc_flags}") + if config.extra_cmake_args: + cmd.extend(config.extra_cmake_args) + if sys.platform == "win32" and ninja: # CC being unset doesn't mean clang isn't in play -- CMake can # auto-detect and pick it up on its own (as this project's own diff --git a/deepity_build/config.py b/deepity_build/config.py index c16e469..d51dca3 100644 --- a/deepity_build/config.py +++ b/deepity_build/config.py @@ -75,6 +75,7 @@ class BuildConfig: clean: bool verbose: bool pgo: bool = False + extra_cmake_args: tuple[str,...] = () build_root: Path = Path("build") @property diff --git a/deepity_build/reporting/__pycache__/__init__.cpython-312.pyc b/deepity_build/reporting/__pycache__/__init__.cpython-312.pyc index 91e0fd977664afd8134f3e2da060d3b12228c42f..b7ffd31f991ed1f779651bec42332ba79f57c597 100644 GIT binary patch delta 73 zcmeAY`YFhJnwOW00SFA<%*dLyk++Rm-B7=B4XzKF$1rh4I$pJsf@jip(0T delta 90 zcmew<*d@e!nwOW00SIDmPRTm8k++T6!pYexCbT%Us5r(eF*!RXpeR2pHMyiXCM7ks mAhV7fK6rQ!$_GUMBkeU#F0-45Dhy~c8V5W+s zl($bRgy<$~23)xQGHu$<6?>VLb-iJ`{zQoezSpMgD&thx-j$GwfQmY!@d=o9eZlyW z?XW}CkGBPsyH1@3TyJ`yw;dZGtSVE_5tWyA*&stVbDE_?+|r2OdiuUjI&Q$aCfLn z^WYs-24;ouEDw*{evQIFxDFNVCZkTBIae4DyQf3Os7=S3K^VGKudTQIYS8j)W8FK; zdP#{4+xG)h5%}SG9G_^pUTr4um)vC-=1#j}-d_qp)Dr(W;C_NX{8ad*VYo>_SRsoL z=XX9Qqy}>%6C+hdyDz|yt^6dtT(xKo{7985*exz1|!26lN zec7deH*68&O=Hmqm3q)%6&{3ava-N~%Mg&TQe&*?igwalRW!a5hXq_(u`Jhjg=LkS z?Wo_fx^HEc#Rov^ApF8T(5#RbL{+ArVzU}pnQM`6~=o$i>7F z@brt_v>;@vD;@AqV2WVNzMZv z^~L(Zgh*`(*Ngg=T)$?8P3Dvojt`a)80Q64FzAL>f}Sl?`7T(J(VPKcg*-NfzV&{Z zSvMx`8WX?tuTP%1GkIcta`x`z?1RFt&hDAD!fZ#IeP9gUoc?lp-6-BMimT_=j3d7d zc8ud4^*G-FdbZ&l0}7Z{Hb>+Qug7^sI+$bVIL3z?*g!l1&m4mjgIT^rB2YGaap4VU zVi*n~@P=-wvWpFYwXLsM@S*mlnSz@83wJGeZvsimRvA1`c+# zga14*hMo~6GZYOBZlt0AH^lef(DURQVx5MIdcR`*Ff(S>%$Zt6 z1rYa1A4gfJ#89U=C?HXK*%chU6ZI`KpK)6RT!JnR1!ms=hGGmhg|Q_s$PVSmQ*3S> z+`#VkY1#drmfatkT#Ms__a`6T1LMr*S0MD(hgb8w23{ONDwMR@Opo^Fd@9M>QL>nR zlYF8qrlz{L6!A=YUY*M{)oo|go-^vJXG)ptzv@udA^%#AkbGEWkh@!feMx4>Dp)C7 zkhOB7Qx9AR`SmPo3My%A0^AD%EZaVm4C;6;)YuXd8dB}pwL0=?!jCaiU8}dppg07) zt$XfOhAagv+a>2@E3kE)HA3dW21!u;CTL2ir}Zv4_0J!^pa&3@2e#F#GW{5e4bmub zBrv?~2)KsU?66)9Y+fq^spJ=s-?!AgkLrH>x<}pl3m?5Kfn|CMm=={*;XekN+Vl4S zN@KGn%B8GKi6}4gBzXo$Bt@AfR6-FI*#0ispsw+yKe1Vb=Qo^jyfsfTT;194pbc10nG8zK$NqrTHsh#q+8Zf3+Ne% zGX-l=VXJ1Bv1IchJSyLVAdNIgMPObaaG8&4)ec#F2Nr7SFOIk24d7tViZ!CA=xE0p zL9OMnX)o)G$w?0-o1JkJ7YKQBP#d|rTL{d=4k Krg9@Eb^ZtR(wlVv diff --git a/deepity_build/reporting/__pycache__/base.cpython-312.pyc b/deepity_build/reporting/__pycache__/base.cpython-312.pyc index f6109544fe541be06d9fd944c2dac33a675d7165..5e9ef2a13ed2501155ae0849f88b5308b0bf28de 100644 GIT binary patch delta 74 zcmew_{Z5+iG%qg~0}vRznUVF0bt9iQle(^cQGR}jzDsIqL1sy%ehP$+Pb$sKNzpG# YEyyn_$;?aF-#n4YoSX612zG%qg~0}#aCoRVe4wvo@9$->UrDkiizwWv78D=|4cCZH%kD>b>KI3^`E nwIH*k5=O@-m1gFo#1y3#=B3AMZeTL!X8bjI2d^0bnsgx+ diff --git a/deepity_build/reporting/__pycache__/base.cpython-314.pyc b/deepity_build/reporting/__pycache__/base.cpython-314.pyc deleted file mode 100644 index 5859bf8c7fc050cb18feaa0eb8eb77d6bbacfa4c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 5945 zcmcIo&2Jn<7O$S}`SAE7{*0Xuh!cm{V;g25jxi=OgdmZ4g%x@cTEQm0?QuIEc0NpX z4Vy%J@ByK{?KQ^{r}&mXfj_}>3ba5%;()k03}UbRUUhX(w;gPc!iw#^-bcOn zx~gaTdb13zaQVNV{*q+uM`D5oo|@Kr7j%o=VWxJI-Oz+~j{1q4i5t4mZy3UmynZuz zBPCMj*a$n%Ok~tJRvYMX6SGa|uts_Re*$&t7!} zw~M0Uh>hABW*N6^u~GC_@Tpo(Ykj4WO1GKNOePX{(n7y&m>Ps8Mp?OkPCX-%Ky{!{ zCxWJcCV{46Xc}l5XeNecfOY}R#?UUH-9UR{XclNM(7qVj4fF)i{utT=bO7jJ4DAIv z1avrt_5mFMdNPKd06GeEEQa<2Jq7f13>^S^2I$!sItcUypfASIA)x1gj>phppyz>J zh@m4uCxBjzp(o9k#He`*8$Rh}OlPykg~N;3F-AXX3H-2^TI(!2TkIotC$aDaV{5?9 zvJbQk&D6|9Uc(Cd&otLdaYxj7^+9$rSZ{Kh0x7j#;}u)DwD1M*YJQ_uaq_(8I)(f# zUi;K33OB#zIGZJLKlry+>!tEqo=d$-)w}ss+jU;qyzli`mI_P9vb>Q%&?`~63v%3E zsq_V2VrGxW&GmoaO`Q9nYKFI;V8!-5{Q^QmZ;rqnyUcB;ScrA%23U^@EVc zp~3S_WB$>HAifR;#)9??gZU5!eE|j^LEpZXxTAg+?q6JA=_{H`NhPcK!4^b0?Y z#}9Q2`NtiUf}R-Umd0OqV!olM4cfuJNl4%D3CwosslXS#GEObGUa8ppzIQToNDQa& z;pd4QKP>nudY*RrphzHVTS4g0HjUZF97tO%hGjd@T0V?9(a0YtmL0olai>_Tu9xoC zx#OJw-ZAnKf0;-q z_Wxwy7=(n`J<~L18W%y@Vx_4domtx-?xj_q=q>qfIdWoetHfTQ6{Cqg7;#Un zvRQV7Y(nu7mcZ|iy!{~KrHGgxjxqZH6 z#h@CD?lCzSxI{WAvtL(Yk8k!{M=^UkYW7^ym}^|#G3FjouT!(H?=`!}H(OFpbs{HhWeHJQ%W^3+FBdC;lPOq!{@PM1*s%pMK6*ch9G7+fNcay-;x>p& zkA{cjUg5aHwJIgo#q}j}FpS}|^qh*7_ml=v$NkBru|T@MMlB{H%KR@xLUAO9T4%ZcKAt!<_b+2oSn={rBmWTln{TY4*J%>mqNSpxu8Zul zWTk6g{X*s0(Y~=KAntB;Fd}_CgdRdn3d)kejzmpW5K|3&Za{dPb{yApze-XI{ zvghMK+N~Fhj_Z2;;pwY7Sg`uN)O zS=}zHkrS4+UgEBxJgn9%i&M_=lSIxDp*Js|AtDPMePHC|9{w(on?yb!@)4206Dbn8 zOXQzKYD8Qj4?w&Ti!OqqRJ3d%cxkncn1NI}ghY*voNxfwM!Bgd8$w-aK=9p&_r^mKc*$SLmR4ti;N zt(QxyoX!c-m3w^bWv==mvBbZ?6pB)pE?m31rfJ`^3qP@m@7dU|sYPw*Kkjb~e^P

*n+H#sGNVqg-B+;Qr zWrj>5Mox{uC=}G&$eUv2HtSXa7O0G*h>I4e3lyl)UjbU6lqEyl!3NEyyFmR9XkBpB)yKyY56Lp3sI; z$?^CE>b$4p>1UJUsj$ncKXLKdvnM0x&Odkh{HY5UBIi$8q4AV46-`I9jKM~GDDpv+ z^ZInkNb5$7W-NR(13qyhhj2mAg)vbR#$1{>=GIy?*EMNO(za`E#In|^Nr*j&WyD^s zP4ggDkoO|4(ROGGVjuD~i2aCthy#fIh-kv1f zz8-O-)}b{Z-hy_Gh?~%E3*xP4*MxW*+HFPLTqBhK(>k?nC~ui*4)3yR&n2SqWMS*s z|M02!%(Rxuqz|T&i5Yb&ZWt*;O^vJPN&w*v6wV{eYTG zsRr7yVKjYOPilHHHls$XSi zsR=_jO*T~ARL#lM8%b&#O=7}>(+SL{PJ>)}G98`J)kHjrzR{$nrgbx|Vgu8eG$u6k z=oB9W>1cdnGL6}cRAyqbhYcN*7=?L2sQ$N&u&PZL>V`;Nj)Ai|idZrNbQF{$`BeB6=P&=NC@VsvHOwU+KBoa@? z(~(GaYZ(jo6w2AOBbfH8kl)%iCtdgF8@D1-@-3Z+f}b~0N^$(OC8yHcDIb48D0Dp{UPM~x_m=!KllZx-M!zsmzD`L9H z72uz${Txmy_WOYG*JX!K%4)z*RmWjsz1Wgc7)P2FFVp%fKGcBss*V$^Xo{@i(mY8~ z^L`-Q@_l5Vj3`teSyNu^Mh}0<&TC5g2p5E~ZzR)#QBQJ~P){W%z+CDVSO1s`hJmbK zb(ne}>rpSKE}0ZZjo2i`v5XcSuw1F6R*^Jp-9U zyk|+YFiT28P+Af#D(o^^>9}?wFo_>j)o7!ZHIWE}m~McXw^cH;UC}{(eux0v+^}_F z|Bd}?4SSXw_RM*Irqt$xT}#p@{@ebw?!o2m!CcqTbwNBPw&YuO{CWGIwC9?;mpn`9 zPu{xy)>`k;<=&&Yo?~cmT-=i1(zf9=q*`-hI*mhS1fuCwd1 zt9C^E(k<2a)_p>t@xA_c`)@vV-@of;f%;s-3#)+_^P$E$ztN6ml}Sfo0TYt0q)#l{~Fsw;e2Zs@AI($#LFeWPf(6(4HoMpeIskk-|2;Vq@e(~#Tph$X(14ZN&6qIV$cEpW4! z=__*}EE~Hqxv_7XLY^kV-6#@Dlkq7 z%}Wcu8@{>OoBd1DTKB+m_du>|5bSWYDc{tx*0guIX>V>z*Ww#X=_0eZ#o80%uifq% zA3tnI(P3Oa{@7vvBN7mQuFKAGqrmJ(*@)wUU8+JF$#5}P%MMW6zS-10ZW#M z3R&}{9c4Ky77jTJf`4QS*ev)Wi`fcwrK8Tes+DTqVkwBFjAUy@QZ|0b$_MeOmWclc zxti_A;Mrj+{_Q_#OOHWPW}4C7a!*CywA=}(Q8&LUayp)dh>B653@Vggl3;|1 zoyd4Jo`5dTwpFk|S?v+@G@nDjF3fvxz5CW5Ub!FIzv!Qnz!cX>hRn26|auhA+wCjF~CVvom^y(267&jb4Vl!oE zG!!Xaq&|aC=*7Poh>fsoWEetoJ4*Q7IE+;3EvS#g5&#k?**v3>O7KAdH0#?0M!dh(M+0ZD5 zPVAni`_!<=_FyXk%WEU6GDT2pBeBfX6jV*NwR|sSl|xvDNg#{e>-o^4_vTD47=EDa z`;9~?KWK(dohu88=3o*GZhu80VQ=mc%Rj^*W(JCHB3mCz=)mNL?nt-W?h+~X1jaBP zAsp)iKS8jA=zMI;T(z^w7M$0 zw2z5FZP=FDBfG3@aHh&G4}FteK8^uko0l6d!ex>j6<--qT_dm?qOnO`1A{W{Q1wQHPTVy$X1HQfO9?AM* zsj2CN4n)^&iD3v?vaX8Tz0Codr4)PvWtX#|WN-E`S#{Ih+bu?E|9klUI}dyux%sn33s;{a;r^3>Tf3hWQm77O}Mgi?Sg9S1lfLm7l-sEPrqs(f7#ykuNsm(>L?uY8HNXe*GWoy>%6+eegac~lOS zGPYwT&Vqua3Zxn!EwcI**R0DFXM`dO0>W}I5Y}RUnuCT4Xy%IhH}-LqmeWVtWOR3- z#bw7KQJ5K6%T8XAXCeM(qklUL+FObi&}u&p7JP`nnzb0oIWLAPLL&xE zY|!`*NiaUtC`-i3O0}x`k|p+LT`#Ma>#&w}?@`r+j!rl;j5D^Q+*YiZh&IjAE$mLnI!FkvqTbrNBAONJ8?y}YrEVZg|<q!-y;AvsOwm(>t3$wUfQx!*E1*o zJhUYrX!u1#)4aa$>Wx6>UPb^`G##|X5Ngx-?P@R zce!EjN<;UY_vbx_ugB(}ym|4)(Z$$uLwGf~?{58^d)*@hLi4+M!~6cd`CWVGPOb(B zS$sJv19Sn;*(3AOBe1mEIo5)=&tvk`(DG{z@4{q4a1+F$UQ&yAn;OtcQ=P% zULnx+6%Z_etubs_MHUu#k*tg&$}HX|M|C)N6MX3K2L`sk;y5#cL2e|Nug4H^l~I%= zOu^Au1quZ@9SWMJ5}23Q8t7;~hQOAW?aNC0s-ouGyFcD}w|OOeBG)~1@3CClsjnem zBn-n4>NO<3&heD$Plf`&_L1Pos)zwt%H?WE0;sU{Mu16XmFF^bSRpAVUO3Fvk<8DR z9W4{rOj47IGDZ=UDJI)xX%Aju{_-Q{b#GA5J_Hp9+{V0oE%~NhOoA5>D=sEh_vW-Q znC&T-YUGVJBlZ@0`V*;GG+_?*IPKrT&}K7&e;5Aqs*pSV%JK^@=Pr!?v-{KI|KdUV zzf4;EJ5Ad}I9SM;!l8h}TjZvTJN7nwQ+kAX?Br=YFq!2xcWv2XDp5jN<=eEZQwYF! zt^4WrHs_mwf8lVh%4_Y9-RZuwouHET5AnZ?FSC8DoPK3x@RbKj3d%W?Ho?7Y6~0-g;$7;q3jt^y zYUJLPIq9eV5NW>?^Xd8V`TY+9?dxtKQ1=yD?f=@~;abbKaf&2-RAF&4ZKklwYyXJ$ zx`*#|N7u)pyUI%U@m$x5dmXvfli!4z8pimxL}!^Gltzr#FjqD(%A}vIfyl%q!AFJa zg-jeKp{<6bYEq(#2IA*ZT?3WYk<;GnK|qoK+J;GjuhKe)?>zrN>F10>%wPCCNU)2L z!}TaNAT$yBK*Z~$d))n~R=6S+uR{^k6w5Tn2q<-xY7XEpE3Mv^{NSZ4wO3r-YhLOp zD_I4-H&uPn+N3?157p+ZVp3Fp;o4+e?;g%+!<%n36sDHW`cHPsO+C7=Db= zP|+CKP1^?`0R3rVZ}t_HGFDofMOgtYl+iU<(!-(grgMZ!T04BQY3sU@)>2{i^41W^ z^&@471B}iQ2l+i5!~x=xgtuQ21RG}rfHOqG84&2CIYx{F2$3Q1XUVTe6B)f!rW-RT z&vuvN3^Nmp!SbqtF#Zg!O|s-T!e|^_K5$|2$o#1ve)s10-v2H#HpHlF5VQRQzr~lW zW4~5WFI*fN!jDykFP_cTbA(K%3>1dUs zWX(Dy=pFV0|7E7QIm~_?Kx%|Nm9iI6ql>acZeuS6gz1>E1>8o@rk3M!)g$RR-c$<@ zu9U7JiOY(O%V^6U2ifG@YWCXH%{Emf`c3N62OxV@_%a~WZ&_{}U3%s2*rzQZ?n>k6 zO5Nz3{0sY?WNz|$2(KTxpt`DbuxAjSuj^Rc{TKT`+`rcO)Nv${|e0)_o>=d&0N0vK}tabJ+clPBvp3VjPSCs+YU}^U!`)}`G+t+u0U*Ctj z@5#B2;au?as`5;E|Hqd*AJ26h&IO-XRgM&@);gb9?tCKGaU>V~omJ&Y6z2mwfBN{b z>%V{V?bT4vy~$tUP465Yck!n8r6SY>vC5zH-r2X(IgslZ%mt6GD#!R(C=IW4_UAeV za)lQl-geJQVV@6_rGx1Tj+#_7T~>aP8nV$!TNCW7EcF;~ zIi<5mWqk?!BH$(Z;n~Y2Ln_8~stmBH)L9r&_`%H&s3&QDg8&mx73bSdzg{6LBNDNE zk;qg^%Oofrh(un?L=$$6HxkiOu}H)q&}@V$I7ESCx){eOJ4C^03ZA8aP^595g6AoC zk%B1-QWO{zT%q791@BOBje1 zYUC{&TSWvON%n07WVv}G=$7|w1VwrOhQ}?pZTMK=LP}wQC=ZKvO`ROx*v+bC`KgVd zB!6Gr2-tmgZunSpNgh}aG|T;;H}77T5yREgw0+%6DMi@QvR*?epWq9A=|_sMoEAM2 z4$|R{XTWecT!SlQ-m@K8AQ*qGvF`AG~%uw>WTNK65gw9_I&A$}X&xH1W6`uM`*#4zQcZvP; M&-_}Tn63GL0f{;ZEC2ui diff --git a/deepity_build/reporting/__pycache__/rich_reporter.cpython-312.pyc b/deepity_build/reporting/__pycache__/rich_reporter.cpython-312.pyc index 306af058a1112fe915ef734b8c5ab045303e59f8..7b9ef29f4c4fb41bc6d9a4919f6368c03b6ed0bf 100644 GIT binary patch delta 193 zcmZ3{$T+u=k?%AwFBbz47`&O0C84vC?=_pcnSN1zeu=(IYHC4dNu_=YgpN-t&CE&B zFG?-QFDl8*OV{6Q$i9<<@z&+A;IODC!+f3}FC4lC=VEEC%aKSz9VnpnP z;E*%SCmheS0ojvzOtC5hMm+j)Na>P&ms3yJnxv3JHj3iu_aL7fS;02}=Tqk+W c2%PK%{$vy5uaL5_v6OQNEF1p7}{%5j<=R!p675BKyc5zoZ;wL{)liqyOREbSe r46a2uWP#}f*GXP80w;T26b_jzZu6Msf==+2&Hrqon1L2VI=BD;iv&+K diff --git a/deepity_build/reporting/__pycache__/rich_reporter.cpython-314.pyc b/deepity_build/reporting/__pycache__/rich_reporter.cpython-314.pyc deleted file mode 100644 index c965ce9f6c099135c5a781dbc47a783f7f1f72f6..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 19742 zcmch9du$v>dS^H1J#3Q0_nT}{q{PvKr1h|9nUXEZk|bJx#ENnSp?YG5k2p1{V6ZU>b+`+=O6V85D+{ME76Yl=v zcyYfJmk<{f$BB~u(s*gBP%Uf{6lblVxJF9%Sa_fDGR3`F@D_hXQLm^K`y-L5xIZ46 zio_7L1%ls81QDf2e2aZfd#zU77bsP`HPejJ2yhU31;WH;rzT_KJP90Z{ zo<8kUj%sD&Q|hEY?h7PT)*>o6m0)yAjR(~+-aq$?_(NRj5@`ZPgt({(@uE>FZc)Uz zRVl)tEww_}6SpZ=gl#NrSL_HoSlB@?7I!Kx_%6i_->nqGFIFU!DPd(Kr4->Z7A{dd z2$!>PsZxP(B@34+RR~wJut%vuxR!;>l{$p$S-3)JK)8{GE0rdMn_0L@X+gM^g{u`A z;Y}=DqqHHsnT2bWc7!`vxK7!E@KzSCS2_{)vT%d44dLx9+^Fn8cqa=tDP0J6vv9N0 z6K_$T!P4#0BqcO9Va)4v@99wFQaC72CgT2!cv7Z(GW14J4)|je7pMGcK*qYr@rj@u zjKo7~P`*fO+BJr`7wDgy6aOF)aitT1tHLk{uJ`u^A%L_d;XC4lsEA6DVo|J$O|dJO z{cbD9;`)kKQnQaS`cUl@v0MtpRR@J!mx7TXC}B#~EU~z%+0|e?p++?8z*Ho7hp5la z2_{CPXw!~%PfSe)yVa>!u%~-aoq9bu7LRoYg28Age#LnD*x-aJ z;y|oACsvaatIdhk8L?sOCSi{rK@0U}x_y%{?8wS<61&=C^lZz^ahW-;OwJyim7?7S zvt41(=*~#Ob8uf)L=P(NQ4#A@yiaG2Xr;+0rAUZ1XJVlC5|yZ(xFGmpY$_b|+BDZ; zzj|aUoS2MgPLS*+H5iL&C8wjI2uNSga1RCFitFL>q0nUTXxJZ(1;JIR-gNQlxL=Kf zl)JjRrYp_`LX#tJ`a_g(uA3qn+bNU&x5B|lf=2q~)gMkbpSw5}4#*e7iQvfFaewS` zS0ET0Q$tbW_51>t7rTN9h&1T@9<8MW{cxE2y3pf zsmW+K7!L*pye*nUo2(1FoZeccInqj4lF^GwGo`kW*GisD-X`*h6jh=?l?YEO!Mk1x z#$#+7$2E5(G3kr?W0(yd9`}dB2-~6){#a1+aK@5QgT5FJsYFbZ*!DMK#qnS)ZbV9W z)+AN{FRRi%)!eK=I1~xSv|3-LwK!J67YN0oVgD7alJgW_PGW_RRn1AO^s&jwE$Gmv zMROXHp*j75fR7eMbMrB=u+0|+SN6I!Yb+QZ&zl%Fsq4;SjM%~!IGH(>cADn&`9hIU z96YW%kH_(3#@~L1l2?T( z^pw+WTj|+q#D7}ba{CGjO84vJ`%?Wor|wHN$=YWZrG59M>UUmYv15zU@%vKqqSU%l zY_E5tbjia~p|SP-o$u{jZs@!&8xyULW!;F z5qYx**&{P1IF%OhX#jH9a*sd^W z9xZFjjDqzQ8Vdn{efk)|9O<-Y4yDbb$w&jkH4mXoTGl88%)8ZKs8+%&$GjG;LJf|C zg--auq$NL+IO5<~() zRi*XOTs|LaqDh~(NbRF4R4+ypvPI?#+E3+bvezD9L9fKtcoYR_C$&9rN;jnI(z3H5 zNXIOr zTRsa|aW=kOF9_D(M6{Kg%wmIdaOJtP6~h)oE@>4MPz3$o5eKkHY}vr!f^QLr1EtWa zY0ltVW4`#6Xi&2tq!q<4#%Pqp8Tl#>P^o-&$lG8q+=2Y7Lb|x@#{TR3|E72oE!4}` zUtX44Q&Q`kb6)*S+V#jHNNt(5Kn>=I1+yvmZTPuc*Ti~7Y)hKhPOpi_Al1%uQOmHU z(8f_Ly{{R_Td}@cwN5EpRw?_d5c{Im;i5vDPH9z~2A+%BhHZl;0?Z63Yh(WU^NCyL zIR!Et2&d4@5jE#I37Bn{g%H}Hi9j<&^xTql+B0!!evLgk!M6%Sw!uPUELF;et!Uj- zctwy?&hvLU3YSnSvev8;YgPr#fa#R#bw+F=TeGNOfORgPDK%)TcEj=4DfQ@qPQyGm zNWd-6;cXh$DJ|y}MN|@YLDDGLcH7$0d82Q{=$o=IP4}mtZYiNA}cKhO2@&m zi5(nwpsyJ8M}lDr43Y4w6$8`7)zHNRfU9N$8V^F;9QViKn*B;J9G-epv#7xU$~&UK zXn@$$9UMrHUGYaoUgDyrd^B5!F=u*Wf-sYOj`#89pWl}0 z)Q+6%<_VgWW*f>S>Ee`%(;LwK3Iq;1bnXOHQ15u&`^$e2q z1U4t?StKy5jV0$ZtXU)e$)IMvq=q1L5L6%I0-S@wDrm$&?Ko(-)M9-dD~G`)0)8gT zX!eVMBV!Y!`Iwvx#i2I~sqrgbI|mmu2;T7}HXJ|Jm%j?cD_W@-VJ8meIhJ?$CW3yv zuvV3eKzQmYl#acFw(Ob*=A9 z?@9CarMj-`{R>AQS_D_+L#I$)GvmJHo_qGDd(pG)nl0@tS$5W?oOOi!pCJ6eS#qP~ zdP!Q^oi6Q|3w(6>gUicX`chl^k{t)`o=r+e)1}_|_K&;nbS1sd-K|YZhtj3(b3-4U z`{3Mi$KF)O-emi;3zw48LBJpgA*F(=l(*NBF0H!x-J2)p#JQ~>?f78FT+95M3)2hX zdz+HdOAqZrRpU(Gt-iV5rONF%n#&J~52|ZdUAB@^z^Iaz^v-8KK6mHb^3DVJOYS_7 zlDx?yr@yXrqmMeXk0$D4Xy&zBug&;w`I1!~bKhOCe&YPG^OLe4mnFMjq|xr)_et-M zdp|k&$-#w#sj}|m@r$b#u_PdVZL_-Ek0ilUee=wG&E1-%vKN!i7uNy=T2lrP4zraq zl|flr-6#QsqQL_AT#>WOSVtwrwlBjREssO-=*co@Dv2;Q-6?i3YExZg zqzXb7(=A*N>YqVVmGl(#oTz82nF`h;CT}4KkWs~$>K1hfRn%efM&M~yV#?qUHpUt> z+nb?4d_r>_VH!v%3N#yI(l`*H8i-BTb2iP=WGGsoL(usZT7vR~X*@!4Fqw$JRP9&I;?mMsT`a_WH&asuEfaOD{6+T6S3d=B}v$8KJljW4(Mq*{BHoX_N> zb*Eap(aulHs&1Z1wjKEN*izZ)r1SJz;x^ImG(>^*(JzRenJl8*e8Ef>qzx@hIts_I zyP4QPL;y`8*Df1G&Qda(KWjl;<~B5ghA;t?Tf}e!tN}_%1~;rTic*Hkj_JZn0lfLV zXdsqL*x*gTB}!S;&ibHH7et}Mg~#d{wi8a=WiiLBZ)c@KseFR5RJ~)SE2UX&hIo@TBj#sWqHn^g$-NZoYOk7!aVoH zEwy8GyYea{y5R^qP|pxwv#M`cZwu-@n^i#%Zdh+C>h(Uh9`PQ7n{_H)Wt+0y;6Zu( z%w}*bWrwm;*`;(D>b1>)3E+j@N)Pr#76Qo|{IUQk^u#%lYD>jZol5VQ9 zBJz<}{g;DeWGH5W4M*hAZN7!K;S$0)mk_9&Je7z?6LC2dlP5xfKrkY|IRWv6nGu;% zn{w1dB!X^aepjWc7kwa`^CX)nhPTm2tkbH^ekHDZ2 z3TL;?veX|?qYgp~Xi5{^kO;}@qFY3`=@gmze4wv2T};B<00Ya&mw)mfWrjNRu^Z`{ zLl0?=(}}S$*e5uYv7jIIO>{r)=J=^=WY$SkBTQ14%#8Q)=`%-;z;=J^%*kmtqX{O4L8@Vj zPl#DW8k*zv1SV(v$~2dGWiG4f!m|FZJWc&Al!rR~cktkVibS{R>g*XcXAsHM=g6Tk zNVTx=>bQ(!up%`;Z4gfNHmk%})!!$t05a3n!Eq43`Ug~os7oszSEnX@lYaFwcF8l8 zOq*6E4x>Sw3q{mCKu79oHB24}ZR!Mh5%LJTs*~gqOICqa1eF+$x|cj+AzlZU``8E> zY-A9S219r?a7A;BB~&P+<1q))a;!KmpI?9feC@N5{zs@18v{%rV7P?~fTV`zO3c~j zrQ$93E9>ris*=^)7Y;0W4k36`x$R7Qs+K)+$|FB46047iD;7(6#frmGReJ5EM;@W7 z9{K5-qv__3uN~H^(uabTNmNjXLNz+uuvEM`UDL2!)0wL2T(0R&*7VMwymp**RDClr zGkI%rZivfucWrks-Tke5H@w#-+3+<2#P z+1r=$_APr4rM!ohq{BvTjgkA@viCX2w;xp9O)Np=)f;X7i_Psr1pchO?IN=hHqK%`Ap??~=>CHY<7QrU&1^TJx-ZwK`OZn#$7Kp+?R zvqMf8gg}t*jPn>rB@uj{%gQYv2^+b_(FjRUSN)K1&SRPB z5mAX8p?6G=xI${x#~B&bS5RO>t4s=ovaB+FWRB^ZZD)^6=P1M+%{*HhV)4{@AGHB&7D@xaZ4B;$ev+-I6h5$euy7dkx03;OL3ra)-ewf{80q%&5M=~2KH`c&_ zt(VWY*yzb@G6E?B&Fuhr04TYH^w6AG%T*P>+`hEHYPq|{@js=>^UFf&xJTo=Sz#F zG#yz&*wX$_1hTdrb$U$?Q*9XV?P73*+#x2-(&O z%M9*_8Hm@w{~#6zCf3QdxI-Apv}<5WM^#WFGx!1B{K#~%4k;+2o<~83`v$P{gA}3j z6<5hf%^zi!U-cML3F%QZr zZ@hi|?RS2#T(&(~wtcSnnl<0-n-hO;?_AaV^xclTwfBmXJ5MIhU07=Lf9AZH*~12H zTG0h#ATsM~{fl?qMVLMtqf5=9-=fhhuysELS^gdRh>*M|`Dj4wu~U;I`j7 zK0;k7fs!VRbFC0|x=NSVSTcDat8Ajx(=#MQ_FR=F5xkj!rw@6!4-fS(;Bh{!^8@_? zPb@JB*mgy0$XOjGbNKW;_8w%$1ZQY&X9{eI6kUH@wswNQWe)u%dR-ywAWfl#rV!rT z9`?v|j(q0y&Rr{q5xolgHttR6Sx{4mh#7Q1bNQHUjqOA0Z&Cjc)wy_Mh)yuGFe8er z(tQaga>0tE=MLaz0O4V-ucYZ$e*`a^4VSap;KgQbttDsb*W}Y2=wy^OZVC0%Ri9?s z1izVe_&L4Gza!7Q_=5;@zoto-(Rj9P%a|^RjhX3I|2;|-m||mL=&~BFLkIGwI6t3W zFMo;;|AwddpJ)sQshiw_ff1cen8b^c=B(59&nNW_8HEVqE@H;PY}=rJY=UfuT)3ob zA%q6hf>fM!Rd9?|)0%UM*qmXWo%M2Q`T31z){1_gm`mT$xwO-`2d~-2!&CmJVbl-N zpUJ4L#AV>k?}dvPP&YSBqEE?x9PH1r#-5Ac~JptpnV@d2l zqx_WS%fXDd`X$jEC4g?3o_6TQ=K*jn!ugrSyQ3rZ|4MI3bfaJ2(JG8R6@*Qe-#^t_ z*IP!t0GldO5L^`=lH*p%PlgQtnclgD4fh*Cv>gL7B!3Qwke>5Eg!FvO>e4~Vz`nVHNYEV)7%`*4RQRr< z-oE_v8Lt1!)w0RQ*nQgpXaVn_l(8@bI&&5#Yj^24bl2NmJe$2GXB{{amYt0$XJfLd z@9y|#&Z8NIZM=ZaI=(F->X#V3c|;qVXBHD{?i!rTx*<>LbbmgxsqL0HD~XyRkk5MWs~t_S1xa8n<# z)_NTm@cWLlL+|_E^DP{@SDqX^m!$iT=O2R`XvUwK^;+r5_tS38U(xW5tyyLpz}6&x z7F&~^bD$u4uHBj>CNUCv0K&Q|kj%o}AW?RnBjbRn->wM-&NHVg%fQ_#UHI0g?n;IB$|2#K{t)X689+j?o;eb;CMEMp@%YX2gQ6 zo955QzODT{DP@_%J9&(7<~eLLm>u5V+;C?vgsTZ8n=+*{+fo0Pa&a3C1{=+S4>`Cj zl)Lu~9deqL#aDD_$jKUt1_SD!B9(1u*b?~n4P29)eTPWDwM3M!)#evk3hwZk^VPK{##1o`^&ZV0 zE&1av@_Vp%S4fs1!Ol1}<`NuoRb?F>*?U?a$FBo&ssEh@tV1WptUsjJ_$zohM{_A( zT@x?mDm8L5=^gb?-qy!T_1_*=QvU}Hd83)Hp;2XKzL(~@cWyV$eDC}o7F)}6&6yuZ z7aJbikCm*k^Brg;&nZPYP>QsI*40)N%BZRcRB^nMrLvlq4%01BJtIL}i^}mt18!OQ z)gg8evt=dprTzq7&f4l z_RpOA)@~~O7~4=y%Dw?*YElPRN8wt2&_|al*@=r!F!1egY}y^hUEFi@9r4IhsHmEV z7zn!XZFm8Pu+?@!oEsx@xF9ZUk|g4Za`NgAg6dDH=4iOtj$)2EdJeQO%h9b~=$&;a2jG?-WO0=ArmE}al{|4|n z-5^a6xMrY!Dgj~}z%}4wAz0zn>quvxLE*`q75;9rP|$hPhPz8wg(2Y>$TBJxk`Lb! zuHtj^S8!;$gckNe#$HQE2H4CW1DYHTs^j?B^7fpL9Xl1>}i zRg-8l(-ZF}OC9qDQa0p|R-;4>31sDC+NJeRfWo4YO z2L+jhUOh@7w!l9^C|g^pFQlPH%aum>Lh5-mIWaCQr>pUWBNnjnIli7)$g=BY%g*MM zvzb0nTwiNR6T=ovMu4R5bxcQAQqQZ)D74{m*Gc0tjyoIacHqr|zu_?#V6nBw9T7AB zT`TYMafpKkNg-}HV|k+Duz6#V-vh*QWy2P|LLu;-5l)zNTHs(bdI2Xh)%#X;9&Pg5 z6?T6#if`}O@9?1Lm%#^5>;cpQU+QP)bB zv)K=SI63eu&FE|CL-}cm>AB_a1CL;18=ui2NBwVT9Ut-1rl*om0SKQba^R-NZv+pq zOL!0J+8^5R1Z`F~JakaVi5o)?T@-Q)?$Sra2=TY2O=&nC;<{D{A>eH@Y*Hd^N+Cdm zOSYt4G3G0~jH&NHzR3o%+Ay^o+1i=&=DDeewE}$uyW6rBE=H|c?N|*EiFQm$2J%Vn zH`u1G#CTa;p4Ee3)l7cD%|2!u$pMITv=ju0-z#5xnP61N1Bl{gfdC>HJ9A)34X|* zw>`7qU2J1pw)G!9m8r_1`O#17@pg-qLyMlFYqrlf?^@WqxS639`GF2zat=9f6~BYX zUXw1{WNssJ#S_yQ?57N(3vPSaMAgZH!k9cw6abCP=iy z&rG;|K77xY2vZn8vh%%!Kdk4J`F!IcT*;vGJTirBhtC(78uR(o1JqPMd1RDQ zk8W(Si$NUb-J$5m2;b4m0wUi_C&@PTW5A{@G?5-D>UAm*nz@$*dh-4Oin55M$-QV~9e&5&oa!+?cL;861>ZL#1p=tj{H(`ngg^hD4T+=RSnaD}a zi^yyAM9ga&rMl?J*on9oM>&#-ooqkR;pwB=pteWLv<+#Q_<4PHARMrMJs4AEFOiA7 zprzWsrrViR#MjT1J=3ub3+0Xi=VjBY=Ii}5tu46O_#YGu&4^sn&SO#;&5K@W_hQ>L zCz6SHZJELO$bm>EI;2_pyoOSYN>-m0@#2|n@)@QVJd4+!waX{%wI?0wiPs_Pb-K%R zBJcBbS(B)>>#Ml*?SOq9d5M$k|b~ipok*NrG+mar4A*6j$S-o&6NblK z*(2}Z^>)Rnxcfb`#F=Rqx!`u5Gs7-0%gGxSJHc}~)3BMv4PnzH66Hn1#Q;uGj$x}g zO|SyD_=17)7G(V*HN=x_GzWF1S|__Dm`&Xly@b+FmN^w_`<1hUa%@vvT2R7qVHG zy6?T_zc#t(l=!4@Ts}29E1XL_@41tKyklj($#ONgo6KfS+eF`Vx#}mgS;Mv+EC$bJ z{o(pXV>8o@&wBA5cNE!wUDGzTM11!;Ww#F9p8nhKuTk+AM`g_hnrL~NRhZ}_@Sl`g zFBd6N+(yeoRzRiM#`Zzo9*q{2(&@AX3Dgrs7J_(KEVyqZ7*c%)*f>^vLJEU49B;6*5agnH1#%%K@c0p&Ow+9%(3f(kuEJd zuE);INj*6ADn^0kFk{IuEhA@IFl@|oSjo5|Wtg2QbNlox-XOS9U=R;dN(z^Wk;-M-WFD7!JMw6zLzZE5Cv`|nLg*OI?=tOdksDAu z?)pim;??h>mhht{{>#FT8F|l-TcupV5QK6wshH@Z{_&#aKs949W6;_pss!Xy3zSnW z4=sV9uh>aXK0+onX*8Z#*O2|o;P$mR-Lz{%!)qN>(zWh^wN5G}sMJNJB$c|U)T8zE zuk})?PwN|8>qiMIo9J2Jf|6Q4`nE99Jf#*MMS<3@u6=U=w2Q#z`N({9J~rPrAD{1- zZ_h;L+b6=_xMrfS2jfB8Vr06Et5Dm-VVHZn;4*{Re3>0J-MoPO*^sm)K0AHNr~(Wi zmL$KPQ*JTPY|-Sx00LB-bg^MOq%x-7z`!MLyV7Elpa*u|v|-5Ec+T-Wr?kIPPBE8E z8SFGEVBJu_s#;92hCi{;ct5{eqSRcNC3VbX+hnJzLq{aV1b?`p$C8 zu$X(^q_tARN+=ejp7r<0Q-KOL1#1sp%Ba zIGC4^qZNoN6tRhHoTnGk6Ku>g?8Px$G0DnUFrZYKVELL1E3BB{3owNuuuYS}vK4sV zs4_^FVR_XwSVYmNSRU-T!=}`6Mm$SY8Sg9nu2giFv|8K^Aq(M`+fCtw!nqC zM7TOgtvZcDA#0I*raHwCbsnK?7iBG@?x8vwT7c4OE*ZmpK?;mt3RV74v+=8qFE#!E zT0rB!nb10W$k6S0m>hbT-2Sk)KlmFMdN?%va2Nahz|THd>(@Gmf2(zN2AWJb7hs)6 zSSc+bP4Y>WFcX~wBHoTk`=*T5Z2;u6Y}1<<(%p&-Gu_nU@fX)A`~0B1)QvJ4$VAhS zM&(5mT8v6W8I=&kLx{fmlfRPN(nqwu7C1X^j2V@HH# zv+d~ElA1~8cj`?D+9nY+KG_C)A6HlzdEbwG0HOfHx-^CM}GOvopHw7v2dh5?Ktd5IsV>Tvt(LYQaGatq7*?Yw_$b! z=`AU2JF30=vZi%vyXh+%`i0Hw?eLtKNlR*c-tvva%vw@IYZ8>0@c??0ICh2Sp_;@D ziWMk2U})J81t3sB&n2WxD8z2Svt9grqA{?zq*pVRL)CCM+k>#9<4Qf_Z z3hxr+6RMNj=ow;xdnslJPJU;p%nK7N0W0Dx*L*{>05l8x0%*X5$>(Cg;(`gBgEQ&o z=?Y*i3eFj~xmajsF7LB(=t4SBg^%oVI#7i30hks+`#p3FTnSv&B*7Al2H)Y>70l7% zZ?NO=qfgJAoI6Wy1e|7f(WzMI%q1%*fcw-uNA`Q{^+N{($o=4VpNLu`EI>|ry2LCf zeaor5G8N23T8fZ<5p%LDr5p$YDBPp2elzotcrPjqZcvV6OW^c0Zd_ovi5n=k1wqM3 zA3AXDk?29Utvu8B7Oql7I19(CVzV~qSc>BbNnfK!ex6=>+D|D zx5GWVrw@OnCsp}I|2=)@Lp^yt^QZc%KC+^Z-1y-=ee|o{d#)${y?@)MmA@Rhei$5Q zb!2K~Wa`V2SMMiZy?*LpvTrrHYbCksL2`6Gt_|#7)8*(pSM;6t^j#QlRo}Iu@4`s6 zv3IZNyYK1jtKOY=^_`DGX<*s4jTbT9r$K2w?xKj1j^mX9 zZ@`zPAg08xoGE?maBl-Q7Z4Vh3I&>oiBb@h_6+b~FIWgZNgKOB;ZzxeJ9Gy)ltJYdw3W5>^Ajs+yUfh>=w=w)!s5jkNcIkXz2 z?2s2L0R9BHj(|t%fw9vICFH?pH7P>uw9tVDG3%f(tqF&C3actZDWN>gqQW0+wp&4C zxr0P#NQwre6n)bBK;Kc{ypZ$N==VRM_0w#|ccR~IG7v=psBcZ<(YO6T6VIHK;TL7W zr0FNCuef-Jc#X=hQ+9~5KcGwnsk{mp#dBJ@tLWywgG^ev9r4L^4cW~tpY6Q4^H%2X zKD_ne{T)+lRQJUedAT!q$NlBUcRpU3`_by$`IWiz_usy-a*^MCo8LcNl%3TUkj{iD zFxxPS--(bvj&_HXw9%xNV-4iH7CR2eciVeY&rQA&e||pleU3tbGR;zmv?c8UId37? zLr7Xqj^cs|<)i}^JPRfa1CbF6qHvhEq)|-}_B&8*MX@DQRuQhJaMH)e z;FM7%)+uLPi}8n%3Cvbay_I`D!5dI)wB_^Qv4DD;yy&Jv45)2I?TvDsCgA+Iiw3+H zGNBDVzyz^tbwI>j5ODE5h?95|8G>Bjr}7Vwk$W!P5b+jO9j5FEWn?n^xFOy6j&LsV z29hcHWsD0U-bPcG9Nq?A*%5qDt8oshrBU(F1&nfUBcqg@kJ#mn{?B&Y-0@f7fiFEA z9J(HBV3ED6TgFzljD5K!bw8Q9KK%%b{6^RM2LAlYpIo`ob|dpMeRccz%J%Uuds9C@ zaJM&oS5H5R0D<|@YMkszr+4y5AwuVMK|syd3#Q1H4VdRflT*d+(lLpaI9mr}4V>x6MWuuBS7CJkw zAK8VQ&={<5v(#gtj!hjkKH4Q&GVg zbP?*U7E%qpY`_(-%C_giJ74 zGea>Jw^t5}i#rrMt$hzj~bH#PSqWNWcVB=S$%z^}F8 dUu!$@_f4A>jqLmM$m-Vg%GUI6HM*0-{2$8&i2ncp diff --git a/include/deepity/backend/Backend.h b/include/deepity/backend/Backend.h index e69de29..e72e80f 100644 --- a/include/deepity/backend/Backend.h +++ b/include/deepity/backend/Backend.h @@ -0,0 +1,10 @@ +#pragma once +#include +#include +#include + +namespace Deep +{ + /// @brief Factory function to create the appropriate backend. + std::unique_ptr CreateBackend(DeviceType device); +} \ No newline at end of file diff --git a/include/deepity/backend/CPUBackend.h b/include/deepity/backend/CPUBackend.h index e69de29..033185c 100644 --- a/include/deepity/backend/CPUBackend.h +++ b/include/deepity/backend/CPUBackend.h @@ -0,0 +1,42 @@ +#pragma once +#include + +namespace Deep +{ + class CPUBackend : public IComputeBackend + { + public: + CPUBackend() = default; + ~CPUBackend() override = default; + + float *Allocate(size_t numFloats) override; + void Free(float *ptr) noexcept override; + void Zero(float *ptr, size_t numFloats) noexcept override; + void Copy(float *dst, const float *src, size_t numFloats) noexcept override; + void CopyFromHost(float *deviceDst, const float *hostSrc, size_t numFloats) noexcept override; + void CopyToHost(float *hostDst, const float *deviceSrc, size_t numFloats) noexcept override; + + void MatMul(bool transA, bool transB, int M, int N, int K, + float alpha, const float *A, int lda, + const float *B, int ldb, + float beta, float *C, int ldc) noexcept override; + + void Scale(float *buf, size_t n, float alpha) noexcept override; + void AxpyInto(float *y, const float *x, size_t n, float alpha) noexcept override; + void Activation(ActivationType type, float *buf, size_t n) noexcept override; + void ActivationInto(ActivationType type, float *dst, const float *src, size_t n) noexcept override; + void ActivationDerivative(ActivationType type, float *buf, size_t n, bool activated) noexcept override; + void ActivationDerivativeInto(ActivationType type, float *dst, const float *src, size_t n) noexcept override; + + void FusedStateUpdate(float *z, const float *feedback, const float *deriv, + const float *e, size_t n, float ir) noexcept override; + float ComputeErrorAndEnergy(float *e, const float *z, const float *mu, size_t n) noexcept override; + + void AdamStep(float *param, const float *grad, float *m, float *v, + size_t n, int t, float lr, + float beta1 = 0.9f, float beta2 = 0.999f, float eps = 1e-8f) noexcept override; + void AdamWStep(float *param, const float *grad, float *m, float *v, + size_t n, int t, float lr, float weightDecay, + float beta1 = 0.9f, float beta2 = 0.999f, float eps = 1e-8f) noexcept override; + }; +} \ No newline at end of file diff --git a/include/deepity/backend/CUDABackend.h b/include/deepity/backend/CUDABackend.h new file mode 100644 index 0000000..bf06703 --- /dev/null +++ b/include/deepity/backend/CUDABackend.h @@ -0,0 +1,42 @@ +#pragma once +#include + +namespace Deep +{ + class CUDABackend : public IComputeBackend + { + public: + CUDABackend(); + ~CUDABackend() override; + + float* Allocate(size_t numFloats) override; + void Free(float* ptr) noexcept override; + void Zero(float* ptr, size_t numFloats) noexcept override; + void Copy(float* dst, const float* src, size_t numFloats) noexcept override; + void CopyFromHost(float* deviceDst, const float* hostSrc, size_t numFloats) noexcept override; + void CopyToHost(float* hostDst, const float* deviceSrc, size_t numFloats) noexcept override; + + void MatMul(bool transA, bool transB, int M, int N, int K, + float alpha, const float* A, int lda, + const float* B, int ldb, + float beta, float* C, int ldc) noexcept override; + + void Scale(float* buf, size_t n, float alpha) noexcept override; + void AxpyInto(float* y, const float* x, size_t n, float alpha) noexcept override; + void Activation(ActivationType type, float* buf, size_t n) noexcept override; + void ActivationInto(ActivationType type, float* dst, const float* src, size_t n) noexcept override; + void ActivationDerivative(ActivationType type, float* buf, size_t n, bool activated) noexcept override; + void ActivationDerivativeInto(ActivationType type, float* dst, const float* src, size_t n) noexcept override; + + void FusedStateUpdate(float* z, const float* feedback, const float* deriv, + const float* e, size_t n, float ir) noexcept override; + float ComputeErrorAndEnergy(float* e, const float* z, const float* mu, size_t n) noexcept override; + + void AdamStep(float* param, const float* grad, float* m, float* v, + size_t n, int t, float lr, + float beta1 = 0.9f, float beta2 = 0.999f, float eps = 1e-8f) noexcept override; + void AdamWStep(float* param, const float* grad, float* m, float* v, + size_t n, int t, float lr, float weightDecay, + float beta1 = 0.9f, float beta2 = 0.999f, float eps = 1e-8f) noexcept override; + }; +} \ No newline at end of file diff --git a/include/deepity/backend/GPUBackend.h b/include/deepity/backend/GPUBackend.h deleted file mode 100644 index e69de29..0000000 diff --git a/logs/build.log b/logs/build.log index ca3e1cf..9978c1c 100644 --- a/logs/build.log +++ b/logs/build.log @@ -1,326 +1,246 @@ --- Deepity Build Log (Release, arch=native) --- === CMake Configuration === --- PGO: instrumented (GENERATE) build -- profile data will be written to /home/rose0/Projects/deepity/build/pgo-data --- CXX compiler: /usr/bin/c++ --- Found OpenMP_C: -fopenmp (found version "5.2") --- Found OpenMP_CXX: -fopenmp (found version "5.2") --- Using Intel MKL (found via MKLConfig.cmake). +-- The C compiler identification is GNU 11.4.0 +-- The CXX compiler identification is GNU 11.4.0 +-- Detecting C compiler ABI info +-- Detecting C compiler ABI info - done +-- Check for working C compiler: /usr/bin/cc - skipped +-- Detecting C compile features +-- Detecting C compile features - done +-- Detecting CXX compiler ABI info +-- Detecting CXX compiler ABI info - done +-- Check for working CXX compiler: /usr/bin/g++ - skipped +-- Detecting CXX compile features +-- Detecting CXX compile features - done +-- CXX compiler: /usr/bin/g++ +-- Found OpenMP_C: -fopenmp (found version "4.5") +-- Found OpenMP_CXX: -fopenmp (found version "4.5") +-- Found OpenMP: TRUE (found version "4.5") +-- Looking for pthread.h +-- Looking for pthread.h - found +-- Performing Test CMAKE_HAVE_LIBC_PTHREAD +-- Performing Test CMAKE_HAVE_LIBC_PTHREAD - Success +-- Found Threads: TRUE +-- CUDA support enabled. +-- Found Python: /usr/bin/python3.12 (found suitable version "3.12.13", minimum required is "3.8") found components: Interpreter Development.Module +-- Performing Test NB_HAS_MTLS_GNU2 +-- Performing Test NB_HAS_MTLS_GNU2 - Success +CMake Warning at CMakeLists.txt:213 (message): + DEEPITY_USE_MKL was set, but MKL was not found (is oneAPI/MKL installed and + sourced? e.g. 'source /opt/intel/oneapi/setvars.sh'). Falling back to + OpenBLAS. + + +-- Looking for sgemm_ +-- Looking for sgemm_ - found +-- Found BLAS: /usr/lib/x86_64-linux-gnu/libopenblas.so -- Configuring SLEEF 3.9.0 -- Could NOT find OpenSSL, try to set the path to OpenSSL root folder in the system variable OPENSSL_ROOT_DIR (missing: OPENSSL_CRYPTO_LIBRARY OPENSSL_INCLUDE_DIR) --- Found OpenMP_C: -fopenmp (found version "5.2") --- Found OpenMP_CXX: -fopenmp (found version "5.2") +-- Found PkgConfig: /usr/bin/pkg-config (found version "0.29.2") +-- Found OpenMP_C: -fopenmp (found version "4.5") +-- Found OpenMP_CXX: -fopenmp (found version "4.5") -- Configuring build for SLEEF-v3.9.0 - Target system: Linux-7.1.12-200.fc44.x86_64 + Target system: Linux-6.8.0-138-generic Target processor: x86_64 - Host system: Linux-7.1.12-200.fc44.x86_64 + Host system: Linux-6.8.0-138-generic Host processor: x86_64 Detected C compiler: GNU @ /usr/bin/cc - CMake: 4.3.0 - Make program: /usr/bin/gmake + CMake: 3.22.1 + Make program: /usr/bin/ninja CMake build type: Release -- Using option `-Wall -Wno-unused-function -Wno-attributes -Wno-unused-result -Wno-psabi -ffp-contract=off -fno-math-errno -fno-trapping-math` to compile libsleef -- Building shared libs : OFF -- Building static test bins: OFF -- MPFR : LIB_MPFR-NOTFOUND -- GMP : LIBGMP-NOTFOUND --- RT : /usr/lib64/librt.a +-- RT : /usr/lib/x86_64-linux-gnu/librt.a -- FFTW3 : LIBFFTW3-NOTFOUND -- FFTW3F : LIBFFTW3F-NOTFOUND -- OPENSSL : -- SDE : SDE_COMMAND-NOTFOUND -- COMPILER_SUPPORTS_OPENMP : 1 -- A version of SLEEF compatible with libm and libmvec in GNU libc will be produced (sleefgnuabi.so) --- Failed to find LLVM FileCheck --- Google Benchmark version: v1.9.1, normalized to 1.9.1 --- Performing Test HAVE_STD_REGEX -- success --- Performing Test HAVE_GNU_POSIX_REGEX -- failed to compile --- Performing Test HAVE_POSIX_REGEX -- success --- Performing Test HAVE_STEADY_CLOCK -- success --- Performing Test HAVE_PTHREAD_AFFINITY -- success --- Configuring done (3.0s) --- Generating done (0.3s) --- Build files have been written to: /home/rose0/Projects/deepity/build/Release +-- Configuring done +-- Generating done +CMake Warning: + Manually-specified variables were not used by the project: + + DEEPITY_BUILD_TESTS + + +-- Build files have been written to: /root/Deepity/build/Release === Compilation === -[ 1%] Building C object _deps/sleef-build/src/common/CMakeFiles/common.dir/common.c.o -[ 1%] Building C object _deps/sleef-build/src/common/CMakeFiles/addSuffix.dir/addSuffix.c.o -[ 1%] Building C object _deps/sleef-build/src/libm/CMakeFiles/mkrename_gnuabi.dir/mkrename_gnuabi.c.o -[ 1%] Building C object _deps/sleef-build/src/common/CMakeFiles/qtesterutil_obj.dir/qtesterutil.c.o -[ 1%] Building CXX object _deps/sleef-build/src/common/CMakeFiles/psha_obj.dir/psha2_capi.cpp.o -[ 1%] Building C object _deps/sleef-build/src/libm/CMakeFiles/mkdisp.dir/mkdisp.c.o -[ 1%] Building C object _deps/sleef-build/src/libm/CMakeFiles/mkmasked_gnuabi.dir/mkmasked_gnuabi.c.o -[ 1%] Building C object _deps/sleef-build/src/libm/CMakeFiles/mkrename.dir/mkrename.c.o -[ 2%] Building C object _deps/sleef-build/src/libm/CMakeFiles/mkalias.dir/mkalias.c.o -[ 3%] Building C object _deps/sleef-build/src/common/CMakeFiles/testerutil_obj.dir/testerutil.c.o -[ 4%] Building CXX object CMakeFiles/nanobind-static.dir/.venv/lib/python3.14/site-packages/nanobind/src/nb_internals.cpp.o -[ 5%] Building CXX object CMakeFiles/nanobind-static.dir/.venv/lib/python3.14/site-packages/nanobind/src/nb_type.cpp.o -[ 5%] Building CXX object _deps/benchmark-build/src/CMakeFiles/benchmark.dir/benchmark_api_internal.cc.o -[ 6%] Building CXX object _deps/benchmark-build/src/CMakeFiles/benchmark.dir/benchmark.cc.o -[ 6%] Building CXX object CMakeFiles/nanobind-static.dir/.venv/lib/python3.14/site-packages/nanobind/src/nb_func.cpp.o -[ 5%] Building CXX object CMakeFiles/nanobind-static.dir/.venv/lib/python3.14/site-packages/nanobind/src/nb_enum.cpp.o -[ 6%] Building CXX object CMakeFiles/nanobind-static.dir/.venv/lib/python3.14/site-packages/nanobind/src/nb_static_property.cpp.o -[ 7%] Building CXX object _deps/benchmark-build/src/CMakeFiles/benchmark.dir/benchmark_name.cc.o -[ 8%] Building CXX object CMakeFiles/nanobind-static.dir/.venv/lib/python3.14/site-packages/nanobind/src/nb_ndarray.cpp.o -[ 8%] Building CXX object _deps/benchmark-build/src/CMakeFiles/benchmark.dir/benchmark_register.cc.o -[ 8%] Built target common -[ 8%] Linking C executable ../../bin/mkrename_gnuabi -[ 9%] Building CXX object _deps/benchmark-build/src/CMakeFiles/benchmark.dir/benchmark_runner.cc.o -[ 9%] Linking C executable ../../bin/mkalias -[ 9%] Built target mkrename_gnuabi -[ 9%] Building CXX object _deps/benchmark-build/src/CMakeFiles/benchmark.dir/check.cc.o -[ 10%] Linking C executable ../../bin/mkdisp -[ 10%] Built target qtesterutil_obj -[ 11%] Linking C executable ../../bin/mkmasked_gnuabi -[ 11%] Building CXX object _deps/benchmark-build/src/CMakeFiles/benchmark.dir/colorprint.cc.o -[ 11%] Linking C executable ../../bin/addSuffix -[ 11%] Built target mkalias -[ 11%] Built target testerutil_obj -[ 11%] Building CXX object CMakeFiles/nanobind-static.dir/.venv/lib/python3.14/site-packages/nanobind/src/nb_datetime.cpp.o -[ 11%] Built target mkdisp -[ 12%] Generating include/renamesse2_gnuabi.h -Generating renamesse2_gnuabi.h: mkrename_gnuabi sse2 b 2 4 _mm128d _mm128 _mm128i _mm128i __SSE2__ -[ 13%] Building CXX object CMakeFiles/nanobind-static.dir/.venv/lib/python3.14/site-packages/nanobind/src/common.cpp.o -[ 13%] Built target mkmasked_gnuabi -[ 14%] Building CXX object _deps/benchmark-build/src/CMakeFiles/benchmark.dir/commandlineflags.cc.o -[ 14%] Built target addSuffix -[ 14%] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefgnuabisse2dp.dir/sleefsimddp.c.o -[ 14%] Building CXX object _deps/benchmark-build/src/CMakeFiles/benchmark.dir/complexity.cc.o -[ 15%] Linking C executable ../../bin/mkrename -[ 15%] Built target mkrename -[ 15%] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefgnuabisse2sp.dir/sleefsimdsp.c.o -[ 15%] Built target psha_obj -[ 15%] Building CXX object CMakeFiles/nanobind-static.dir/.venv/lib/python3.14/site-packages/nanobind/src/error.cpp.o -[ 16%] Building CXX object CMakeFiles/nanobind-static.dir/.venv/lib/python3.14/site-packages/nanobind/src/trampoline.cpp.o -[ 16%] Generating include/renameavx_gnuabi.h -Generating renameavx_gnuabi.h: mkrename_gnuabi avx c 4 8 __m256d __m256 __m128i struct\ {\ __m128i\ x,\ y;\ } __AVX__ -[ 16%] Building CXX object CMakeFiles/nanobind-static.dir/.venv/lib/python3.14/site-packages/nanobind/src/implicit.cpp.o -[ 16%] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefgnuabiavxdp.dir/sleefsimddp.c.o -[ 16%] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefgnuabiavxsp.dir/sleefsimdsp.c.o -[ 17%] Generating include/renameavx2_gnuabi.h +[1/147] Building C object _deps/sleef-build/src/libm/CMakeFiles/mkmasked_gnuabi.dir/mkmasked_gnuabi.c.o +[2/147] Building C object _deps/sleef-build/src/common/CMakeFiles/common.dir/common.c.o +[3/147] Building C object _deps/sleef-build/src/common/CMakeFiles/testerutil_obj.dir/testerutil.c.o +[4/147] Building C object _deps/sleef-build/src/libm/CMakeFiles/mkrename_gnuabi.dir/mkrename_gnuabi.c.o +[5/147] Building C object _deps/sleef-build/src/libm/CMakeFiles/mkdisp.dir/mkdisp.c.o +[6/147] Linking C executable _deps/sleef-build/bin/mkmasked_gnuabi +[7/147] Generating include/masked_avx512f_sp_gnuabi.h +[8/147] Generating include/masked_avx512f_dp_gnuabi.h +[9/147] Linking C executable _deps/sleef-build/bin/mkrename_gnuabi +[10/147] Generating include/renameavx512f_gnuabi.h +Generating renameavx512f_gnuabi.h: mkrename_gnuabi avx512f e 8 16 __m512d __m512 __m256i __m512i __AVX512F__ +[11/147] Generating include/renameavx2_gnuabi.h Generating renameavx2_gnuabi.h: mkrename_gnuabi avx2 d 4 8 __m256d __m256 __m128i __m256i __AVX2__ -[ 18%] Building CXX object _deps/benchmark-build/src/CMakeFiles/benchmark.dir/console_reporter.cc.o -[ 18%] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefgnuabiavx2dp.dir/sleefsimddp.c.o -[ 19%] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefgnuabiavx2sp.dir/sleefsimdsp.c.o -[ 19%] Building CXX object _deps/benchmark-build/src/CMakeFiles/benchmark.dir/counter.cc.o -[ 19%] Building CXX object _deps/benchmark-build/src/CMakeFiles/benchmark.dir/csv_reporter.cc.o -[ 20%] Building CXX object _deps/benchmark-build/src/CMakeFiles/benchmark.dir/json_reporter.cc.o -[ 20%] Building CXX object _deps/benchmark-build/src/CMakeFiles/benchmark.dir/perf_counters.cc.o -[ 21%] Generating include/masked_avx512f_dp_gnuabi.h -[ 21%] Generating include/masked_avx512f_sp_gnuabi.h -[ 21%] Built target maskedAVX512F_generated -[ 21%] Generating sleeflibm_SSE_.h.tmp -[ 22%] Building CXX object _deps/benchmark-build/src/CMakeFiles/benchmark.dir/reporter.cc.o -[ 22%] Generating sleeflibm_SSE2.h.tmp -[ 23%] Generating sleeflibm_SSE4.h.tmp -[ 23%] Generating sleeflibm_AVX_.h.tmp -[ 24%] Generating sleeflibm_AVX.h.tmp -[ 24%] Generating include/renameavx512fnofma.h +[12/147] Generating include/renameavx_gnuabi.h +Generating renameavx_gnuabi.h: mkrename_gnuabi avx c 4 8 __m256d __m256 __m128i struct\ {\ __m128i\ x,\ y;\ } __AVX__ +[13/147] Generating include/renamesse2_gnuabi.h +Generating renamesse2_gnuabi.h: mkrename_gnuabi sse2 b 2 4 _mm128d _mm128 _mm128i _mm128i __SSE2__ +[14/147] Building C object _deps/sleef-build/src/libm/CMakeFiles/mkalias.dir/mkalias.c.o +[15/147] Linking C executable _deps/sleef-build/bin/mkdisp +[16/147] Building C object _deps/sleef-build/src/common/CMakeFiles/qtesterutil_obj.dir/qtesterutil.c.o +[17/147] Building C object _deps/sleef-build/src/common/CMakeFiles/addSuffix.dir/addSuffix.c.o +[18/147] Generating dispscalar.c.body +[19/147] Linking C executable _deps/sleef-build/bin/mkalias +[20/147] Generating alias_AVX512F_dp.h.tmp +[21/147] Generating alias_AVX512F_sp.h.tmp +[22/147] Generating dispscalar.c +[23/147] Generating dispsse.c.tmp +[24/147] Generating include/alias_avx512f.h +[25/147] Generating dispavx.c.tmp +[26/147] Generating dispsse.c +[27/147] Generating dispavx.c +[28/147] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefgnuabi.dir/rempitab.c.o +[29/147] Building CXX object _deps/sleef-build/src/common/CMakeFiles/psha_obj.dir/psha2_capi.cpp.o +[30/147] Linking C executable _deps/sleef-build/bin/addSuffix +[31/147] Building C object _deps/sleef-build/src/libm/CMakeFiles/mkrename.dir/mkrename.c.o +[32/147] Linking C executable _deps/sleef-build/bin/mkrename +[33/147] Generating include/renameavx512fnofma.h Generating renameavx512fnofma.h: mkrename cinz_ 8 16 avx512fnofma -[ 24%] Generating sleeflibm_FMA4.h.tmp -[ 24%] Built target renameAVX512FNOFMA.h_generated -[ 24%] Generating sleeflibm_AVX2.h.tmp -[ 25%] Generating sleeflibm_AVX2128.h.tmp -[ 25%] Generating alias_AVX512F_dp.h.tmp -[ 26%] Generating sleeflibm_AVX512F_.h.tmp -[ 26%] Generating alias_AVX512F_sp.h.tmp -[ 26%] Generating sleeflibm_AVX512F.h.tmp -[ 27%] Generating include/alias_avx512f.h -[ 27%] Generating sleeflibm_AVX512FNOFMA.h.tmp -[ 28%] Built target sleefgnuabisse2sp -[ 28%] Built target alias_avx512f.h_generated -[ 28%] Generating sleeflibm_PUREC_SCALAR.h.tmp -[ 28%] Building CXX object _deps/benchmark-build/src/CMakeFiles/benchmark.dir/statistics.cc.o -[ 29%] Building CXX object _deps/benchmark-build/src/CMakeFiles/benchmark.dir/string_util.cc.o -[ 30%] Generating sleeflibm_PURECFMA_SCALAR.h.tmp -[ 31%] Generating sleeflibm_DSP_SCALAR.h.tmp -[ 31%] Generating include/renameavx512f.h +[34/147] Generating include/renameavx512f.h Generating renameavx512f.h: mkrename finz_ 8 16 avx512f -[ 31%] Built target sleefgnuabisse2dp -[ 32%] Generating include/renameavx2.h +[35/147] Generating include/renameavx2.h Generating renameavx2.h: mkrename finz_ 4 8 avx2 -[ 32%] Generating include/renameavx2128.h +[36/147] Generating include/renameavx2128.h Generating renameavx2128.h: mkrename finz_ 2 4 avx2128 -[ 32%] Built target renameAVX512F.h_generated -[ 32%] Generating include/renamefma4.h +[37/147] Generating include/renamefma4.h Generating renamefma4.h: mkrename finz_ 4 8 fma4 -[ 32%] Generating include/renameavx.h -[ 33%] Built target renameAVX2128.h_generated +[38/147] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetavx512fnofma.dir/sleefsimdsp.c.o +[39/147] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetavx512fnofma.dir/sleefsimddp.c.o +[40/147] Building CXX object CMakeFiles/nanobind-static.dir/usr/local/lib/python3.12/dist-packages/nanobind/src/nb_static_property.cpp.o +[41/147] Building CXX object CMakeFiles/nanobind-static.dir/usr/local/lib/python3.12/dist-packages/nanobind/src/implicit.cpp.o +[42/147] Building CXX object CMakeFiles/nanobind-static.dir/usr/local/lib/python3.12/dist-packages/nanobind/src/nb_datetime.cpp.o +[43/147] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefgnuabiavx2sp.dir/sleefsimdsp.c.o +[44/147] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefgnuabiavx2dp.dir/sleefsimddp.c.o +[45/147] Building CXX object CMakeFiles/nanobind-static.dir/usr/local/lib/python3.12/dist-packages/nanobind/src/trampoline.cpp.o +[46/147] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefgnuabiavx512fsp.dir/sleefsimdsp.c.o +[47/147] Building CXX object CMakeFiles/nanobind-static.dir/usr/local/lib/python3.12/dist-packages/nanobind/src/error.cpp.o +[48/147] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefavx512f.dir/sleefsimdsp.c.o +[49/147] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefgnuabiavx512fdp.dir/sleefsimddp.c.o +[50/147] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetavx512f.dir/sleefsimdsp.c.o +[51/147] Building CXX object CMakeFiles/nanobind-static.dir/usr/local/lib/python3.12/dist-packages/nanobind/src/common.cpp.o +[52/147] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefavx512fnofma.dir/sleefsimdsp.c.o +[53/147] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefgnuabisse2sp.dir/sleefsimdsp.c.o +[54/147] Generating include/renameavx.h Generating renameavx.h: mkrename cinz_ 4 8 avx -[ 33%] Built target renameAVX2.h_generated -[ 33%] Generating include/renamesse4.h +[55/147] Building CXX object CMakeFiles/nanobind-static.dir/usr/local/lib/python3.12/dist-packages/nanobind/src/nb_ndarray.cpp.o +[56/147] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetavx2.dir/sleefsimdsp.c.o +[57/147] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefgnuabiavxdp.dir/sleefsimddp.c.o +[58/147] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefavx512fnofma.dir/sleefsimddp.c.o +[59/147] Generating include/renamesse4.h Generating renamesse4.h: mkrename cinz_ 2 4 sse4 -[ 33%] Built target renameFMA4.h_generated -[ 34%] Generating include/renamesse2.h +[60/147] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetavx512f.dir/sleefsimddp.c.o +[61/147] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetavx2128.dir/sleefsimdsp.c.o +[62/147] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetavx2.dir/sleefsimddp.c.o +[63/147] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefgnuabisse2dp.dir/sleefsimddp.c.o +[64/147] Generating include/renamesse2.h Generating renamesse2.h: mkrename cinz_ 2 4 sse2 -[ 35%] Built target renameAVX.h_generated -[ 35%] Building CXX object _deps/benchmark-build/src/CMakeFiles/benchmark.dir/sysinfo.cc.o -[ 35%] Building CXX object _deps/benchmark-build/src/CMakeFiles/benchmark.dir/timers.cc.o -[ 36%] Generating include/renamepurec_scalar.h +[65/147] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefavx512f.dir/sleefsimddp.c.o +[66/147] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefgnuabiavxsp.dir/sleefsimdsp.c.o +[67/147] Building CXX object CMakeFiles/nanobind-static.dir/usr/local/lib/python3.12/dist-packages/nanobind/src/nb_enum.cpp.o +[68/147] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetavx2128.dir/sleefsimddp.c.o +[69/147] Building CXX object CMakeFiles/nanobind-static.dir/usr/local/lib/python3.12/dist-packages/nanobind/src/nb_internals.cpp.o +[70/147] Linking C static library _deps/sleef-build/lib/libsleefgnuabi.a +[71/147] Generating include/renamepurec_scalar.h Generating renamepurec_scalar.h: mkrename cinz_ 1 1 purec -[ 36%] Generating include/renamepurecfma_scalar.h +[72/147] Generating include/renamepurecfma_scalar.h Generating renamepurecfma_scalar.h: mkrename finz_ 1 1 purecfma -[ 37%] Generating include/renamecuda.h +[73/147] Building CXX object CMakeFiles/nanobind-static.dir/usr/local/lib/python3.12/dist-packages/nanobind/src/nb_func.cpp.o +[74/147] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefavx2.dir/sleefsimdsp.c.o +[75/147] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefavx2128.dir/sleefsimdsp.c.o +[76/147] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefavx2.dir/sleefsimddp.c.o +[77/147] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetfma4.dir/sleefsimdsp.c.o +[78/147] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetfma4.dir/sleefsimddp.c.o +[79/147] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefavx2128.dir/sleefsimddp.c.o +[80/147] Generating include/renamecuda.h Generating renamecuda.h: mkrename finz_ 1 1 cuda -[ 38%] Built target renameSSE4.h_generated -[ 39%] Generating include/renameavx512f_gnuabi.h -Generating renameavx512f_gnuabi.h: mkrename_gnuabi avx512f e 8 16 __m512d __m512 __m256i __m512i __AVX512F__ -[ 39%] Generating ../../include/sleef.h -[ 39%] Built target renameSSE2.h_generated -[ 40%] Built target headers -[ 40%] Built target renamePUREC_SCALAR.h_generated -[ 41%] Built target renamePURECFMA_SCALAR.h_generated -[ 41%] Generating include/renamedspscalar.h -[ 41%] Generating dispscalar.c.body -[ 41%] Built target renamedspscalar.h_generated -[ 42%] Generating dispscalar.c -[ 42%] Generating include/renamedsp128.h -[ 42%] Built target dispscalar.c_generated -[ 42%] Built target renamedsp128.h_generated -[ 42%] Generating dispsse.c.tmp -[ 43%] Generating include/renamedsp256.h -[ 43%] Built target sleefgnuabiavx2dp -[ 43%] Built target renamedsp256.h_generated -[ 44%] Generating dispsse.c -[ 44%] Generating dispavx.c.tmp -[ 44%] Built target dispsse.c_generated -[ 45%] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefgnuabiavx512fdp.dir/sleefsimddp.c.o -[ 46%] Generating dispavx.c -[ 47%] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefgnuabiavx512fsp.dir/sleefsimdsp.c.o -[ 47%] Built target dispavx.c_generated -[ 48%] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefavx512fnofma.dir/sleefsimdsp.c.o -[ 48%] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetavx512fnofma.dir/sleefsimdsp.c.o -[ 48%] Built target sleefgnuabiavxdp -[ 49%] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetavx512fnofma.dir/sleefsimddp.c.o -[ 49%] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefavx512fnofma.dir/sleefsimddp.c.o -[ 50%] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefavx512f.dir/sleefsimdsp.c.o -[ 50%] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefavx512f.dir/sleefsimddp.c.o -[ 50%] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetavx512f.dir/sleefsimdsp.c.o -[ 50%] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefavx2.dir/sleefsimdsp.c.o -[ 50%] Built target sleefgnuabiavx2sp -[ 50%] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetavx512f.dir/sleefsimddp.c.o -[ 51%] Built target sleefgnuabiavxsp -[ 51%] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefavx2.dir/sleefsimddp.c.o -[ 51%] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetavx2.dir/sleefsimdsp.c.o -[ 51%] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefavx2128.dir/sleefsimdsp.c.o -[ 52%] Linking CXX static library bin/libnanobind-static.a -[ 52%] Built target nanobind-static -[ 53%] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetavx2.dir/sleefsimddp.c.o -[ 54%] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefavx2128.dir/sleefsimddp.c.o -[ 55%] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetavx2128.dir/sleefsimdsp.c.o -[ 56%] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleeffma4.dir/sleefsimdsp.c.o -[ 57%] Built target sleefdetavx512fnofma -[ 58%] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetfma4.dir/sleefsimdsp.c.o -[ 58%] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleeffma4.dir/sleefsimddp.c.o -[ 59%] Built target sleefdetavx512f -[ 59%] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetfma4.dir/sleefsimddp.c.o -[ 59%] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetavx2128.dir/sleefsimddp.c.o -[ 60%] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefavx.dir/sleefsimdsp.c.o -[ 60%] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetavx.dir/sleefsimdsp.c.o -[ 60%] Built target sleefdetavx2 -[ 61%] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetavx.dir/sleefsimddp.c.o -[ 61%] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefavx.dir/sleefsimddp.c.o -[ 61%] Built target sleefavx512fnofma -[ 61%] Built target sleefgnuabiavx512fdp -[ 61%] Built target sleefgnuabiavx512fsp -[ 61%] Built target sleefavx512f -[ 61%] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefsse2.dir/sleefsimdsp.c.o -[ 61%] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetsse4.dir/sleefsimddp.c.o -[ 62%] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefsse4.dir/sleefsimdsp.c.o -[ 63%] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefsse2.dir/sleefsimddp.c.o -[ 64%] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetsse4.dir/sleefsimdsp.c.o -[ 64%] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetsse2.dir/sleefsimdsp.c.o -[ 65%] Built target sleefavx2 -[ 65%] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefsse4.dir/sleefsimddp.c.o -[ 66%] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetsse2.dir/sleefsimddp.c.o -[ 66%] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefpurec_scalar.dir/sleefsimddp.c.o -[ 67%] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefpurec_scalar.dir/sleefsimdsp.c.o -[ 67%] Built target sleefdetavx2128 -[ 68%] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetpurec_scalar.dir/sleefsimdsp.c.o -[ 68%] Built target sleefdetfma4 -[ 68%] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefpurecfma_scalar.dir/sleefsimdsp.c.o -[ 69%] Built target sleefavx2128 -[ 70%] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefpurecfma_scalar.dir/sleefsimddp.c.o -[ 70%] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetpurec_scalar.dir/sleefsimddp.c.o -[ 70%] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetpurecfma_scalar.dir/sleefsimdsp.c.o -[ 70%] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetpurecfma_scalar.dir/sleefsimddp.c.o -[ 70%] Building C object _deps/sleef-build/src/libm/CMakeFiles/dispscalar_obj.dir/dispscalar.c.o -[ 71%] Building C object _deps/sleef-build/src/libm/CMakeFiles/dispsse_obj.dir/dispsse.c.o -[ 72%] Built target sleefdetavx -[ 72%] Building C object _deps/sleef-build/src/libm/CMakeFiles/dispavx_obj.dir/dispavx.c.o -[ 72%] Built target sleefdetsse4 -[ 73%] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefgnuabi.dir/rempitab.c.o -[ 73%] Linking C static library ../../lib/libsleefgnuabi.a -[ 73%] Built target sleeffma4 -[ 74%] Built target sleefdetsse2 -[ 74%] Built target sleefgnuabi -[ 75%] Built target sleefdetpurecfma_scalar -[ 75%] Built target sleefdetpurec_scalar -[ 75%] Built target sleefsse2 -[ 75%] Built target sleefsse4 -[ 75%] Built target sleefavx -[ 76%] Built target sleefpurecfma_scalar -[ 76%] Built target sleefpurec_scalar -[ 77%] Built target dispscalar_obj -[ 78%] Built target dispsse_obj -[ 79%] Built target dispavx_obj -[ 79%] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleef.dir/sleefld.c.o -[ 79%] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleef.dir/sleefqp.c.o -[ 80%] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleef.dir/rempitab.c.o -[ 81%] Linking C static library ../../lib/libsleef.a -[ 81%] Built target sleef -[ 81%] Building CXX object CMakeFiles/DeepityProfiled.dir/src/SimplePCLayer.cpp.o -[ 81%] Building CXX object CMakeFiles/DeepityProfiled.dir/src/DiscriminativePCLayer.cpp.o -[ 82%] Building CXX object CMakeFiles/DeepityProfiled.dir/src/DiscriminativePCNetwork.cpp.o -[ 83%] Building CXX object CMakeFiles/DeepityProfiled.dir/src/RBLayer.cpp.o -[ 83%] Building CXX object CMakeFiles/DeepityProfiled.dir/src/ConvPCLayer.cpp.o -[ 83%] Building CXX object CMakeFiles/DeepityProfiled.dir/src/ConvPCNetwork.cpp.o -[ 84%] Building CXX object CMakeFiles/DeepityProfiled.dir/src/SimpleConvPCLayer.cpp.o -[ 84%] Building CXX object CMakeFiles/DeepityProfiled.dir/src/StreamAlignedBatcher.cpp.o -[ 84%] Building CXX object CMakeFiles/DeepityProfiled.dir/src/SimplePCNetwork.cpp.o -[ 85%] Building CXX object CMakeFiles/DeepityProfiled.dir/src/ModelIO.cpp.o -[ 86%] Building CXX object CMakeFiles/Deepity.dir/src/DiscriminativePCLayer.cpp.o -[ 86%] Building CXX object CMakeFiles/Deepity.dir/src/RBLayer.cpp.o -[ 86%] Building CXX object CMakeFiles/Deepity.dir/src/SimpleConvPCLayer.cpp.o -[ 87%] Building CXX object CMakeFiles/Deepity.dir/src/ConvPCLayer.cpp.o -[ 88%] Building CXX object CMakeFiles/Deepity.dir/src/GaussSeidelPCLayer.cpp.o -[ 88%] Building CXX object CMakeFiles/Deepity.dir/src/SimplePCLayer.cpp.o -[ 88%] Building CXX object CMakeFiles/Deepity.dir/src/DirectKPPCLayer.cpp.o -[ 88%] Building CXX object CMakeFiles/Deepity.dir/src/ConvPCNetwork.cpp.o -[ 89%] Building CXX object CMakeFiles/Deepity.dir/src/DiscriminativePCNetwork.cpp.o -[ 90%] Linking CXX static library ../../../bin/libbenchmark.a -[ 90%] Built target benchmark -[ 91%] Building CXX object CMakeFiles/Deepity.dir/src/SimplePCNetwork.cpp.o -[ 91%] Building CXX object CMakeFiles/Deepity.dir/src/GaussSeidelPCNetwork.cpp.o -[ 91%] Building CXX object _deps/benchmark-build/src/CMakeFiles/benchmark_main.dir/benchmark_main.cc.o -[ 91%] Building CXX object CMakeFiles/Deepity.dir/src/SimpleConvPCNetwork.cpp.o -[ 92%] Building CXX object CMakeFiles/Deepity.dir/src/DirectKPPCNetwork.cpp.o -[ 92%] Building CXX object CMakeFiles/Deepity.dir/src/ModelIO.cpp.o -[ 93%] Building CXX object CMakeFiles/Deepity.dir/src/StreamAlignedBatcher.cpp.o -[ 94%] Linking CXX static library ../../../bin/libbenchmark_main.a -[ 94%] Built target benchmark_main -[ 95%] Linking CXX static library bin/libDeepityProfiled.a -[ 95%] Built target DeepityProfiled -[ 95%] Building CXX object CMakeFiles/ActivationBenchmark.dir/tests/tActivations.cpp.o -[ 97%] Building CXX object CMakeFiles/DeepityProfile.dir/tests/tProfile.cpp.o -[ 97%] Building CXX object CMakeFiles/GflopsBenchmark.dir/tests/tGflopsBenchmark.cpp.o -[ 97%] Linking CXX static library bin/libDeepity.a -[ 97%] Built target Deepity -[ 97%] Building CXX object CMakeFiles/GaussSeidelMiddleLayerVerify.dir/tests/tGaussSeidelMiddleLayerVerify.cpp.o -[ 97%] Building CXX object CMakeFiles/pydeepity.dir/bindings/pybinding.cpp.o -[ 97%] Building CXX object CMakeFiles/DirectKPVerify.dir/tests/tDirectKPVerify.cpp.o -[ 98%] Linking CXX executable bin/DirectKPVerify -[ 98%] Linking CXX executable bin/DeepityProfile -[ 98%] Linking CXX executable bin/GflopsBenchmark -[ 98%] Linking CXX executable bin/GaussSeidelMiddleLayerVerify -[ 98%] Built target DeepityProfile -[ 98%] Built target GflopsBenchmark -[ 98%] Linking CXX executable bin/ActivationBenchmark -[ 98%] Built target ActivationBenchmark -[ 98%] Built target GaussSeidelMiddleLayerVerify -[ 98%] Built target DirectKPVerify -[100%] Linking CXX shared module bin/pydeepity.cpython-314-x86_64-linux-gnu.so -[100%] Built target pydeepity +[81/147] Building CXX object CMakeFiles/nanobind-static.dir/usr/local/lib/python3.12/dist-packages/nanobind/src/nb_type.cpp.o +[82/147] Generating sleeflibm_AVX.h.tmp +[83/147] Generating sleeflibm_AVX2.h.tmp +[84/147] Generating sleeflibm_AVX2128.h.tmp +[85/147] Generating sleeflibm_AVX512F.h.tmp +[86/147] Generating sleeflibm_AVX512FNOFMA.h.tmp +[87/147] Generating sleeflibm_AVX512F_.h.tmp +[88/147] Generating sleeflibm_AVX_.h.tmp +[89/147] Generating sleeflibm_DSP_SCALAR.h.tmp +[90/147] Generating sleeflibm_FMA4.h.tmp +[91/147] Generating sleeflibm_PURECFMA_SCALAR.h.tmp +[92/147] Generating sleeflibm_PUREC_SCALAR.h.tmp +[93/147] Linking CXX static library bin/libnanobind-static.a +[94/147] Generating sleeflibm_SSE4.h.tmp +[95/147] Generating sleeflibm_SSE2.h.tmp +[96/147] Generating sleeflibm_SSE_.h.tmp +[97/147] Generating include/renamedspscalar.h +[98/147] Generating include/renamedsp128.h +[99/147] Generating include/renamedsp256.h +[100/147] Generating ../../include/sleef.h +[101/147] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetsse4.dir/sleefsimdsp.c.o +[102/147] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetpurec_scalar.dir/sleefsimdsp.c.o +[103/147] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetavx.dir/sleefsimddp.c.o +[104/147] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetpurecfma_scalar.dir/sleefsimdsp.c.o +[105/147] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetavx.dir/sleefsimdsp.c.o +[106/147] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetsse2.dir/sleefsimdsp.c.o +[107/147] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleeffma4.dir/sleefsimdsp.c.o +[108/147] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetsse4.dir/sleefsimddp.c.o +[109/147] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleeffma4.dir/sleefsimddp.c.o +[110/147] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetpurecfma_scalar.dir/sleefsimddp.c.o +[111/147] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetpurec_scalar.dir/sleefsimddp.c.o +[112/147] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetsse2.dir/sleefsimddp.c.o +[113/147] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefsse4.dir/sleefsimdsp.c.o +[114/147] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefavx.dir/sleefsimddp.c.o +[115/147] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefsse4.dir/sleefsimddp.c.o +[116/147] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleef.dir/rempitab.c.o +[117/147] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefpurecfma_scalar.dir/sleefsimdsp.c.o +[118/147] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleef.dir/sleefld.c.o +[119/147] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefpurecfma_scalar.dir/sleefsimddp.c.o +[120/147] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleef.dir/sleefqp.c.o +[121/147] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefpurec_scalar.dir/sleefsimdsp.c.o +[122/147] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefsse2.dir/sleefsimdsp.c.o +[123/147] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefpurec_scalar.dir/sleefsimddp.c.o +[124/147] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefavx.dir/sleefsimdsp.c.o +[125/147] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefsse2.dir/sleefsimddp.c.o +[126/147] Building C object _deps/sleef-build/src/libm/CMakeFiles/dispscalar_obj.dir/dispscalar.c.o +[127/147] Building CXX object CMakeFiles/Deepity.dir/src/DirectKPPCLayer.cpp.o +[128/147] Building CXX object CMakeFiles/Deepity.dir/src/RBLayer.cpp.o +[129/147] Building CXX object CMakeFiles/Deepity.dir/src/DirectKPPCNetwork.cpp.o +[130/147] Building CXX object CMakeFiles/Deepity.dir/src/ConvPCNetwork.cpp.o +[131/147] Building CXX object CMakeFiles/Deepity.dir/src/StreamAlignedBatcher.cpp.o +[132/147] Building CXX object CMakeFiles/Deepity.dir/src/ConvPCLayer.cpp.o +[133/147] Building CXX object CMakeFiles/Deepity.dir/src/SimpleConvPCLayer.cpp.o +[134/147] Building CXX object CMakeFiles/Deepity.dir/src/GaussSeidelPCLayer.cpp.o +[135/147] Building CXX object CMakeFiles/Deepity.dir/src/DiscriminativePCLayer.cpp.o +[136/147] Building CXX object CMakeFiles/Deepity.dir/src/SimplePCLayer.cpp.o +[137/147] Building CXX object CMakeFiles/Deepity.dir/src/SimplePCNetwork.cpp.o +[138/147] Building CXX object CMakeFiles/Deepity.dir/src/GaussSeidelPCNetwork.cpp.o +[139/147] Building CXX object CMakeFiles/Deepity.dir/src/SimpleConvPCNetwork.cpp.o +[140/147] Building CXX object CMakeFiles/Deepity.dir/src/DiscriminativePCNetwork.cpp.o +[141/147] Building C object _deps/sleef-build/src/libm/CMakeFiles/dispsse_obj.dir/dispsse.c.o +[142/147] Building CXX object CMakeFiles/Deepity.dir/src/ModelIO.cpp.o +[143/147] Building C object _deps/sleef-build/src/libm/CMakeFiles/dispavx_obj.dir/dispavx.c.o +[144/147] Linking C static library _deps/sleef-build/lib/libsleef.a +[145/147] Linking CXX static library bin/libDeepity.a +[146/147] Building CXX object CMakeFiles/pydeepity.dir/bindings/pybinding.cpp.o +[147/147] Linking CXX shared module bin/pydeepity.cpython-312-x86_64-linux-gnu.so +lto-wrapper: warning: using serial compilation of 6 LTRANS jobs + + +=== Tests === +Internal ctest changing into directory: /root/Deepity/build/Release +Test project /root/Deepity/build/Release +No tests were found!!! diff --git a/pydeepity/__pycache__/ConvolutionalPCN.cpython-312.pyc b/pydeepity/__pycache__/ConvolutionalPCN.cpython-312.pyc index 29095bede62c6f0cf7bfa9fbf16a9513a1712277..0af672c7a6e1b4ed816d3ec51160f0c4017c2079 100644 GIT binary patch delta 145 zcmaE$b6SV*G%qg~0}vRznUQsfdn4Z)7D;XWqWt_4eV5eKg3OXi{esFAC~LC{>lP-) zTa$0H+ewH5^}JyC(ZFy))M91S3dRYEGfJl9u8i8O$DzW+yfW(Y=1{Kbj7oAqWep5B rL?x$3Pl@huy}=`Rok#g1kMaVit32u-nHj_sHw*J{Z~+aK5pw|mHQO*_ delta 166 zcmX@D^FW91G%qg~0}#aCoRTHLvytx&iYugNFa?ZicZhP+_-(ZFy?)M5qWgv1#oQ*tkgT5Oi%P+?+T z5p`v=E7x>Jb!DKc28J7=lGCFnMt@>pnE${g-oQTX~s-Knb6F>$P8&67olU$rh`@EQNuYA}2YK>m@y$ z$~7Tx>NWM4RWtBJH1}G1tg4mqmR_4`2brap@3E*QO{AJM6VX~rMB52&pej3f{@H2P zd9_ruc!|&cfP;4*ra!*cP zX~@f%yecQJHss}Ejq37qs@qSV|&a&(lUC(ud^O@Ws1$G2>mqAqyS8m!##h-K;{vL)6*8(88!zU-ns&DTxWf68xO zd~WXr%U!;XdZ2Q}mJ1-jpIDj&vi6p^!stepGYDCsZjI1_;PeMdMG)6c{O-<0iGzNG#u= z1YW|-8Z;i*$w)|rwu(w4AsKra+@xkr1VSKSk4eoGrJ5-M{|TyDqvNMS0nHo;suZ=| z@3T-RJSkv+Xx7na9B^)ifqW)fhVpVC2GaE{bfdHa2yUqIArOYgE!NEs36S5{Y{*#g z#YQS?GIr#6_X-CF|Z*J^Uft&}q zO601DyE0RaTn%!y$kmZDPi6yh^~g0K*NEIkph2@JE7b6^XIrQvSBt;3vfAI~}%u#H;cce@E%5|*shBr5zBC&{j(;A*~tuEtLW zau%ewL^=wpZY0N1*Q|)9Z2vI#=pK_WAKIO;z2t%M&ATRH{Ngdp`7w^$vLg>xf?dNd z4euZJ4||8t4Zl77^Woq5U82SB+MaC$Ptp2}p@Amw0kDF%;1^(VxB}5uXdMZU-!s6% zd_i|$L5ct>Dj-GV6qOD$c$T~_IWG}m-76(NJG1X>Pqgh0_+M%T(n6ZZ6d5AV^MstK z5Xm9#9GQZIp%3O&LN>wfc*)ny5sp?t;}ikcZEAa3;{pu8TLcDRtz`F@9Fxx_=pJpy zDo+#$g}|vLW!kno8V?4d%AU4d^A;FXK?n2zCNWpND>X99O{Zvu1vNelH>Gh( zt5HyrQwVKj4bb%&&a4D^whmtvMfcB|(2dA#LV{6`3P_rfJmH)vSRzcEg(oRM#}v&T zlbC`uE6hd?(GAc*L9A>kDuN&w5$jnTqyF(YVP5bSGy*S}*-UC%X2WyU&s=ca^Hir1 zB6q5H-SO$XVp-KR_r~#ip2oS2XYP3DoRj`Z4~QEUowZkzi_WUqx{tdTT@};f^~#TxxtEXr zI+?MPlKRhyrNohO5@+3X^QWZ^i&b^go=ZI!x)+6~7KEK?VdwkZ3!MkkodGlQ3p%^XX4uGOdN z=Qnm{P1k;pRH#}x@v>n>7zX^Mw6?1{cCbXcYTdpvLx8!F7tysaCz(1dJ z1_cU-P>x55tZEBYYSLBeg2Oi+k0dxyO~NTJWzubj`C(jW_M9*TyteWv$xciwyR*4t znvvIK(_Q`%7Q*YZCQCm8=5aa%OL%0`_GA|FAN9sViABUiiM1VWFH%;09}#34_X-@m zR5_tAn@ibfFG%SjBz;I;M1mRE6S~;_&{V;m#)fynF_N=(aWh?)`)2x5rfZIrW4;m2 zlqkGm{;2du>CFST4&6L7zw59rOdmaFFt5sbUmYjthO-4tj^X4aFhz!G$sRe^GE8vO zYnZLyoW1WUGFpd91CSJihAw*)!LYspNaFU+=!-Ixxo{$XiY}-Tjhbkit=h&|^T> z_aPYJ6m)$Cqy`zJ2Dwz%wZ2r}4bw-C8;<$yT?m>%?zZ`p(%Yr!A^*bA$@I|4`5sA^ zr^BZWCNY7i8sz?eP*tA?0pzLrB_jGUNQN28D56{`!qX{`Y1`JoQ}IwtjVtl>)%Tk} z^nKv_$+qifK1$w5{_ORS0bk-AFRsRy9exfhjx7BU$bZepSwwb$^6zF6?Pl<9mb`b< ztr6oD80zC3+xh@}0R~WZ12F*DfdJ~5sl41U({NRMcjW5G{D!9yT^%YQ{=?pB&*+>t?8yrofy@i6uBuoTyNqnqVA(C z1wJb8p77W6-YE=kTHPu%8RIYo;8hkVfVrlN-0Ry9g>M)>?4R5K+YYnM}55x~gJ{Xzz?L!oFakm*7y14X_p@k#I)9`mfS~xM5 zJ~1|bBwQHU{x_(oHFmSuG3Y)ff-74Iig6HWt&amm@i0s^#^K6{Uv2BhEd((0E1Q6M zoL|xNPbxx_C(Juvf~JLerx(@jy#kkr@W^8oTCsvNV5Ybkm?BeLKksx;Btm$_p8Kdk zUxs1yTOlL*p8Vlmn9V8n`oPaZ)z8U<1ID~SFI7KzDJ{+@W>KK-89j#4D}uY428 z7vCdOp$`CDZEYUw`vB4Sm8G7}*Mt--xb$jS{pa-Uc?7*8ba~yY{)ZHgf h-6svdC7%1F{&UL<9Or$LF4S&K*KYkA!8bP2KLH_p`S$<- diff --git a/pydeepity/__pycache__/DKPPCN.cpython-312.pyc b/pydeepity/__pycache__/DKPPCN.cpython-312.pyc index 9ae2834c2c342a12cefc53d0ac374a3f6ef786f7..9064da17f67df2f9db7fce686bdac30895cb866b 100644 GIT binary patch delta 60 zcmZ2r({ICfnwOW00SFA<%*gsGx{*(nOHxt4C_leM-z7D*AhV=Wzo0S&%Gw;uRVU1N JYw{gg7XU%r6np>x delta 77 zcmeCTSzyC=nwOW00SM-3Ps^$k+sLQNrDf!76%$&VT2vh4m6)6z6Ht_&m6}{q9Fvlo ZT98>%8BW{LkOG=`-(Q>IjxTz&sVnoSeBvZ^%CS%vudbo3UccS6U z`p&HAm4R9h0UA9NK^}t2O@K)4LnIV!{Zb%(%uCQbH9~5|n8puo3lu?6U{%^-`_}K= znb{>N*|KA2K%TjC@0~m6o^$T`-GhC-x`OMMt?z&98P<*L*s^WMJ=kc z%6ZjS_bOMEVq!LNKItcAJ2{*3)96diX3i)4F7&Cz)U0+s>u2{W+m%tJnBJ)rGq0)h z!|~#s`>|x$_PdM8eM+wDH!7ZUYLGCSfZC2ORJV&E7gTt72Na6 zorM%76+Ntd6>AuZq25qlQ+&0k_=zi;pWLCi*n^*1RFA5~#7@Q578SIT)UpTsbaA&F zNwr6SHI3M+pOGV(_DEOsPM7T0+Wpz6Uz7cMyT3c?&&vLuc7Jcwugm_vc7K1=-!1zG z+Wmu3e@}5ljtsR&wuK|WwO5V|w@0=|@AMV-$dMiNT-*srJL@m*@&^`Ge{fMbuKGjZ zz*9kH=ED5^nK>h!Q51ce%?oZ?W#78S*%{ZgoGP2+{;DgkvW7=bGnU|GpIvZm?o7_R zwr!NJvI(q_V-?QLWjyMJZB?BW?)Z6D=YpLvJjW_qd3KRk`0_Qu^K8awta969n}OBb(J%Ijf*C8 z{Tk51rdioGJdarp^J|vJY-?E5@+NQ@q0o`&;UgrRgZ#O4xZU~2) zY?WKpn$K1YUsz@CJ;#>V1eOQ28ivg(u2?lhJj=`JhGVjM&tek>Geb(oRAY6+udNzQ zRuS$>glNq&@wo^S&ld)LOZQ!%%bCx`iiN`x4nN~DE`%%c%yE4tIAL#wLzBxumo}3t zu&;5T#C3Y#6<{y6%RKJ;Hu2VW-8x(5Rm<`6%wVp=Cw*&$Z^g93UoigMbdELZCb&kt z_8Vduh)62SPXeu_NOCOSlUpYe;QM+Te~CAnoD7)&KCIRZpV=Jq4Yn{l_rk@q9UMKM z6OOvqs8k?SQpg%1b7@n5cH|JFDKQcutFIZtG8}&^qjkSox5|cXH<<(B@d}tw;Sk6& zFNKR>Ng*l5OkOb>Hc$ff59f*Bf90V%gpD3Iv>v+}wQe(&E`UHao}AntAEW<=IPf#OEc@stQ^V_gKyFz)wi| zh7H(F9U^2^EKr5cd&P1rpHG?=P?GFnuU2W!SP3sjz1t#M*91AC0Jk24ElD7%6V5YdY4k52f4caB$gLFjVLfWmNr zAaw#WryA-xlD}9Krt73CtT)+o;c$-jsLx%x%$nQ}_uAeatq&;|+<*%78lH~@vFUhW zCK^tGUB)obbDDZA0YK3y+tA3a6Y2|Epz~I^Y8`g74#CIK^saJ zSdva-1Dt@3T~w|lzWQxNG12>!@(r~HE4!fnDSU)t zaw-9flKM>oQv-dOPDsgv6b2c=p%|PwnE0gS__;)ofrR4-1-+$`RKa9&N+plp9J>)5 znyTS+ObJl?(A2zezXdt;rcBQ3mfxJJH{(`lxeE1W&{HZI4iq2uS}FzGqJfQ_6Ij~Y z1Mt5$lyyZ-A9zUM&e0a#?jDm*YZz9_ghYJN*72{cjiW)^e>EOa77~lf>u{SE)ngH{ z_9{hXpK?!~!v@n{13DqlN+lBAQYn`VQXaP}q8keY*=XC?GJRou{Dv67UF@X6{x2IjxkdT(~!*LK|+d7$kV+t9g*TSMre1=r>PKAl^VXGsZDfNC)Q?~;bX!ULBhg+#SYT;%Ccz$NgOQEgO}%WNzqG- zq${=y?k8c=atYCgF3}GV@js-+AobFQ4$=q*keu6@)*mVWANCEl()5r~hIh8QsHG{p z_O!Cp(v{w!RyVbJsMSlYK5F$-Yd{&;-WsIVkb+gVq175z^uCAN(F%#0#pmLSFb7e4 z)=s0bMFz5=3@i;TCYR1GRTs6Qx~S#jH*qf}{y16-(m^h7%6JvxqMxoC!dPkBa7jCM z8485R8`=d9qa-~qXrFVlbJG`@+wkj9XwtCi4A%rNR156zA!#sPoSR*^oFn}z_!|v4 z7bYxtWQz^cEXmiP*U8{XFF`ILEgt4V`&h7hWSOBy8e829s0Zg6pIvpx#>Fax%N17P zBDHbtsB<`0d}Z0@W+C2e#fAq6uPL@f66?_MBJTv-l%7#+RARp(AK|&dyj7SjRksB< z&2^x$Lt6!31HN9Mnz@YF3*rk#V3fcZKmhd#$5yJlV4%}3nJ^jGaB9U4nw4!6E2oQH z^zu^#o+j`|1Rk@r*hd4q0X#Y-#v|M7>LewdwD6?Gz7u^Xk5wm(b3-he$T+I z*WT|Ryj6UE=g8fKcaQ$v%dIXYJMudvm5rp6^i8C5KEHy#mkJMZwr&5zvw2Sy2e9sw z&*Kbcbu?)roqMPN-0@m;yX(uR@Dx2pllFX8O{(IoB(6^^kQYeyw~g&{Y@ zO<=cD&41VSr4>{N0n|0 zi%6xrAB|5}x}k{_lZ4F_z>|r69zR zlm#_3LC2z=&>trmAYl~dx`KcNHs;fi`|YBXa{`}(W1!`Lo(LDPICCpUi-7VpfL%Nn!v1 zF#tDW0B&Cv11@Y(-Z5n!&w|J4?oDrOVN&dmZDNw7#iLC6`c@`wdRfvVR==6H`h7f` zG%Q5|u=dvgpQL^fp(yhkr5AJQqR3?)3C*onex%b=OnjlYDRxQ~DRxRvrQocjL$^x( zW}v*sgO#`$Cw8I4&dsbgIOK9b$$JXUc2>AtZm0O4OtL@5x?(6pCtaP(B(%*ne~Ne*ft0(I4mTPoBI0 zB`}LB5+&sLbd#5jrt}GdF0Y1a3~F~Ibb<_buIm#ePujR3wd}g~lZDX3T3!P{3gM14 z0wjP(?Hr9f#>`Q-rVI|ZveeSa-gQ%}N7+8o>ZMj6wfd$>|8?|Y7ytc_x7UC6?(@~ZdWWumI#r!K^VDzu{n?RI z|Mk}J-@owT^>3dFQYdgWa;hLb3Pnjb!2zoH$JIJ@Ik%0f7esheRP<75#Wfo?wNYz% z19h@!q`!pUX$X&;Dekx!?kbgF-%BM~%n+2`6o&~=W=5PKFin8MIYHrqAlofS4~sHD zupLDa)N!nG2~L2pKziB5E3kgJBsg$VUy$$ex ze-1i6Ogx>|)}Kf9XAMT|zpSXe?&>75)Ji3bL(AF5{1@{RNIuyShQk_88W0J$65_%FL%e9 zomt(P6|>Ys1=MJxhoTKyAW~bvQu`1Q0pbKfQ44)={nFG=5z$HxrtyQ(0!1DQP^DY8 zZ~e}l*pT=l7m_=Gl&RRYLpSqkr^fty7Zzi5}`L5g{8TAUC8- zvMx_a=VV_#C|#2D*i`IX+>Z-9KGotUK#NZ$&&B){XtLfirJPIq>4VZ9X-Lu&dnG;j zhCI_(U%dH$G9Kc7s~$frWl}$v@tl!^SfK`}T&Vb#>lpU*svPv>mn=WOZ26^pg}X)0 zJkM+{BuOpN!^SAqFeF31A-y5_vM%|tONt-wmt64Rx6I4qvL4$j*~+{GEKZo+)qB$u+R~qzmL{|h|ZO~gIdRoxi8ua#vt_pewJ?Wi*U4E;e zbTvkHMz%@tWp}iP)TG+J@(87A;_Tnr$s|1N~Kxd84X2#|@Jew1UN%>Dx7JiMwUZ zDVjN(8Qgis(>OD1jeV1uRWdHE$c;)#9hp3logCM4&uc5kvMa~4D<`rm;&I`4cHu;J zLE!9omYvA5@ywSrLu_1YFzETLqFJ8mmpEgZVBgaWle^exxP9BGf$v;SThXeOg5fhy zTe7(4Ya>;UM0CxttDu#bT~J*x9B!OMRN?HJMUtVHWU?eShloNmu0tG|Zn;wR8Dv09 zVSu^>2PtVQ&p_m2JmcKuzi=pCL_@bMII7J-u&T{`NEbWhfY`hcxgmA|#+f!bp?_s+ zQja>H(F_}+E!1)$dj&P(h%80uifOYGuTF~jS*_~W))l6Ox^AdiTZVD4trAmR2TCLn z(mTDnn1ohIILs+%?h^6XloQ#Mcn!?qYtu9Oh55;8ZSu95Nqy?o$>}*#0*$Xa1;*_f zNvsU3)Jle@iMK5>2-qvTS6B{e@>xM0F7*@ z(9iURIjzQg4STkhEZE{T%fK+z!uu*12p@?i=L*)+5*T(&Ms`@RsLL!d^9ob|VfvU> zgBh7s7CQKVDhSduGg(dskz(&A>@JOSA;8@`nw5EAi7~-dCd1bKbTQ-WYVVUjYNT;vM~&V#PN-Kk2D*bmoDQ7 zo|jKX!aFGG(qU;;o(97SuUcU|Q1W@QzIb|W>fV7)-*f;%9(J`9`F^dNYyS!L1g1BM&ybU9#GjZNdGZIU-3$@2R zpJ!s6w#AXfNRi@0rn`yS13)I{T?7y+N+-DdSR(jwNB4R{JSL^Sz3VA~DpGsTdRm|= zp{)XKle+e-w+pmG!n968*SjRO<8e2kp{P=LN4*haLc^zl?9gy3)bK*j!pVhsU7Am3 z>$C8$%fAt=1HF?Y!~?mIN$@YCj~^tU0R$>!VqFuS9CS7DJi?lF4VJ?DX;j)M6!Ic~ zXB57VCLbX11p?a_4v$Pj=7+Jwlg(ia zqnaKYOJp8P0C&B0`n%_gr*NOYz+>B<)?;O8F19i2ch6spF-y2Skj|AfN()MF+g!~;8vgbYi%@}c}P zvx+6(<0MuQMeDM94ul{~_8T{Z(OvVzwkZtPrM9LFJyH(u7Tf1__b^(CGhkKbhw!E+ z%knpgmn}#0Fv)WG?j#+}!}ki}Dcm6`v)&paefpz$oAq^*|B3qsvZFk%Jgn}-BH#*R zq<1u&__y$|OTxPr5zK5qrHAeudZZlvUzD*pU4)GJDQxP=51%p$=Mmzz)@T9`MSp^l*)(P8&Gf_DvLZ%FHk)Iac@*Z=tSZ%zKRt^32a zBWrC(?&y!&MmFM7|Ils)(E&@Z-1u{VPm8ojl%$!hhKaG%D1n+uhWJ*P9cVu3v0rL7 zrEM)DZA*-`AlfIw6IXsMkiU)$6gN>%g#OiJ5O*qmkeYsJf^(x5#GFcy5~2QpXK2)pRzGWCi=WjSuUE8i^vf%6 z&Dgk{;d)3=ln8M3oW7vp#j`DC+POKG?&W&mp8FzC;^wg47z@4#v z2WN&^(!_-*y4k}leVcB^xMd29truj%D#R#4kH`a6v5h=!gA`==K69W}kz2eBZuvO^ zQv^hSA~q=ljPgHn&|v=#muN02)OQqPpuSI%NR@9cuXPUH8TxMKZssqZy?6DU+G_24 z*WaD|`_3~Dlr#VD{egd(> zAT<+p%S9@*p|0b*MTWV&;bH?#e`r*yBM0YSC4$g{*j^JGuFfT7-L0O5&pVx027@!P#`58fWUtAF?6 z-HQ+RAEVT8{C=>{E&W_J7vdcXRpXZ{NE5cI|fU zPp{vDy_|TUoY-wIly3@qc^Am9%w9gXr3gC<6O|U7>Prco>Prbd1(?(oJw2wtmeeh_ zL>?3*S;aL=o;dFUlxD^0n@7z_Eiluli8L`td8O*olFdY>!xhZSq(0UBM4l70EV{1! zWJ?r*po#^BC3Yro3kLYOwc`&?-8}Ve{+*R?uRLr$w4R`zq}0~Aotz@FB6atzrwKz8 z=v{9mtWDZ8u-;Bs2VtFrbxA$_>)pWCdn9$>aWAk?%fwH;MWw2?jYFWI6j{IGN(=JB z;f2YC;>l?ALFDQpx4EwBReXDhK7^F3wr}NxOv7}C0;lb|m8dL@uQd3E#4Y45^{;pM zMB`Nu{zui9n9Yi~BzYR@nmLi%Z23UryV~VuTph_=9#uJo(0VninY+9JYM8)KcE|Gs zE(2_>W(|u3KZxUFV$icA=z=EnLZwnpD(+Gs!cd*%~z|29rbkP@vRe|hVbc(IWD?W`Fz1O^LbIk(Q`v`8p@P%x zoYMu8pCLeb8mBXbQz+mjK(GhJVAQcKGmor~TZ{Nq8PaA4;xmmM-%P@mH_37S zjX1*zgB@satf9~!l{-kHF0_bZ2}*Sy6I>M68JD^dz|i9 zLEm;02|s=c&V><#ucQzeD5pbhKF`Um$$vaj)OuW&<&UI&A4!A%DD`|K4SXaW{UrHy VS?>Eo{lh)kwLRJYkmyOQ^xqqe7eW94 diff --git a/pydeepity/__pycache__/SequentialPCN.cpython-312.pyc b/pydeepity/__pycache__/SequentialPCN.cpython-312.pyc index 4e46af24705651378b67c7a5ef3e969c6320821b..23153d42a2d5d0f9fead088b5ca2bebdec782909 100644 GIT binary patch delta 2204 zcmZWqTTB#J7(Qn&>@4iCyRaaaT^QJ1c1I9Yyk8InE7VH`TD2R^X4n~Uac?~{s8AAp z=mWG(L{DllY1N0O(WFUhY@3>QO%#i#<>QUYpR8m@8O6+Xg(U=(`Wi)NA#3l3{>$fg}EhjYqwEzLSFE4Op z0TXq*B+Jo+G^7yGPRe1j3ZNQ50GM0lhhYGql7VVxz+5uEqr?k(k-_kS=RJAG59I11 z27ez#4?W0!!WP0lio@m9%89E>*o0^zhEHx)(V_`OBB^+)KT0G`5zRT@1ZI`n9WIsK z2l3KR?Pc_9n;VDePquQ#c*nM7&3rZ<*wPs-o{SH(Q$As8_&*b98tHyc5bBwzx)?B5 zZ*lyXEz9{PV$)nL@iAo#Qz;apKXabO`TN*n>0~CJP|lljz8aTgV)o!A`;o=eZLjb( zGEwCjJmns9PIKii5HFce>?Tr5PJ{Q+LrT1VzowEUKr)}ZnSO2$lpE^8dE8m9W}5R8 z-^Kg(4wK6W?=m(`mIJJyoelwe>3~Db1FbqvPdV0_PI~E0hu@5fswn1{VBf`NegN-V z#SB=6tbv7V8R)jYgrsRoimV4!3%~}@n^GCwDiI9$@a5m)+D(UQ9nW9t=Q+0Ni; zP75rCj|=^In?V<2E_mq%XJN}p@94%G4%duB$U1~6N39|IvJT(0qxznedH0Y*uA*ay z#|}?eCb~~?GiBjyS@@b$oa97y;BluDcfWApZLqbm|8|ge0K5Xbatv~bEHH1_KsWcr zwPE-3naor>>3vt2T}g$TcUW{B)@i-ZXEj#P?Y`ypqre7r@;iigvU=O4kqcWsYx}C` zY|YuWv&+xnbG7F@=ej;}UK*jF_)cS;ZmsIXGJUUVWXWo_OvL1X@zbrcHk45aT>DF` zUsz&kR?^S~M3Zij4#!oJq?@Xj;shP6j@k&EfE3YRt2=j-0+1a5kY)txlt5=AkQv0u zK2B6D;VA0&#I2O3bLPfvya!HqVrc^l-p$eZZlmE%f8SQuFoT zO8Q%1A+uOToxy+^^ecnwOsDtK;h?`1RV}>7p_1|$PbljNO?kxJ%1gnu=Ib@h^xt3? zhiaFxoMp@Q)b*KB#X@?eZcRO^2tBf+!qR)&Ovvs!#*gunrCn28_YG(9Xu&@ZdQ4~| zR`4CZWONU03MCun_XLY79aPRg>1C(jT8wrx*A=gZyuLq}{sBBEEEjJ%J!6N)MkYG0 zIfIj2kU-~c6`rF^q$Qbr05>{EBZa-ScY*#x0O)?iScCf)wR!)%y`o|RYud&1Ot`0~ zj7{p+KE|TyW>q7`YhdIQfu2Ve12lt?IhE0L+m7~)&?QI*=-&i@4nVeog{3#0PLM9T zSzJ|N91JsUYp#vD!3w~A@qO`KQ-R?WjKxyg*bu-Sw5P#;#6jY*{f)8QD{GX8&Ks%> zN%yl9SLfBm3k5S*CJWl*YK*XlBPD6^g9>|C741-(95A*g`@j)ncQ2E4Ze21h4B$z#UzT2W3=izt1^PTU{ z@0{VlbK1MXY%7`WjjW}R^^>|-jo*%9% zvb_2H&CcsNw4X&*@~Oc)v^CL{JS0n4PVN3K^xCEEFF}gVM<{ zDW_zF?Eb_+tWd-?H0*fYYIQ1=$z_yOs)_`RH)~wf0=og$0(i&+V{ovVimF8r#Z*=( zN|?7`J1o`#)B`-`)c`sVz#0m&g@Ug#&SKau_LHp1gR04dX%lji8B_0ej;eVb)>7GJ zNl9fT0p~KgV<{{s60a|()Dmey;e>p?O1J=tn}ZFFR3*a}|2fP2VCE)FC9{93dCnZd zPIAWF*R?VQ2PyIeB{P~Cr#q^I4^rnc=T+guZ&-ZDPy7})CEIO@zpyfrX*fSx$YiB! zx-vzLoetPl$VI69+<^gq*b2Bj@!A>r=WKtxPS{5Vr!v$j>$(au6NI8`pvRo1~rVHWAMfqqF!( zjtl7Fdx2}G0lwfy(2*yo6)krMcz04zP3f$lC{hk@1l1;h&7jxi3aUZCSSYHdqf#~_ zrDdKiABdJe29`Y(WR?P42J9fOR|kyUpgT@^cs5*dUvgh=m^FufvDZzGRal+dxnQfA zxA{so->j`!lf5OIch1(dXaKUP#zVNZ|kL`9Bq}1|03DRzWKFDUK01RMg*HP7{ zGs?Jqb&05IH#uXE(&*E6&c-gI=rZ}(>ovdyUMAmrH?DgHqBT?a4GQ>OQ~!s3A0%$n zPWh)2QyZtzbn~=h`oPudQd2+aZhRLl6SZ*GpQ&#UnhdeN=;}6bllD)85S^ zl^xF0#|j6<>r_@am65STN=+R|B&;u0rJ54?oSe@}*h)rxdx#%}U6x(}C-T{`(VSYl zFP|Pr&`X(>6p6%9%%z&)?J9zNLMxkU7>37%1LRL%ol&c8oLK$AGrcsUR%W~dqzwT7 zl^Ab<=}ro&2}|;Ep&+U3s3hc6qabVayI{>YRuJLAhF26{yL@}W{r{Rg4by&reE?7* zd$~pfFc$!7$tV6c`l&eu+WO zj*FHBSL4NE$;HpR+DfjrIam9lp6bbo;8yb;H-DdLv)7ULf`O_Q5Me{>SWRdT1zBxpBKFPNc5Bib(9%;}Tv70nSBj!f|$UwAyht{yKP|xw&)h{HQ7sPuI z@f`psfaZvfwI)1&+}pHv2W{w8|wwdqGqiFaKBISAzje*Wcz}(^YAtV9wh} zXG;*p$xusx%(S?pn&lxd*KD*$ZSEP(i(^@7JN}3+Kv+3SP;0&WZ#Jn%;mO{4N37(C KJz!v_`TYypf$$vw diff --git a/pydeepity/__pycache__/SequentialPCN.cpython-314.pyc b/pydeepity/__pycache__/SequentialPCN.cpython-314.pyc deleted file mode 100644 index 76ca75673edaeb52cae3275398fa1f484c904038..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 13800 zcmd5jZEPIZac}SYjz=DOJd(%n{?YxhjH*aR%ym@ooWu+bh>A$yuEp zNX{@EGe{0ID$`8P5Y94aIc!y}v}_%;sdj)_2OWnksuN%gXB%`KcB}4YQbk$_XRjfg z<1{l8FbCgw?zHM_lCbj=2HwMrW{FKh?s`^I^! zd4f^xTm!AM<>~-)qs~EV9Jv~&UZXlG+?9j74Y-TKJvn%Z0e4flHwP~@;2sJu!#B|ZOK_OnQk;42GR*5jcy3~LMD74CemeH2g2&1#C!4iz>qwru3UTeV1xjMCi zXH-8=`Wdwn)}l!(Jt}@7Ehg2tkQf;_GV_dQAG-lzM`SS;kE-!=B0G?Z#gh~45m7y# zl3!xa%Yr0{GCQ7ZmDMaJ1VzC~L?Q}y4_(xD^-ZRx#6CHth}-%`_0D(yToE|x`Wx( z-PZ=**ptQLjRO?24oyfD(J~WK2>xB`r=UQo#Ee%XeehjRBl+zOSPMa_VtR%-#z>Y^ zrlgRgR-T+KC4{%^G*+USaHN%7V~&JvGJa0;328MI$)l=WNyFM|u1F-F1bK{vZJJFH z6XP<@OS6tCGn(6=EKn6c)zQITl*^$Y2eCpo% z#;%lwZoD{az3nQ!S+m69Uy?Pe5LGpMVrncV$X@7o zBy5#|yhO8)CsM$2Rz;P88_NJB`yexzPA5(ULj91@nv+nV3{qhC|EskST7+ zI*ep+T{nQVnnJG2jZcZa1zG^HKMfFf8S+?d##^%xx?#!qYi_jOw0raGBVEo!6Zh0%_ngy61DYe zC6c&xAwb{$?@3&WyUVLbCia48>m17nh*{6cjZ?3luo%FGR1+=2+ccW8ypik7u=Ia9MG{S z6&iuRQ*X2cY}694_t{|gvg><3b6jIy(CmsTN($8>VTWc-N~-2Oa$uh<3)7k2<}}!0ZV|k^oY3>W(#<3h!WJ}@)V`~9f5~! zJ)htsU4;m=MW6`s4`i(?!X|s0Ss{NRkb(L(`-vnQfcN z5Nz*m(uA|j>t{CKXb1JU*B7Py`l30aJyw+RS|g)aX9f>m5Tj{TR9Jx>PbU(15psA@NA zDEt2@^t?zua(UNWK``o9Ty+_=FXIiaHEdgI*tXoTebu{tb~xiLU-LFDc^hweoA20(zbWe??$R|^{gSJG#nnhh zZ(MRU!brKXo0eQnD=zjUU;U!1{$9m}9E_5&{!Pe!Z;H!Ck0QUd^6NvtiA~>`>2l#p z>H5kpqmq#iVfGYecr>Dl>cM*w-4h;%_Ebl??Ok^WWLGL*ufI}%m3#BV)f20=+p-9I z2TUV)yj%IB`giN!=l<=)`zKcS9jDE8lIANH8;l6=-gW=)q?$HUK&Eh-%BGp*si52e z7gSIys317_-cJQJqdK|2+5v+d)y^rjhG-hWy|5^?z&Koi!(s%@vZ7>S3=3(QIl!fp z3Oj)^1}fv6*cXc{Q5o#QBz7>oMGqUF);=;pdDDyDMGJ21fPJT z(=a^bXCT9UN>EiXDL;##5zL;$4EaK{3v@qoj){r5IIaLYZAd?5CmvcEXZa38)SaL^ z&~;L5QW;>%KT+uRnL_@0?UmZU0MTs!*@jGm2(wFGcEuaIN1Q)(k#hel7ruI7&Njz= z%e5BlTnctB`@)N^@V%0YL`Jq-zX93rO>%#0nWYl1n;bR{oey>noey^Ed~k4u&IdWC zgf)jKrJ|FHu8ckCHv!jN1RM~am>3nN>CsGcDwFB)@q|cqfQ-gh*!h`?fLbh?Z7h{a zJXj~7B2m(i8FmX8@;-aurzK^t^k3?KReo*e)tS|jP}UAW@KShvgE%f5`>^@hANW~6u@DTvvNk6 z(bmy}qZ2!g)+o*(T+GbN9x7eNW<8ZosPV0orYzl_(21Eyr6d-;%&fpB(77qcqm!)Z zab`gpC2&midJc#QaRT;Rg#~xmR_Y*QMQRPecFL+L_Uy)v*BL(B>ygoD3cK<|YXAVU zfEg9CF)T$fi$SI!?xrV0-(9WX0M{xCw^abqWkI(IcrY4>yMAux{Jw?s&8`&}pYc`B zmfR~I=!XNsk@a36+5c1V_-x_IX91W5rhMa6x8&~;D?8@`5koOU4<8k}9R)^h!5D66 z=uKr~AvHH}gzj6`N|Na*y@-y(s1ThLb-$J( z=ayTm(oH?R7W-0`%#&ZwX7a1q!Jlm$JTzM(u7K-NO-ms7ur(1`gzUj0eH>aTKZJ~m z^g#=V9>M%xp{8}4NHv9{Sie^CYRQ%TuOGg0c(uGe>!xrIihBtydQsj>X|aq1YO+3B zEGN~q*$P_plbZS~4_~dk8l4Y*Ywv1xceawk1E>_LXt5fLHMAHcY+JS#i`hEV5cOEp zIX$-{!?e)k2Tu2{^KjtER}rSQAbDY@w?M0+B$aNs$u~QHwtzR3xAM%$CfsBwaZYpr_fQTLp^4%Q>ct+I@8IOKx^+V)HaY#ps{z%xd_OEk<<=+BFb902VADen{$=eytz6z=6Yx=%%X*F;hP_YPR7?k zJMqnYOa997EVOatuLR!$I>h`CJPT{v$q;zh4W|eCh>KDoUr3?6d6jAS3jXaVE zFp508Gq;vQf1CRrPNgXE4I^mo7bWvkCOyyyZp@=Rns`MCPlqE9 zT))lMhO6O%Cijit5w0j9|C*~OwdK(%pF6=q0Pbx2&3KO?BZEaBgV!_p61ADE)aJ0o zlW9@Pqta%5^X6JC-F!<2j5Rc&ug=i-%~dMmGduworH!@a>-g3JZm0uDeWCpIz`kH6 zsU7qmo}!BqUk!mTb$l&XUrHd5q?K!gXA|F!&+QJb+1weLK?e@~p@Hew%4aL=ip?Z#l`18FoUo*u zl};-=O)6v>@z|V;MhrIkE*gLOzlg=Cz4`p zKgN1MoC*3=W7A1tDh^)I#5AjjVodQo^Zf9!!I9yE>|{I^6O$~xPF7eYj>kSBDW=j2 zwB5Ru6%20+AVZzBY-$|8UUPAwm^Pt$Xmfighrx`;?{`vc^{@2kJ?K}$ ztR%!`1<}S)r#&nr+7=G0;C79{TvUaP!++|05?n$tQG#$A2;!l~RB-VsivC4#P6L^s z15l5S5)~wjZ;0swL!b^2x7YYIoh=*J*T8s5!WkC>KYDe-(lI)>v3uw`jRTTukM5G% zD#TtCqKE=**-sm&qKq!BgboA0c;dV;t#j)?$_CenyE zth|VHh^t#*O6YF~pqk1Jp$an40uJop<5m;XJ-~grb%=#~Ju{XbmPU}6a_oS@4)|Zk zTe}eO_(gJz90FI;d0=nZ4^a;c$RNX8Ah&F{UmzFlurol!29-8&!#->UX|QR`u*P(1 zO!$a?{b$uIz1tx8D0V@!C^HJ!ASCB6cR!Afz6+B4gciQ%Tpx)8|MxzgNC8(XPxR)R z{sD}ju#kNM7%nyszw^bnpSnIU-!eZi|L_8{5LyT@Or^~s(Y+wR`&<1-GHg=O9Y?VUpG zoB)np88!zpqx<^UnKG<4`hal3C_BfGr15sySW}3E6qBbv{^r;9^<;C)Ym%kcSYo}Q zNB0ePYR&YCV_>lo)`Hbp{WAQ2RU_y`-F=l`&5yXj8 zI^P;V7sl`)mr$3sW`|oeaASjE4E}8DP|@5V|2oFHBODDS%?|MuDTQ8zpz8)gj1pvVygjk71odzgi1-6g&D} z#O?xSXbQ?NVV1yb3bQfHqL?Ky`!Z%2=cqY#lFOPqX;NP+p?F7N#{;jD%k)jk7qAn$ z>S03YX;L%~l|V`*Gc*z@rr9Z6bL-=vYVt(WGF&S_oR6$|^X{R64yflu_xI)mFCzh zl{d%!sRrI>9ZOHfeqLJj)#I7kmbKchrP{8ACvH~1Cok4^E!T3hPj7(xZ^SpiKdWt; zJNJ6yW&2MmJAd5XJ?G8@o93ohgW_c<*pVYHg9=_eZ z^{w51z5Dvn8(oV#5B@8?eq8B3J~w>3y=$TWR{O4bCezxsVE?PfGRIFZo{6j;KRZ8s zz2f={-)X$ra#Q$W`+JYB?mPBD+w*W|+_5|BBJJTfhprA?cdxcRHh1uLL(}Vn-x!=1 z-<-NSb^X+>hW&Rc0hKH`B5QT*=11R*T#YQZKe7_o1t`z?|LpKupmQnExlr>~^R?#X z@M9~1ek4xIw>V{ux;H+LOTIdpaxvq z)an!OS_*Wn1dti3TILU}2D&Z}XIi%EUt4Y2b@}O!0u8S{`|7iqUk1albe6jz9#kv#Fxq^kx75+QZ(0R*UHz}*m4qJ%3 z90hi<8e$4KdOz~(^`jQDj}aM=j#rNV7d5SZ;P7#N#}7gSOziT}N1wr@>HMWH&K+L%wJ+W`;PdlSRQ@iE zErajnb5QPMZ2FIwp-+4B82jIbJP_m^li!6Q3rX@X04XL3Hl=9%vxx>hE$HNCCbkYV z{Kv(XVC;&#UsFWIL;^tO4=nUoxQjw&*@%08@<~1J9i7qin%W5k z<*ShW`j5#)a>rt^yY4#R_Yl9tA4S}CLDc#$F%bT46Y%vqGWT5vgydqW6-#wks>4zn zmfG$*Ab^fW%H0h?d$`1uVe|`n*>(4bEq}<4yEr)g0_6|=4&4hh zH-5aL8_N!``QRj@|F*;q#<4W5M^kudx!0&Z2ouum=<85q2bOCN6a;j@&qs8x2>p!C z1jv?lQ#3YD zXOv}R)SP^fF?F&O^E5_rAj^321r{4YHJ}2C4u%TGD9K<3O|4sElMnKVZ8l~VWRf)0 zFUrp^(RWErEyygX)Gw$^fwCq$N=j^QV_(F?cx$pImy3ib&=D^fel###5VcqtwSsX% z;*63hxhtbKw{XcbF|Uleym=YVbQUE!pt1&r8={iaqo+i7xZdCqyw0P1kwIYO<ss6F{0W9non=4p(=KvprM+T^v&#jNT;B_fkIGUxMa09g_p3>Az~lEDm`I-B)b zG?}!VoULL)i&Kk=W4scRvtt5^^0QKtONwJsQd0{uODbauDq*b6g>1W+7=KOH;dBuf z0lMo2!;c1rOQIGl7$+poD4CLbQPg5{2d6v}^NOe|n^$p9XHi!Ms%l`kAu2gNdSdh^ z21ZVe4%Ztzg4cPJFY+iaaJtH){*f6dI5~wsZ}LTf98C3^Q1vPoc~lkzUFFdLsn?tw zFWAqxa`JmYH#Se84ONPh3)#dc-x1|v4FGaKPfir-7Fx+%#0TWuVlOC4P036yDe{{9 zPe_B!AEaIms9u5_A|4FVvDshPj!`rS$SiUL5g@^$5D*bMd6CEig(43S+Z#v}vjPbP zAZREO28p5C&zTMq^8pf*M3+f zb!S%8Qa~yc1|k(M+5i?pr2#CZA0`pBwow$Z+5(P#YO6pIv9=^)Y8P?aAE79)>bhm~ z)t+-_W|x$>rjs~L9~b1AGw0s9k8{uKp1ZrNtAYf|x3o9TinWCNBfdDrYa_)JAeYH$ zPUMb|!<@mjA?`ZjI_x&wfD_Su#Bxx)hL6?wN;Q7_8$YA-C3?W7^Nb!W(L**p!02Io6RQB$7(qs>HvRhldY$?&V+Vlu(-&~@%*z_t!Z!OWc z+4LH5yHOkGjA)$f8OR0&JkjyB0dY2(67gZQ4 z1+(uBCQwW89}CJoYEiKhzF#lXO)DZ_azl2s~Y*fY~I1jo3vIYdb4>mEJ?udj3FC}B$*u*8m?^=fuZ#_ z0RM5BtP+>^AoqyDoBaj$TpY08i_nB5aH38@_P{@18~_AabkeCIgI$u$5Nx|{NY);@O^>c<(=-Hqn*p0^z%d+r z=g!Nt2A;GDE88kA*6NonbPKeEg?_?UUG*>XTNau><~vvR9k}AT&u^i1P`OSBwgUkN zD*gb#7YhLs1vi5pfJ`*Ho8dFNh~5pUz745<)(_8MfWaVxA%HG1%wQFR)d1aMM6BtH zfC$!7bUkJmw4kJUCVx($pn_7Wgw2_uHL7U_O1f$Cj&hp3ZcsI;;8!*_XL_@85(w-u zXSy_Mx>RarCovbTMmpOFOX98V120Kv z6aa33{>K2AEbj7lJ|X}eH5UAc@rVTw3zEj|*Av%H6+-wHCiSh?hpzor zp$gxsNnOkJ{%bvj2)@C#q$*n2idZvZEr_*}nufwQ z#I}>_+Cm#(1%U*sAGHH!$qlZ`DYzD<85cnMQEnzj;=X=&HgeK2FQ5%gBp#_Ds5@93 zJ;~O@Yfh8BuYjR>8VyaHjFoi8YKd?f7~M)vf`(Z|bUELrxQ)MrtncM0^zYs~ z2mL?Zhr8UzN!&YD#=#k#feo4*oQzL)OkSM)-sE3S{=MjlS9d#2;3;~)Zm+)$bROpa z2J{b0!}*K8wR(`Pjn|H`0BOzVfC3eT=dl7*6q(0LzZ)z`nUh=x5nnSYC^tGk@%)CaP?*hu*V41#$fxt*7E8*GKGOEH-GZD>;;ZFsVWygz zPz^2LJ1?&JVk5`QdY@8+83-iO!r)6U2}Wi{0e?${3>A*Od}Lx`WLN-~T%z}yTrw7* z9WZg)i2#>JcOuw@0J%YobbSu=dCKRnWMGr>>1nmfRAlQ7m6m$?K!Ca9f z(;40LrzOS~(+d+(k`zt6j&^J3`XUH|?VO&)F4n|V>s)qeHO%z=H8cg&SJ*+Cb}Xju zHhld`@RMl6GD^os4SPR|?!DswX(;*!^6UFoB6SZUZOf6izm4qrq)}L@KU;LWnxf#c zYoi4pd1l~2-+^WL3wK{Rdh_r~Rs90@`pHkCt#`Mb|0qiDhUq_99&qbQxamfIC0xI_ z<-?JcNbL>phwcv(e-kM9Nuc?2;t2!`VG`c5(Ef3#Wu<=0LiF0%P9qjp}KSjYtP5e26D$An(=f8C2d5OI>WTrOHN={D5ODg3LBR?ciZw@& zX$++Sq~+dG;3QM8D)>?=O+V(g)aVef!IE_3OeyRdh8MH5g7I6|5!(i|cCZbf^CVD> z(bu*G^*-PAq?F)JFeyRBtrT^Mu0G)UlAFngB^0y|+HWorxEQ@2022w$+x&K!%x4xy-O~OlgwP;0-ezH2;k=S3Ouww zW;4c+3TDhA*Q=5H{MPbR9as2sFpjtwk& zj9x~UIJz62n+xG#XjQgLE^ikevbt`L;F+syG0EPn8RDio#hxKwL=NDQbj+z)^Qt-yc6%(QfvCyNm@5+QGYc>rfIMz5(Ts27z1?Z5cIR$01mLz zj)>?4R#e)SC%R)CKn=f4RvNZ`_tiIE1;-MueGu+g4tLx*bU(bS=q8QrU*R^fAcSob zJr3h;zDX}ZtF@a1&cg-H;~Th1%z41$*Re@-9623DmX$Zrk^cy()bsO|e{47>M zZ=URpEq|jO>Rb+Y-VpDHyDIDu598p#n;x4tI@;i|cC2*#fL`pf_7EYupw)&4qc8O} z9@9M{9@D*jkRM(0T0sW)iphNw(h-J|)tN_G@|o^*)-e5}!-JH{dDE57ntpbm?cZaa zalf3lqnZ|v0j zr2gvMa-{u6`&+S_u|L|qc>eADQvQdR-W&NSa_BC9=t-mDhYy)ZejkuZF~u(>R49@j zk!65>QL*lGgD`rmh=wta@eC~%uv3NyokEa6kVJqn!RBTRtwnJ^09*R{QGoid*S*|nU`Sb#%0h&rmt{zYw~A4e-5X;A3ZNrESAToq z{G0jf`9Hk02y(OMF2CnVxxw@zlbiPd`IX7dS7FLLEW_v(F{9`eF@xw6c`?w(!`URb zMy601O`np@BvLw4N;8Q0QB~F?NazKVN4-5#`QP_@;CL~oo3qe$Ga9Gil8Fnm#GYt(`n z*o#25tZW{!c?6dL=s0&LIbr3l+;9PCHdipYP{;NMlj?9fr5~fMd0N7kuXRxV)pbkI+neP z%Mzqs@vcueswFFq+{|7#*Gnno?j z0NC6H%wp3l0%ily(m4g~0kZ)I0r^V8AWESpY5-t20qdh2if)rl=K{83(?NITxUBz^ z>_^V)F95LB+c`fi)xF)a)bg(QqZ99*xEDK!+B?MEVL|K=w|so!!SR#J@OMgjaB61x z)Xcr(skLXPU+peBn%iWDpBt;~2D|*buuikLg4R~(j4{YgbK5H0yWycI#<6>qSZH)~ zlw$PFz(YU?eI3Be@BPc4-B|srcb=R1-gPXu4$Slnwf^%z{ATk3(?b<47jw^Cs{fN0 zKAV5-0J<6|g##0@dc3|*%d_~SgPcGqg}bOr`da{g_HA;RJaYNG{AxA+ zLZS!>po82h50}6HhBwr!A_vzAMZ8m34MQ@y2!hzW>VxzsVx6l#Nb0f7-YO3%BbGi| zCsc7J6vz@2wvw=zR07z!09TF`bTC4F~!Xl9Qn+QEt6&WWG1ul$TLlo znXJX5%*-Gpw>g043?pMQP(}+R>c=Mo7R}`|VpUN9oBKhGK}-tCWQ~hF8Y?8P@)$!* zJ|WD{!*YvJ&QF#uUaY4k;JIlTU`72)+;$d_ed>(n0YH;mI9r7adX_ z@rq8?RWnwaFFS$trm)6sRqYv^Hzed{R9=+Oye^@CQ9}Q!gy9Ef1~r|@y=s=s*CjP4 zpHow2{mj8&WHRLb*7L!xeRhU8Iz3Mhh^I|4nP*)aXkkk6W!5}Jm zT}1t&i24-~O^8l@4Qu9WipGCqFTKQSKdb26d8Ais8$xM7&a49`L8L*Tu zpAoC3D%hG2VhmzZFpEr}7HM4M(O4mQmB$!lk;&%8!u&iezZf-sP2MK2$oOmW4S6$F zC7|vX3_lteF3J0Bbl>2-Lui5G0_O#$D;QTOtq@rexPHeXf{W?`(D z{9Gl4c}LBa$$qMnH9p8OL@=f>UUx`28FnK0LQwDl;R8tr#VdZKKznT%}WmR)vNXaHLPX3~%#R6t1 zs;jVoK8@SuZUy-<8>nd D-F&gY diff --git a/pydeepity/__pycache__/SimplePCN.cpython-314.pyc b/pydeepity/__pycache__/SimplePCN.cpython-314.pyc deleted file mode 100644 index d0a150756d2078e3bafc64149ca6818ee8ac5864..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 13758 zcmd5jYiu0Hd9(MvcbkK~a&lFO%f6kmGFqGem6C6gvC(<^Bu>vdY*Ey=TwcbnZk zNo0~j4N`L`D%rE)v8B3Y0WS@vu=4CruF;{c{SPu<4)n z`(}6Vjud6((EuIdW@l&T@y$2idxqOfiai9<;KyHkX|S1)f5#Uy*^R>5EEKMi3k=5$ zk^x3#8ps90Sq3etl|F3)cGW@atb_Ier|P6-+n{T}t-5L1KIj=JQi}$>s&}AREgmRQ zO9o2S(gC098z@uDpbf)02K@u&YIy^xAdQ4`RuRs1o*8L3fyqC2TlKP9!C9I~*!?jB z?_ox>#HNnH!}YwR##2cl0cB@@DygIrV%VvfLe=aq4tUom=B}7#;>dH>UC&eQPK~lt+QB{sroH!}Olkw!ZUU`$z{E@M^ z8o3--CnAzDZ#2IGLTtv%ngf6rh`?MW=ZVU2M73P-tJX@A0%WT;u8L=NF`T7}BwRd! zk`+q{UxHEXT#$aV<-XZ--yHOd1K$ym37_huHLhG+x87ED(Rxp=zR0L|(|RwyamA4P zR1d8wp*5wnriiPibv{~GhIIg!m)7}dT{&%6Y_to|x(cIR30J9>@{H=^Ngtz@!A90- z?&ER9_elS-Y2I^)JuZGGEhg2tkYGn-F&2-i@k=7xpNavd*<+%5IVHc$j;7-Yl^v5) zldL3*(YOM8-=(N>T$02XYYsRpOo_76?TI3-;13gS0@BjjZpa>oE#S!o%lf3yG1gIO%z zewx;-K@$>1aLk05fuC#b926*am@ybFMDSftBZcjax<-Q7#Pl3n2C^xE3WRp;Fl3)bHk<{ChqEI2a5kFTA-8Z2T6S`-9w!Pg z*$)G0Me$^0Oca24L`Cza(rW(Al8`k^T-K~YRMqT>$W%@|a6#KJoEP1e0 zge5O2Da#gPsf73{vZYYU`iQ6Gei@W>TsdKAGof;!MD#cm5LbEPIqn2@U|rLzIwogA}by5!FaS6l7GW5gDW#P&wT+1RcnqigKI)SY0RW{rI6a zzcCMtKo?yPt$^bzs$W0*S7&FAWV~go-sVMb^W5RP-fe4EQrY+*uOkj{sx|EVu$|#7 zJq(C3tG*}tcKjAQf%~OkYq}GRz2-iFY&Jk2c~GDMO$Y}3Tx%wg6tEJbv(j#(sV-Ly z#Of^IjDjf>Plz`yIxd@`DHRWG25~$f*VPnrT4P?+?20N%3ROa3hh|Mms^&cQ^dVUmrZh`Z(wtNw?b=Dz z({Cx6P`oNfAOp;;lVW@5K4#bZ-x#_+bkp*-=cZ>Rv@45s?^?fC^ls66hky9&d(W=y zf1cJYo<6HTFN(CEj&NH6ey~g#=m2a1m0{*q6g0}S5Xd;(o=vGxit(VC3K2d6+H7;$ zVC*oG?OaYf@>WwGg4_g+CfQFtHR9gfpXAJ`PUJ!(Nj8A4$b8E`^Ux20B0~ua?F#yt zE%WUjSVO7VCjTsGjf7y8k=cJ?e*gG?Acb985!fvtxfQfWwIVQbCnYf2q_k?j=wIx> zG2^0E8Wj@JbOLQxz22LaV)?fcn12D(~m<>Xvc~vlm6A=-7 zyYVTFi73b*sKHrXPiP)h20slZ6eTT?7j^ksN`ha2UZ5vj-c?r+tiWYgO|Cp!w(P2f zw;Ap$u2olP(G{9~VcFI2QGMf#_g-1$we(+yW)5Y%!PQXjVyJg1v~$I~b7nZ>^{smA z7QJmVjaR?us$X`oAC=ZFxN0917)Uu(V0;4#e>8z% zqxS0WEp&RUKp{3=6VmO$ozdllU49nJLzoR?_8ewtzdl+I9D-H~>J2IzT=tIp1hVVp zZ`59|y}`Y8=Ej+o>fS8Yy<1Kn@0I_s_PyEw{IuTvbv1M9=Iv!GsqldDBcBC6_fG_tU8Gqj~Q;OW*4X$aE^+JxHzT&H8kS4 zJT;twSHo8`EI7>xzB{_ag%*fWMYlc*T^`DlZ&Y8e{u|(=)}QBD5IBxq^s>v|rU$t1 z6BqIMU%m3>E3>v)?whXFVEbaQeW^6O;0iy;bci3ybhn}KM`OC*o82h0>RQ^yp>t`c z&ZV7mLfOH&ICqZ=_R&+okSH%ThbX0@6N)Z3J?IxhlqCXgrY9ywg(+%wYffb%JvNpQ zDf`NZiLmpbJWQ3mW*beV5|3tQlmSW_GRn@ScCdLN`)N_ht9`HZy(Ygt{o3?OQB&5A zUmV0+y0O}c)h<#N$hxuQA-?i#5th89q9$96r4lTaV#!DRmDw^VW&Olcalaf&`YxmW zWhTQ-^{zER4LGSF4fJ%1In{jX$f@z&Mr-6mz$Y{PSr3;^Dy+bcr4tEuGM!N4T@|JL&f!{eo~zdmP5@-@*t7V!4W0o!bSTk2F_QtMckTb!b7fFG3S%(UaZy!R zbwUIx=!yzaD1ecY04qenYGb7=Y^8Wfl&4tgzaC(_b|3`^qX|**U@xSY_Lw*(04aq# z*vSJT4~F@uawM9JZ^Ay(A>!XrIN{OK|}j z$|6K%!QN970C*P~Rv5EP!TOZT5Vq{hul!GXyL*)I#|2=5KiRXlYj1BWt4N|4 zOY02c8H+=_p4AOE#NHHQBbtD4DAGSJUzO7$J1Riz55VhYk3D~aof6eNSRRP4vT@i5 zacm3#L3C3Px8zJA8coYWbP5-yP6+BYSh`0MB>|%ifJYIKt_ZLe*vu$|m_-aW~-JpULe3ID|994k7Nq5VmPQ(>J zse%z~b^^9zJO#~Rl2m$pq8o!*`+%smO0=>k(-BH)ksJ(WkR%ChP#M4? zFco@sx*Pm}g0*ulRI~6ybxYOnt_>dHhB4gG#>IJPUIe+D^K!*KUXUXt52)va0M?Tt z=R~^Z2Hl{a9B4(!^rT+IK(S5+Iv?i+zgD4pWBOO@OO;cBCiiTm$u&EiRv2EpW=q5s z2sf)~IFg~-U5*1<9xbO|f>z28Afw9thy|1|!Tdq>PQPuT-)TK6_}7bGE4qI8je+X} zE56pOo7Q_!@fXpe7d3w|EtZf#RkoBCeWbEFTSkk1QdN6>&yDgM(YfF^pIoWjo-L>K z0W=RPXt5HDRkRo+Y)iHpi`g2q6>71lYm}T>4a-6g87SSm#)G$BXe^i}zvKmTy-){1 z2TW3N{xD>#;7udM!g(vtjBG)T7dDvf5Mg~}n^TuAbeLv~RKk~hngbz~a?Dn70z7%- zUA|ll!gLKJAMYL6B2vIvd7GDTHr@`ooiByl!TTV0?sS`d!4WFc@j=KrJjS63P1Svd z*>w(mLcP{w_Bw~A2~8K;xgu!o-G|Z!(m6cXpvlC4LI2`he>5B(-3nvT4{KFoj>UVq z(tT($LXyf5KPUYs%hT2#lC~TLnz#&mFZrc%-rqw;EkoeAp=%}uu z7b1KB+)y~fkb-=G4;IR-k!Ew8{L{p80~vzRaAl{y&R2C7&|{}P-&(2$UTDp~ns`#(@3IuM z>dk*OOI)S7R*5ZGt24jWqg}9yM!u16Kpk()4$_Hl;2R5fj%T5bqi`qqMvx)qMDX1B zwUrD(rv*mZAJ%)@7J~-cnLXO&sSE_T+qVKvX6<}$rz6ERD%-wt=9u5zU>$@}b zJ#&{f(HRCWH>LdE@-=+(25P8*n%a&0*9m&TOj0Yzzvs=j{FASQi%~UvHCJ0qxDemW z)xoo#Z^h?M2iIU~h6a#Vh=ePtndcgCX2ETJ-Tm{`dycGk^8odXEPO_k~5HrEn3fqsDY`PAC$1RMDo_^E8 z9*hW=#>3t0b3zj8b5s$^d1JuoKR*9pVDI~>IT7kCURZ>qH<_2J>bH}E}&>8fkP}evvle`=qzDY65_IgVABYNhlNDjLf8t(u2W zi%?1sQGlCSG++fG2?%shN28OX2|57w=qOP^!uu>S!|A25t!`s#x>^=oUUZU;mn3-m zc)h1vM=Twqy_Y>m+l&E_Y9};>Go>ye_L2}q5NOLj+CUZMNs3%r*U1Wr%fggG2f;Z2 zPF0@>;G95I(lEj(&K^yqV*n=Xyoh*+V@SY=wLkE;ij3In8Hv&ZXaUd zZqKx(lcjfsOcurys9o?|!z&6+K;u`*33Ak@OLN%|w_X_FL58F4HF&C=ZqSIc5oG>bB=d-~=fm-}%D;)quy`3bFcLxe99 z2ios`Jdpy5R-Wk2HM<6&DJ*0k19S@w!|#6Xoo8h4394of4AgPZ;42mNM7bp8OZ~Y%D%`{&Q!0@WD=3uzJ*aQ*}My ze+&?xjAG;rj+HlkZ-uxIGNsa^2Us@@igX98bb!?uk7h%7;MymgPERGo={gLN#xQT~S+`789H*<)aJt0V8Dr2*v&pI!c9y4_>qRx9Ef2EUJ5uehgk%(P_0KVDf=qMjXp%K| z(!`inL_{HNFn57#sY6~~I^p&Ua|WL{E5bZK^4uetScP>gsWA_^=B zH?L&PtM}6BN@GSf=V^Uv&H18H@aR41jaNIJO!Ly)mByi;#zbil5)PX5(|9M16^k*r zv3F@aqNE~Yf~=sSWn8hN4^%u`C;iyMg_o*PiWOe#H0x;C=`kBUz}Qd9YrYzuab`;V zuclr}%~*f#53KsbANs=?Z}GJ@$Xa z-FMGhF=v@?yHmOB9k}PMU-Gi_)3zmV`#o>nlDDDYwPDHInCqAc)P7P?6!6Tr?$?vk zOpIB6DznJv$6|HI{1bO7-@snP>CWeYfNJ>YrEF&t7_??wb84vEq3%RbsSs_Y`gZ%?5Q~+69}#bS{4H>%Yn8v7uos5Vz6u0J$E3( zw#`kgusde$KdEJ}@42_V>+Su2zyH?p+Z_wLkNhh=>0jP{dUp6;YsY-w-PV0`Os2VI z-u`z7GN;ckT!^fkzBo60tL)Z`->$pUct`j_>-&$b96Iq~%SpJB(Y7D1WVD9g8oDub z%e~TaVD`wpQ2iT&Umu(k-#018CKE2?$q=1#p8xe-}v-Mbvv2O!V- zzc#QMXkQGp&sV+OaI;}4d|)}yhsbHXcJy9d)7+lFJvYk$3g=o@oAxd??OkfxzY^R( z>&OH{tHJPMFnl+-9YCq?08ns_ZGe#FK)4VC_nKPXYX4UI{O-5=ZuZ@pTG@VRb^GAr z_Q8dblgmx~YSV?qrV9&^(S=xYsVOz1PR(p=ss8cr@$d5Ar&p^MPM*2VFC9D!S6Azr=N?-L zbCD3tgIMdjr&$iOI@7l8;1wyYs z|Jw7JU=u)+VVmckomUo`cinpIHh<^o+t0mUxiE5O;cR5#`HKtnLe@ztTGolJqVoPJ zhSYRqM;TI9{d(zZrI|o|CRCRR0Y2wN@l#PKlGnJ+gtNw*gF2%z&||m)9Eih34SA{ z?2{+Le!RDiLK?2;dm@oZIL%9787}UBCM_h4FQpOqUlFiu@o*@ag3r!KB$kRsA~Fl( z$U8AZl_q24K}MHfK8x8XW-ntVVfG!!va(pxmr^+y5S;0RYM8!k` zs?7gG&|e{_-;9G)K}N4TKJ}D-vlc8>LQnqv_ysl9Z@vUp;(d$7?pk-izYqKh|A%1R z1(&~ng}2DpLvV|jUL{_4!DZGpywSSufGe9=YFT%{d=ci>j5~C|0N#( zuQ$N~E*v%`*FrXr(VsFmhSY~2r~FzPgd z5obehJ+ANl6Y>X8$H0%?Wy)F^hWRlG{)qU0Olp2iYX5_<>lPPdStpRel_kdVD?ee{ YKCzv*GJ$KBRbSJhuj$tW-zY-<7g%(LDF6Tf diff --git a/pydeepity/__pycache__/__init__.cpython-312.pyc b/pydeepity/__pycache__/__init__.cpython-312.pyc index d0ec1ba71f158a38c1fbd120da81e6657ec71ba3..e13c1b2a2ec4470ca43afc02fb25b79c9d401111 100644 GIT binary patch delta 64 zcmcc1afXBUG%qg~0}vRznUPhzkvEE2Raw6%KfgrZB{j7mv!qhLpfUx@ijU9C%PfhH N*DI*JHF+-c0RWy-7ViK6 delta 81 zcmX@ZahHSlG%qg~0}#y7o|d(IBX1P5nTfMiOlWaxQE`k{Vsds&Kv8~HYH~?&OiF5M dL1sy1OhF}#6(66Omst`YuUAm{YjO|s0RUMG9v1)r diff --git a/pydeepity/__pycache__/__init__.cpython-314.pyc b/pydeepity/__pycache__/__init__.cpython-314.pyc deleted file mode 100644 index 0da68a4483dcb43ec4675e44e72fee662bca765f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1112 zcmZvb&2G~`6oto*9Xm<$=O$@d5RFQOnjj^mONx-7NF+ikQL=`S8;=?n|Ag^POS&>F zKwWjk8?a%^lQgSDmLee`_Eq}=%%rgl6|*?^d}qevv32KRCY?lV{J8phGak0KFADLQ7~0AgpsQw#23g;@rntv1SZnb1$`|rUcU5%PqMngFN@~R=gR9 z_}nWkrKy6tj^3zkEqrBEA;I!O#G!*E%S$0e6;doOM;tm>V)%PSFw4stBt2q~(NXZdEtp#zQO4?>D6EVFzo;?O~%jyj3g0u(#yeN;`J%8%;X z_3f$)H4YJrQ z{^C{pW+%}Y?N{@ZcLR-aVSR#VoQuLFD0G1m-F{}7hYm5G>(J{H8fb?Eu(gAGIzxOg z0^$W(eF!c-;483ao{}JAj2zuIdl>p`3nR$7_5j=ZhXs27}p1*Kz(An#?T^9=^eHOyMD;~#^f?42*NZ|^i#jl)(ll7f73Gg^9K~);9g@G`~E_Z;bOBe&+8qr%iH| zajxR0*^_GW%gM(RUz)1PbM;L1<*Aamog|meYiBiInWoa`yJx$;dL@-p;wOc!vV;Bs DgNXK* diff --git a/pydeepity/__pycache__/_backend.cpython-312.pyc b/pydeepity/__pycache__/_backend.cpython-312.pyc index 824fb488a8eaf9be6675fa74d6540cfb26ed4b47..b1a8b1b3a8567f18c5dc8befc097b0d3a1d5690d 100644 GIT binary patch delta 111 zcmdnayo{OeG%qg~0}u#(pOGcUxRFnUQB_&LC_leM-z7D*AhV=Wzo0S&%8E})OwLZt zOVKN+yfrz3Q9)Dn9*63W8|*@#7?>GVZ}LfgU}BJrV*1R?#4q*#O6Q}C$56nQh I$qO090DpxdlK=n! delta 128 zcmZ3+yq%fvG%qg~0}#aCoRam0VI!XiqnU}bRZM7cYEf~FS7LH@Oh8e7R%&udaZE~T zYC&d6WlTXOj1`{*RFImNqE}G)YqC3|f+p`h7R4Vo*o8hZFf;Pri_@% diff --git a/pydeepity/__pycache__/_backend.cpython-314.pyc b/pydeepity/__pycache__/_backend.cpython-314.pyc deleted file mode 100644 index e582871b7768c2d8fd7a27de8c484f4e8a8a3309..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 443 zcmYLEJxjwt7=G_AX>DUw1P8z96>%tqM$}Caq%Bwnk?d7sa@uIx3zsVr2N6d%aq}Pe zk2H&dlZyoEAi8+5t-izi`8*HrQ<*5_LB;Iy-RZlkujw!fZP@hds=0!BI6xL^!VZ$S z2o_la3N4ZWcM@O$q}C!Ou<$w>uv%IhTSRKxnuk2tAdBFkvi6o<%P7tTk;_=%$>`#; z#(Ta?yIfMA+b)%DMjhS>JfFF=J^;~Lr9zKv=ZtmTYQE78dxHG}1GTeH7(KT`>UBeD z`zg!eLNG`A5#{1v>sSpD?+I!Pr|n4vP%72EvL1sV>a7^NQEW6j0T;3%1Q#)4571BT z#H(hTcbF-7$X3jK!A})CG=~<IbyRrP zN}rF}=?@^N@RXlPFywY|pI{^NVvHv`)(hVWQMa#fO7X5UdAP|I$!%_q-0Ma114Qn4 Au>b%7 diff --git a/pydeepity/__pycache__/utils.cpython-312.pyc b/pydeepity/__pycache__/utils.cpython-312.pyc index 1fb73023d591aa4574d30cbecaa4e57b908531b6..8cdb1049b1e41e2db6f882cdf458ca2f8da4c1be 100644 GIT binary patch delta 135 zcmew*eO;RGG%qg~0}vRznUS@QYa`z}PDus*qWt_4eV5eKg3OXi{esFAC~NXFPBlia z$-G?KSU*WIa0yL5%Vo?oEp{?HcNX(CQPs(n+*6nt#MCAW@K`7?h$(;P1giPq#mK0NDvP)Bpeg diff --git a/pydeepity/__pycache__/utils.cpython-314.pyc b/pydeepity/__pycache__/utils.cpython-314.pyc deleted file mode 100644 index 1a8a732ecb3cd961da349b5fc27326c26383b5b8..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 3932 zcmb_fU2Gf25#Hm^@pvS~ABp;*DCum~vRNx+D!Y~KR84J1Hf6`Me3s8YqRgDgqjbsf zc-=cDCDjxPnj&)ZU7`65ws67Y*av>x_dt& z#7D%^fEq#C`Syu;O-5o#*NrV<; ziGGmqe2G6ao*;D3>5{CLBq8C^sR>w`g@UT;fy;_0%K|)IlFCI{_nzWylAaWPtXb^H zg;HVsbg8IFf+kqJ@T6E2MkQWRg}lXT#Vl4R@}extvnDe~pwHtUZXxu;8-Mx_)GYuG z+;kj!7l1tC#la?zOS~JMxTX`=by84Dxj9vLtD2ywI*pD?6L~2sL7mPEIlhvG zye8-#b*@|}NPZf8`{Ed_{N2itZ^$C0;u+q&r1f)7{2Qcg>Va z5{fxlD8VAHsW<@p^;BxER1{LMqzW&l5CmQqa+;dT3xXnQm6TGksj?~TbrE$7U79x zJf?sGfzY$Ht6jLAn7Nq2K^NI0&=?nLKEyF z0+j3N3IfXl$k^ZmN3kku3YD>MAK-|imR#|2U}6_urlz4yU;Xu96BHF6M@@Jda+_ia z&gn-tmUE%r$@x(4;%L;nhrIST`as5O>t}p6h3>6q*zzJ1*yU38stn$d(QY1My-1rL zw*1e8ZHc|NruDwv^{xMvR`fq;`E6Sq#Ri5g*em`?|4T%cC_Bo|?C$CI0nR_sVcTkq zYzyaPgKWt5T?u@HUYPUk(QCTFINUojh;9N&7G%B538`n9wi(uqm8{$0Y&)_cZk-Q7--#-2(^IXeKf^FkM zY%~DacCM4{Kw~G@g~uV1?Xoe{2hRT-059mpnEN_A0c0n}1^1o^x|7|=FJH4KY#dp3 zHCrAf&Lccv+iVa1vv-LQ^Y!K2^(ZK@_CCI;EmxR;S;_ zWm3*8ld68hhv`xCtp*8)?$*##C}AIwa@ET)r_4>w+R#l9@DQS_P6>`woyqSOu#$kv ztJ%s()paa6RH%{+!@oH@f~+;#5YrSN`kD@jT6`13PiO6#E%`s*%EqihYKo*6Dh=R z6vRnhr}%t6i_c$oLtdU0^sp*uSxMj_uJDX91>#(fN)%qsmx{Vi5n!=wViv6tkh)Gc zu&0w{C69obh@nbG&>+qg5Jw5(HxAqQntWD6&IWXR%5=>U=Q{^k}ZfDLa8>FwOxj3Brr;;Gbl4GfM;f!oPj$%WJPQ21BW!q{V%@4XA}T&UB>*9X=+YbUw7FSK~D7EY~s zSLbUduGF~ewJSGjH>S6pgf~k(b2{DRGa7_Cx2WjynTJ&W78R+r4sB3Jwy5y(sg=Sy z_mFyJi;8VfaYNLe`^Xo2K0xRL}0hdZ^=B)E6Qb(mOFgM}E;+kM=x@4m^ks zto(GHyFXNm4s1ln7wHCjem&7(e-Vu>fA7xp;%L3Af9=ZV!4sQZFD;&{`-6}C`yTlB zt$h2T|AoH=+qYuzj}spy9>r1*VyQJ^BQ~_;sqfprGW7c;9|Z zJnrsYe6{XxyFGqueEG!U_(T6N{?75wq6e0UdW=~KuU!75WA)@3ad&j>^4*bI>^L&_ z4gAMl@sE2y>Rt7$b*vxQ==$LjSr5fm$j#8elE2=QSiQ6szI$b}=g1QMIMjJ3_(8DV z)?1JF)Z=|C^y-mXUuw0o-hR)u9=ONdKUcdtRXd-l#iqACAk6#|xWbVqVbDIX9S4Ev z?TfcA)T!EG+!=Sx? z$BT}A`y2t>evE}$y2RwvC7;!;reLoqj(w6dX@Ndg9I7sZVPV&o1Z zi|kDx2GD;GU>`DQ-UjOX#Zta32|s`r5f4|r>S5F&*b>AaLF7*$_E*5{I4Hug15oz_ PcROuHsc-$KVHo>29Fu`d diff --git a/pydeepity/_backend.py b/pydeepity/_backend.py index 953875a..33f2bdc 100644 --- a/pydeepity/_backend.py +++ b/pydeepity/_backend.py @@ -1,6 +1,6 @@ try: - from . import pydeepity as dy + from . import pydeepity as dy except ImportError as e: - raise ImportError( - "Could not load the compiled Deepity C++ backend.\nEnsure the package was installed correctly or compiled for your architecture." - ) from e + raise ImportError( + "Could not load the compiled Deepity C++ backend.\nEnsure the package was installed correctly or compiled for your architecture." + ) from e diff --git a/src/backend/Backend.cpp b/src/backend/Backend.cpp new file mode 100644 index 0000000..96d6001 --- /dev/null +++ b/src/backend/Backend.cpp @@ -0,0 +1,28 @@ +#include +#include +#include + +#ifdef DEEPITY_ENABLE_CUDA +#include +#endif + +namespace Deep +{ + std::unique_ptr CreateBackend(DeviceType device) + { + if (device == DeviceType::DEVICE_CPU) + { + return std::make_unique(); + } + else if (device == DeviceType::DEVICE_GPU) + { +#ifdef DEEPITY_ENABLE_CUDA + return std::make_unique(); +#else + throw std::runtime_error("Deepity was compiled without CUDA support (-DDEEPITY_ENABLE_CUDA=OFF). Cannot create CUDABackend."); +#endif + } + + throw std::invalid_argument("Unknown DeviceType requested from CreateBackend."); + } +} \ No newline at end of file diff --git a/src/backend/CPUBackend.cpp b/src/backend/CPUBackend.cpp new file mode 100644 index 0000000..768b02c --- /dev/null +++ b/src/backend/CPUBackend.cpp @@ -0,0 +1,159 @@ +#include +#include +#include +#include +#include + +#ifdef DEEPITY_USE_MKL +#include +#else +#include +#endif + +namespace Deep +{ + float *CPUBackend::Allocate(size_t numFloats) noexcept + { +#ifdef _WIN32 + return (float *)_aligned_alloc(numFloats * sizeof(float), 16); +#else + return (float *)aligned_alloc(16, numFloats); +#endif + } + + void CPUBackend::Free(float *ptr) noexcept + { +#ifdef _WIN32 + _aligned_free(ptr); +#else + free(ptr); +#endif + } + + void CPUBackend::Zero(float *ptr, size_t numFloats) noexcept + { + memset(ptr, 0, numFloats * sizeof(float)); + } + + void CPUBackend::Copy(float *dst, const float *src, size_t numFloats) noexcept + { + memcpy(dst, src, numFloats * sizeof(float)); + } + + void CPUBackend::CopyFromHost(float *deviceDst, const float *hostSrc, size_t numFloats) noexcept + { + memcpy(deviceDst, hostSrc, numFloats); + } + + void CPUBackend::CopyToHost(float *hostDst, const float *deviceSrc, size_t numFloats) noexcept + { + memcpy(hostDst, deviceSrc, numFloats); + } + + void CPUBackend::MatMul(bool transA, bool transB, int M, int N, int K, + float alpha, const float *A, int lda, + const float *B, int ldb, + float beta, float *C, int ldc) noexcept + { // Multithreading is the caller's job. + cblas_sgemm(CblasRowMajor, transA ? CblasTrans : CblasNoTrans, transB ? CblasTrans : CblasNoTrans, + M, N, K, alpha, A, lda, B, ldb, beta, C, ldc); + } + + void CPUBackend::Scale(float *buf, size_t n, float alpha) noexcept + { + cblas_sscal(n, alpha, buf, 1); + } + + void CPUBackend::AxpyInto(float *y, const float *x, size_t n, float alpha) noexcept + { + cblas_saxpy(n, alpha, x, 1, y, 1); + } + + void CPUBackend::Activation(ActivationType type, float *buf, size_t n) noexcept + { + To_Fn(type)(buf, n); + } + + void CPUBackend::ActivationInto(ActivationType type, float *dst, const float *src, size_t n) noexcept + { + cblas_scopy(n, src, 1, dst, 1); + To_Fn(type)(dst, n); + } + + void CPUBackend::ActivationDerivative(ActivationType type, float *buf, size_t n, bool activated) noexcept + { + To_dFn(type)(buf, n, activated); + } + + void CPUBackend::ActivationDerivativeInto(ActivationType type, float *dst, const float *src, size_t n) noexcept + { + To_dFn2(type)(dst, src, n); + } + + void CPUBackend::FusedStateUpdate(float *z, const float *feedback, const float *deriv, + const float *e, size_t n, float ir) noexcept + { // TODO: Unknown if this is how it should look? +#pragma omp parallel for schedule(static) if (n > 4 && !omp_in_parallel()) + for (size_t i = 0; i < n; ++i) + { + z[i] += ir * ((feedback[i] * deriv[i]) - e[i]); + } + } + + float CPUBackend::ComputeErrorAndEnergy(float *e, const float *z, const float *mu, size_t n) noexcept + { + float energy = 0.0f; + +#pragma omp parallel for reduction(+ : energy) schedule(static) if (n > 256 && !omp_in_parallel()) + for (size_t i = 0; i < n; ++i) + { + float err = z[i] - mu[i]; + e[i] = err; + energy += err * err; + } + + return 0.5f * energy; + } + + void CPUBackend::AdamStep(float *param, const float *grad, float *m, float *v, + size_t n, int t, float lr, + float beta1, float beta2, float eps) noexcept + { + // Precompute bias correction + float beta1_t = 1.0f - Sleef_powf_u10(beta1, static_cast(t)); + float beta2_t = 1.0f - Sleef_powf_u10(beta2, static_cast(t)); + float step_size = lr * Sleef_sqrtf(beta2_t) / beta1_t; + +#pragma omp parallel for schedule(static) if (n > 256 && !omp_in_parallel()) + for (size_t i = 0; i < n; ++i) + { + float g = grad[i]; + m[i] = beta1 * m[i] + (1.0f - beta1) * g; + v[i] = beta2 * v[i] + (1.0f - beta2) * (g * g); + + param[i] -= step_size * m[i] / (std::sqrt(v[i]) + eps); + } + } + + void CPUBackend::AdamWStep(float *param, const float *grad, float *m, float *v, + size_t n, int t, float lr, float weightDecay, + float beta1, float beta2, float eps) noexcept + { + float beta1_t = 1.0f - Sleef_powf_u10(beta1, static_cast(t)); + float beta2_t = 1.0f - Sleef_powf_u10(beta2, static_cast(t)); + float step_size = lr * Sleef_sqrtf(beta2_t) / beta1_t; + +#pragma omp parallel for schedule(static) if (n > 256 && !omp_in_parallel()) + for (size_t i = 0; i < n; ++i) + { + float g = grad[i]; + m[i] = beta1 * m[i] + (1.0f - beta1) * g; + v[i] = beta2 * v[i] + (1.0f - beta2) * (g * g); + + param[i] -= lr * weightDecay * param[i]; + param[i] -= step_size * m[i] / (Sleef_sqrtf(v[i]) + eps); + } + } + + // ... (remaining methods stubbed out for you to map to cblas) ... +} \ No newline at end of file diff --git a/src/backend/CUDABackend.cu b/src/backend/CUDABackend.cu new file mode 100644 index 0000000..95bf15c --- /dev/null +++ b/src/backend/CUDABackend.cu @@ -0,0 +1,9 @@ +#include + +namespace Deep +{ + CUDABackend::CUDABackend() {} + CUDABackend::~CUDABackend() {} + + // ... (remaining methods left completely blank for your CUDA implementation) ... +} \ No newline at end of file diff --git a/src/backend/Tensor.cpp b/src/backend/Tensor.cpp new file mode 100644 index 0000000..89f35e3 --- /dev/null +++ b/src/backend/Tensor.cpp @@ -0,0 +1 @@ +#include \ No newline at end of file From 3b03e2968349b50713f118f78aa335e9e95cb94d Mon Sep 17 00:00:00 2001 From: Ra4ster Date: Sat, 5 Sep 2026 06:57:51 +0000 Subject: [PATCH 02/11] See last commit. --- src/backend/CUDABackend.cu | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/CUDABackend.cu b/src/backend/CUDABackend.cu index 95bf15c..82803d1 100644 --- a/src/backend/CUDABackend.cu +++ b/src/backend/CUDABackend.cu @@ -5,5 +5,5 @@ namespace Deep CUDABackend::CUDABackend() {} CUDABackend::~CUDABackend() {} - // ... (remaining methods left completely blank for your CUDA implementation) ... + // ... (remaining methods left completely blank for your CUDA implementation) ... } \ No newline at end of file From 11a6f8d68706f1849b7e619717ed8e292becc78d Mon Sep 17 00:00:00 2001 From: Ra4ster Date: Sat, 5 Sep 2026 18:22:34 +0000 Subject: [PATCH 03/11] Implemented CUDABackend.cpp --- CMakeLists.txt | 8 +- bindings/pybinding.cpp | 6 + include/deepity/backend/CUDABackend.h | 48 +-- include/deepity/utils/Activations.h | 474 ++++++++++++++++++++++---- src/backend/CPUBackend.cpp | 2 - src/backend/CUDABackend.cu | 358 ++++++++++++++++++- 6 files changed, 802 insertions(+), 94 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 5dcdd7a..f65b485 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -199,10 +199,6 @@ else() set(DEEPITY_BLAS_RESOLVED FALSE) if(DEEPITY_USE_MKL) - # Modern, recommended integration: Intel's own MKLConfig.cmake, - # present if oneAPI/MKL is properly installed and sourced (e.g. - # via `source /opt/intel/oneapi/setvars.sh`). Exposes a single - # target (MKL::MKL) with both include dirs and link libraries. find_package(MKL CONFIG QUIET) if(MKL_FOUND) message(STATUS "Using Intel MKL (found via MKLConfig.cmake).") @@ -275,6 +271,10 @@ add_library(Deepity src/DirectKPPCNetwork.cpp src/ModelIO.cpp src/StreamAlignedBatcher.cpp + src/backend/Backend.cpp + src/backend/CPUBackend.cpp + src/backend/CUDABackend.cpp + src/backend/Tensor.cpp ) if(DEEPITY_USE_MKL AND DEEPITY_BLAS_RESOLVED) diff --git a/bindings/pybinding.cpp b/bindings/pybinding.cpp index 275a8ba..144950c 100644 --- a/bindings/pybinding.cpp +++ b/bindings/pybinding.cpp @@ -52,6 +52,8 @@ namespace return Deep::tanh; if (act == "sigmoid") return Deep::sigmoid; + if (act == "esigmoid") + return Deep::e_sigmoid; if (act == "relu") return Deep::relu; if (act == "linear") @@ -65,6 +67,8 @@ namespace return Deep::dTanh; if (act == "dsigmoid") return Deep::dSigmoid; + if (act == "d_esigmoid") + return Deep::d_eSigmoid; if (act == "drelu") return Deep::dRelu; if (act == "dLinear") @@ -88,6 +92,8 @@ namespace return Deep::ActivationType::dSIGMOID; if (act == "esigmoid") return Deep::ActivationType::eSIGMOID; + if (act == "d_esigmoid") + return Deep::ActivationType::d_eSIGMOID; if (act == "dlinear") return Deep::ActivationType::dLINEAR; return Deep::ActivationType::LINEAR; diff --git a/include/deepity/backend/CUDABackend.h b/include/deepity/backend/CUDABackend.h index bf06703..b6b53bc 100644 --- a/include/deepity/backend/CUDABackend.h +++ b/include/deepity/backend/CUDABackend.h @@ -1,5 +1,6 @@ #pragma once #include +#include namespace Deep { @@ -9,34 +10,37 @@ namespace Deep CUDABackend(); ~CUDABackend() override; - float* Allocate(size_t numFloats) override; - void Free(float* ptr) noexcept override; - void Zero(float* ptr, size_t numFloats) noexcept override; - void Copy(float* dst, const float* src, size_t numFloats) noexcept override; - void CopyFromHost(float* deviceDst, const float* hostSrc, size_t numFloats) noexcept override; - void CopyToHost(float* hostDst, const float* deviceSrc, size_t numFloats) noexcept override; + float *Allocate(size_t numFloats) override; + void Free(float *ptr) noexcept override; + void Zero(float *ptr, size_t numFloats) noexcept override; + void Copy(float *dst, const float *src, size_t numFloats) noexcept override; + void CopyFromHost(float *deviceDst, const float *hostSrc, size_t numFloats) noexcept override; + void CopyToHost(float *hostDst, const float *deviceSrc, size_t numFloats) noexcept override; void MatMul(bool transA, bool transB, int M, int N, int K, - float alpha, const float* A, int lda, - const float* B, int ldb, - float beta, float* C, int ldc) noexcept override; + float alpha, const float *A, int lda, + const float *B, int ldb, + float beta, float *C, int ldc) noexcept override; - void Scale(float* buf, size_t n, float alpha) noexcept override; - void AxpyInto(float* y, const float* x, size_t n, float alpha) noexcept override; - void Activation(ActivationType type, float* buf, size_t n) noexcept override; - void ActivationInto(ActivationType type, float* dst, const float* src, size_t n) noexcept override; - void ActivationDerivative(ActivationType type, float* buf, size_t n, bool activated) noexcept override; - void ActivationDerivativeInto(ActivationType type, float* dst, const float* src, size_t n) noexcept override; - - void FusedStateUpdate(float* z, const float* feedback, const float* deriv, - const float* e, size_t n, float ir) noexcept override; - float ComputeErrorAndEnergy(float* e, const float* z, const float* mu, size_t n) noexcept override; - - void AdamStep(float* param, const float* grad, float* m, float* v, + void Scale(float *buf, size_t n, float alpha) noexcept override; + void AxpyInto(float *y, const float *x, size_t n, float alpha) noexcept override; + void Activation(ActivationType type, float *buf, size_t n) noexcept override; + void ActivationInto(ActivationType type, float *dst, const float *src, size_t n) noexcept override; + void ActivationDerivative(ActivationType type, float *buf, size_t n, bool activated) noexcept override; + void ActivationDerivativeInto(ActivationType type, float *dst, const float *src, size_t n) noexcept override; + + void FusedStateUpdate(float *z, const float *feedback, const float *deriv, + const float *e, size_t n, float ir) noexcept override; + float ComputeErrorAndEnergy(float *e, const float *z, const float *mu, size_t n) noexcept override; + + void AdamStep(float *param, const float *grad, float *m, float *v, size_t n, int t, float lr, float beta1 = 0.9f, float beta2 = 0.999f, float eps = 1e-8f) noexcept override; - void AdamWStep(float* param, const float* grad, float* m, float* v, + void AdamWStep(float *param, const float *grad, float *m, float *v, size_t n, int t, float lr, float weightDecay, float beta1 = 0.9f, float beta2 = 0.999f, float eps = 1e-8f) noexcept override; + + private: + cublasHandle_t handle; }; } \ No newline at end of file diff --git a/include/deepity/utils/Activations.h b/include/deepity/utils/Activations.h index 0254b90..1cfa2cb 100644 --- a/include/deepity/utils/Activations.h +++ b/include/deepity/utils/Activations.h @@ -53,6 +53,7 @@ namespace Deep SIGMOID, dSIGMOID, eSIGMOID, + d_eSIGMOID, TANH, dTANH, LINEAR, @@ -104,6 +105,8 @@ namespace Deep return dRelu; case ActivationType::dSIGMOID: return dSigmoid; + case ActivationType::d_eSIGMOID: + return d_eSigmoid; case ActivationType::dTANH: return dTanh; case ActivationType::dLINEAR: @@ -124,6 +127,8 @@ namespace Deep return dReluInto; case ActivationType::dSIGMOID: return dSigmoidInto; + case ActivationType::d_eSIGMOID: + return d_eSigmoidInto; case ActivationType::dTANH: return dTanhInto; case ActivationType::dLINEAR: @@ -155,6 +160,8 @@ namespace Deep return ActivationType::dRELU; if (dfn == dSigmoid) return ActivationType::dSIGMOID; + if (dfn == d_eSigmoid) + return ActivationType::d_eSIGMOID; if (dfn == dTanh) return ActivationType::dTANH; if (dfn == dLinear) @@ -509,19 +516,8 @@ namespace Deep } #pragma endregion -#define TANH_TINYLIMIT 0.000244140625f -#define TANH_POLY_LIMIT 0.625f -#define TANH_BIGLIMIT 6.0f - -#define P0 -0.9643991794f -#define P1 -99.28772310f -#define P2 -1614.687684f - -#define Q0 112.8116785f -#define Q1 2235.488391f -#define Q2 4844.063053f - #pragma region tanh + static inline void tanh(float *RESTRICT x, const size_t n) noexcept { assert(n != 0 && "n must not be 0."); @@ -530,106 +526,279 @@ namespace Deep size_t simd_end = 0; #if defined(__AVX512F__) - simd_end = n - (n % 16); -#pragma omp parallel for schedule(static) if (n > 65536 && !omp_in_parallel()) - for (ptrdiff_t i = 0; i < (ptrdiff_t)(simd_end); i += 16) + + simd_end = n & ~(size_t)15; + +#pragma omp parallel for schedule(static) if (n > 16384 && !omp_in_parallel()) + for (ptrdiff_t i = 0; i < static_cast(simd_end); i += 16) { - __m512 x_512 = _mm512_loadu_ps(x + i); - __m512 res = Sleef_tanhf16_u10avx512f(x_512); - _mm512_storeu_ps(x + i, res); + __m512 v = _mm512_loadu_ps(x + i); + v = Sleef_tanhf16_u10avx512f(v); + _mm512_storeu_ps(x + i, v); } #elif defined(__AVX2__) - simd_end = n - (n % 8); -#pragma omp parallel for schedule(static) if (n > 65536 && !omp_in_parallel()) - for (ptrdiff_t i = 0; i < (ptrdiff_t)(simd_end); i += 8) + + simd_end = n & ~(size_t)7; + +#pragma omp parallel for schedule(static) if (n > 16384 && !omp_in_parallel()) + for (ptrdiff_t i = 0; i < static_cast(simd_end); i += 8) { - __m256 x_256 = _mm256_loadu_ps(x + i); - __m256 res = Sleef_tanhf8_u10avx2(x_256); - _mm256_storeu_ps(x + i, res); + __m256 v = _mm256_loadu_ps(x + i); + v = Sleef_tanhf8_u10avx2(v); + _mm256_storeu_ps(x + i, v); + } + +#elif defined(__AVX__) + + simd_end = n & ~(size_t)7; + +#pragma omp parallel for schedule(static) if (n > 16384 && !omp_in_parallel()) + for (ptrdiff_t i = 0; i < static_cast(simd_end); i += 8) + { + __m256 v = _mm256_loadu_ps(x + i); + v = Sleef_tanhf8_u10avx(v); + _mm256_storeu_ps(x + i, v); } #elif defined(__SSE4_1__) || defined(__SSE2__) || defined(_M_AMD64) || defined(_M_X64) - simd_end = n - (n % 4); -#pragma omp parallel for schedule(static) if (n > 65536 && !omp_in_parallel()) - for (ptrdiff_t i = 0; i < (ptrdiff_t)(simd_end); i += 4) + + simd_end = n & ~(size_t)3; + +#pragma omp parallel for schedule(static) if (n > 16384 && !omp_in_parallel()) + for (ptrdiff_t i = 0; i < static_cast(simd_end); i += 4) { - __m128 x_128 = _mm_loadu_ps(x + i); + __m128 v = _mm_loadu_ps(x + i); + #if defined(__SSE4_1__) - __m128 res = Sleef_tanhf4_u10sse4(x_128); + v = Sleef_tanhf4_u10sse4(v); #else - __m128 res = Sleef_tanhf4_u10sse2(x_128); + v = Sleef_tanhf4_u10sse2(v); #endif - _mm_storeu_ps(x + i, res); + + _mm_storeu_ps(x + i, v); } + #endif - for (size_t i = simd_end; i < n; i++) + for (size_t i = simd_end; i < n; ++i) { x[i] = Sleef_tanhf_u10(x[i]); } } - - static inline void dTanh(float *RESTRICT x, const size_t n, bool activated = false) noexcept + static inline void dTanh(float *RESTRICT x, + const size_t n, + const bool activated = false) noexcept { assert(n != 0 && "n must not be 0."); assert(x != nullptr && "x must not be null."); - if (!activated) - tanh(x, n); - size_t i = 0; - [[maybe_unused]] size_t simd_end; + + if (activated) + { +#if defined(__AVX512F__) + + const __m512 ones = _mm512_set1_ps(1.0f); + const size_t simd_end = n & ~(size_t)15; + + for (; i < simd_end; i += 16) + { + const __m512 t = _mm512_loadu_ps(x + i); + +#ifdef __FMA__ + const __m512 res = _mm512_fnmadd_ps(t, t, ones); +#else + const __m512 res = _mm512_sub_ps( + ones, + _mm512_mul_ps(t, t)); +#endif + + _mm512_storeu_ps(x + i, res); + } + +#elif defined(__AVX2__) + + const __m256 ones = _mm256_set1_ps(1.0f); + const size_t simd_end = n & ~(size_t)7; + + for (; i < simd_end; i += 8) + { + const __m256 t = _mm256_loadu_ps(x + i); + +#ifdef __FMA__ + const __m256 res = _mm256_fnmadd_ps(t, t, ones); +#else + const __m256 res = _mm256_sub_ps( + ones, + _mm256_mul_ps(t, t)); +#endif + + _mm256_storeu_ps(x + i, res); + } + +#elif defined(__SSE2__) || defined(_M_AMD64) || defined(_M_X64) + + const __m128 ones = _mm_set1_ps(1.0f); + const size_t simd_end = n & ~(size_t)3; + + for (; i < simd_end; i += 4) + { + const __m128 t = _mm_loadu_ps(x + i); + +#ifdef __FMA__ + const __m128 res = _mm_fnmadd_ps(t, t, ones); +#else + const __m128 res = _mm_sub_ps( + ones, + _mm_mul_ps(t, t)); +#endif + + _mm_storeu_ps(x + i, res); + } + +#endif + + for (; i < n; ++i) + { + const float t = x[i]; + x[i] = 1.0f - t * t; + } + + return; + } #if defined(__AVX512F__) - __m512 ones = _mm512_set1_ps(1.0f); - simd_end = n - (n % 16); - for (; i < simd_end; i += 16) { - __m512 t = _mm512_loadu_ps(x + i); + const __m512 ones = _mm512_set1_ps(1.0f); + const size_t simd_end = n & ~(size_t)15; + +#pragma omp parallel for schedule(static) if (n > 16384 && !omp_in_parallel()) + for (ptrdiff_t j = 0; + j < static_cast(simd_end); + j += 16) + { + __m512 t = _mm512_loadu_ps(x + j); + t = Sleef_tanhf16_u10avx512f(t); #ifdef __FMA__ - __m512 res = _mm512_fnmadd_ps(t, t, ones); // 1 - t*t + t = _mm512_fnmadd_ps(t, t, ones); #else - __m512 res = _mm512_sub_ps(ones, _mm512_mul_ps(t, t)); + t = _mm512_sub_ps( + ones, + _mm512_mul_ps(t, t)); #endif - _mm512_storeu_ps(x + i, res); + + _mm512_storeu_ps(x + j, t); + } + + i = simd_end; } #elif defined(__AVX2__) - __m256 ones = _mm256_set1_ps(1.0f); - simd_end = n - (n % 8); - for (; i < simd_end; i += 8) { - __m256 t = _mm256_loadu_ps(x + i); + const __m256 ones = _mm256_set1_ps(1.0f); + const size_t simd_end = n & ~(size_t)7; + +#pragma omp parallel for schedule(static) if (n > 16384 && !omp_in_parallel()) + for (ptrdiff_t j = 0; + j < static_cast(simd_end); + j += 8) + { + __m256 t = _mm256_loadu_ps(x + j); + t = Sleef_tanhf8_u10avx2(t); + #ifdef __FMA__ - __m256 res = _mm256_fnmadd_ps(t, t, ones); // 1 - t*t + t = _mm256_fnmadd_ps(t, t, ones); #else - __m256 res = _mm256_sub_ps(ones, _mm256_mul_ps(t, t)); + t = _mm256_sub_ps( + ones, + _mm256_mul_ps(t, t)); #endif - _mm256_storeu_ps(x + i, res); + + _mm256_storeu_ps(x + j, t); + } + + i = simd_end; } -#elif defined(__SSE4_1__) || defined(_M_AMD64) || defined(_M_X64) - __m128 ones = _mm_set1_ps(1.0f); - simd_end = n - (n % 4); - for (; i < simd_end; i += 4) +#elif defined(__AVX__) + { - __m128 t = _mm_loadu_ps(x + i); + const __m256 ones = _mm256_set1_ps(1.0f); + const size_t simd_end = n & ~(size_t)7; + +#pragma omp parallel for schedule(static) if (n > 16384 && !omp_in_parallel()) + for (ptrdiff_t j = 0; + j < static_cast(simd_end); + j += 8) + { + __m256 t = _mm256_loadu_ps(x + j); + t = Sleef_tanhf8_u10avx(t); #ifdef __FMA__ - __m128 res = _mm_fnmadd_ps(t, t, ones); // 1 - t*t + t = _mm256_fnmadd_ps(t, t, ones); #else - __m128 res = _mm_sub_ps(ones, _mm_mul_ps(t, t)); + t = _mm256_sub_ps( + ones, + _mm256_mul_ps(t, t)); #endif - _mm_storeu_ps(x + i, res); + + _mm256_storeu_ps(x + j, t); + } + + i = simd_end; } + +#elif defined(__SSE2__) || defined(_M_AMD64) || defined(_M_X64) + + { + const __m128 ones = _mm_set1_ps(1.0f); + const size_t simd_end = n & ~(size_t)3; + +#pragma omp parallel for schedule(static) if (n > 16384 && !omp_in_parallel()) + for (ptrdiff_t j = 0; + j < static_cast(simd_end); + j += 4) + { + __m128 t = _mm_loadu_ps(x + j); + +#if defined(__SSE4_1__) + t = Sleef_tanhf4_u10sse4(t); +#else + t = Sleef_tanhf4_u10sse2(t); +#endif + +#ifdef __FMA__ + t = _mm_fnmadd_ps(t, t, ones); +#else + t = _mm_sub_ps( + ones, + _mm_mul_ps(t, t)); #endif + _mm_storeu_ps(x + j, t); + } + + i = simd_end; + } + +#else + + i = 0; + +#endif + + /* + * Scalar tail. + */ for (; i < n; ++i) - x[i] = 1.0f - x[i] * x[i]; + { + const float t = Sleef_tanhf_u10(x[i]); + x[i] = 1.0f - t * t; + } } /// @brief Two-buffer, single-pass tanh derivative: reads src, writes @@ -978,13 +1147,192 @@ namespace Deep dst[i] = sig * (1.0f - sig); } } + + static inline void d_eSigmoid(float *RESTRICT x, + const size_t n, + const bool activated = false) noexcept + { + assert(n != 0 && "n must not be 0."); + assert(x != nullptr && "x must not be null."); + + if (!activated) + e_sigmoid(x, n); + + size_t i = 0; + +#if defined(__AVX512F__) + + const __m512 two = _mm512_set1_ps(2.0f); + + const size_t r = n % 16; + const size_t simd_end = n - r; + + for (; i < simd_end; i += 16) + { + const __m512 x_512 = _mm512_loadu_ps(x + i); + + // d = 2 * x * (1 - x) + const __m512 d = _mm512_mul_ps( + _mm512_fnmadd_ps(x_512, x_512, x_512), + two); + + _mm512_storeu_ps(x + i, d); + } + +#elif defined(__AVX2__) + + const __m256 two = _mm256_set1_ps(2.0f); + + const size_t r = n % 8; + const size_t simd_end = n - r; + + for (; i < simd_end; i += 8) + { + const __m256 x_256 = _mm256_loadu_ps(x + i); + +#ifdef __FMA__ + // x * (1 - x) = x - x^2 + const __m256 d = _mm256_mul_ps( + _mm256_fnmadd_ps(x_256, x_256, x_256), + two); +#else + const __m256 d = _mm256_mul_ps( + _mm256_sub_ps(x_256, _mm256_mul_ps(x_256, x_256)), + two); +#endif + + _mm256_storeu_ps(x + i, d); + } + +#elif defined(__SSE2__) || defined(_M_AMD64) || defined(_M_X64) + + const __m128 two = _mm_set1_ps(2.0f); + + const size_t r = n % 4; + const size_t simd_end = n - r; + + for (; i < simd_end; i += 4) + { + const __m128 x_128 = _mm_loadu_ps(x + i); + +#ifdef __FMA__ + const __m128 d = _mm_mul_ps( + _mm_fnmadd_ps(x_128, x_128, x_128), + two); +#else + const __m128 d = _mm_mul_ps( + _mm_sub_ps(x_128, _mm_mul_ps(x_128, x_128)), + two); +#endif + + _mm_storeu_ps(x + i, d); + } + +#endif + + for (; i < n; ++i) + { + x[i] = 2.0f * x[i] * (1.0f - x[i]); + } + } + + static inline void d_eSigmoidInto(float *RESTRICT dst, + const float *RESTRICT src, + const size_t n) noexcept + { + assert(n != 0 && "n must not be 0."); + assert(dst != nullptr && "dst must not be null."); + assert(src != nullptr && "src must not be null."); + + size_t i = 0; + +#if defined(__AVX512F__) + + const __m512 half = _mm512_set1_ps(0.5f); + const __m512 one = _mm512_set1_ps(1.0f); + + const size_t simd_end = n - (n % 16); + +#pragma omp parallel for schedule(static) if (n > 16384 && !omp_in_parallel()) + for (ptrdiff_t j = 0; j < (ptrdiff_t)simd_end; j += 16) + { + const __m512 s = _mm512_loadu_ps(src + j); + const __m512 a = _mm512_add_ps(_mm512_abs_ps(s), one); + const __m512 d = _mm512_div_ps(half, _mm512_mul_ps(a, a)); + + _mm512_storeu_ps(dst + j, d); + } + + i = simd_end; + +#elif defined(__AVX2__) + + const __m256 half = _mm256_set1_ps(0.5f); + const __m256 one = _mm256_set1_ps(1.0f); + const __m256 abs_mask = + _mm256_castsi256_ps(_mm256_set1_epi32(0x7FFFFFFF)); + + const size_t simd_end = n - (n % 8); + +#pragma omp parallel for schedule(static) if (n > 16384 && !omp_in_parallel()) + for (ptrdiff_t j = 0; j < (ptrdiff_t)simd_end; j += 8) + { + const __m256 s = _mm256_loadu_ps(src + j); + const __m256 a = _mm256_add_ps( + _mm256_and_ps(s, abs_mask), + one); + + const __m256 d = _mm256_div_ps( + half, + _mm256_mul_ps(a, a)); + + _mm256_storeu_ps(dst + j, d); + } + + i = simd_end; + +#elif defined(__SSE2__) || defined(_M_AMD64) || defined(_M_X64) + + const __m128 half = _mm_set1_ps(0.5f); + const __m128 one = _mm_set1_ps(1.0f); + const __m128 abs_mask = + _mm_castsi128_ps(_mm_set1_epi32(0x7FFFFFFF)); + + const size_t simd_end = n - (n % 4); + +#pragma omp parallel for schedule(static) if (n > 16384 && !omp_in_parallel()) + for (ptrdiff_t j = 0; j < (ptrdiff_t)simd_end; j += 4) + { + const __m128 s = _mm_loadu_ps(src + j); + const __m128 a = _mm_add_ps( + _mm_and_ps(s, abs_mask), + one); + + const __m128 d = _mm_div_ps( + half, + _mm_mul_ps(a, a)); + + _mm_storeu_ps(dst + j, d); + } + + i = simd_end; + +#endif + + for (; i < n; ++i) + { + const float a = 1.0f + std::fabs(src[i]); + dst[i] = 0.5f / (a * a); + } + } + #pragma endregion static inline void linear([[maybe_unused]] float *x, [[maybe_unused]] size_t n) noexcept {} static inline void dLinear(float *x, size_t n, [[maybe_unused]] bool activated = false) noexcept { - std::fill_n(x, n, 1.0f); + std::memset(x, 1.0f, n); } /// @brief Two-buffer variant of dLinear -- src is unused (the @@ -992,6 +1340,6 @@ namespace Deep /// signature consistency with To_dFn2's dispatch table. static inline void dLinearInto(float *RESTRICT dst, [[maybe_unused]] const float *RESTRICT src, size_t n) noexcept { - std::fill_n(dst, n, 1.0f); + std::memset(dst, 1.0f, n); } } diff --git a/src/backend/CPUBackend.cpp b/src/backend/CPUBackend.cpp index 768b02c..ff902cf 100644 --- a/src/backend/CPUBackend.cpp +++ b/src/backend/CPUBackend.cpp @@ -154,6 +154,4 @@ namespace Deep param[i] -= step_size * m[i] / (Sleef_sqrtf(v[i]) + eps); } } - - // ... (remaining methods stubbed out for you to map to cblas) ... } \ No newline at end of file diff --git a/src/backend/CUDABackend.cu b/src/backend/CUDABackend.cu index 82803d1..a10c9b5 100644 --- a/src/backend/CUDABackend.cu +++ b/src/backend/CUDABackend.cu @@ -1,9 +1,361 @@ #include +#include +#include +#include namespace Deep { - CUDABackend::CUDABackend() {} - CUDABackend::~CUDABackend() {} + CUDABackend::CUDABackend() + { + cublasCreate(&this->handle); + } - // ... (remaining methods left completely blank for your CUDA implementation) ... + CUDABackend::~CUDABackend() + { + cublasDestroy(this->handle); + } + + float *CUDABackend::Allocate(size_t numFloats) + { + float *ptr = nullptr; + if (cudaMalloc(&ptr, numFloats * sizeof(float)) != cudaSuccess) + { + std::cerr << "Could not allocate memory to CUDA backend.\n"; + return nullptr; + } + return ptr; + } + + void CUDABackend::Free(float *ptr) noexcept + { + if (cudaFree(ptr) != cudaSuccess) + std::cerr << "Could not free memory from CUDA backend.\n"; + } + + void CUDABackend::Zero(float *ptr, size_t numFloats) noexcept + { + cudaMemset(ptr, 0, numFloats * sizeof(float)); + } + + void CUDABackend::Copy(float *dst, const float *src, size_t numFloats) noexcept + { + cudaMemcpy(dst, src, numFloats * sizeof(float), cudaMemcpyDefault); + } + + void CUDABackend::CopyFromHost(float *deviceDst, const float *hostSrc, size_t numFloats) noexcept + { + cudaMemcpy(deviceDst, hostSrc, numFloats * sizeof(float), cudaMemcpyHostToDevice); + } + + void CUDABackend::CopyToHost(float *hostDst, const float *deviceSrc, size_t numFloats) noexcept + { + cudaMemcpy(hostDst, deviceSrc, numFloats * sizeof(float), cudaMemcpyDeviceToHost); + } + + void CUDABackend::MatMul(bool transA, bool transB, int M, int N, int K, + float alpha, const float *A, int lda, + const float *B, int ldb, + float beta, float *C, int ldc) noexcept + { + // CUDA is *column-major*. + cublasOperation_t cuTransA = transA ? CUBLAS_OP_N : CUBLAS_OP_T; + cublasOperation_t cuTransB = transB ? CUBLAS_OP_N : CUBLAS_OP_T; + + if (cublasSgemm(handle, cuTransB, cuTransA, + N, M, K, &alpha, B, ldb, A, lda, &beta, C, ldc) != CUBLAS_STATUS_SUCCESS) + std::cerr << "Failed to perform CUDA MatMul.\n"; + } + + void CUDABackend::Scale(float *buf, size_t n, float alpha) noexcept + { + if (cublasSscal(handle, n, &alpha, buf, 1) != CUBLAS_STATUS_SUCCESS) + std::cerr << "Failed to perform CUDA Scale.\n"; + } + + void CUDABackend::AxpyInto(float *y, const float *x, size_t n, float alpha) noexcept + { + if (cublasSaxpy(handle, n, &alpha, x, 1, y, 1)) + std::cerr << "Failed to perform CUDA Axpy.\n"; + } + +#pragma region ACTIVATIONS_AND_KERNELS + + __global__ void ReluKernelInto(float *dst, const float *src, size_t n) + { + size_t i = (size_t)blockIdx.x * blockDim.x + threadIdx.x; + if (i < n) + dst[i] = fmaxf(0.0f, src[i]); + } + + __global__ void tanhKernelInto(float *dst, const float *src, size_t n) + { + size_t i = (size_t)blockIdx.x * blockDim.x + threadIdx.x; + if (i < n) + { + float y; + asm("tanh.approx.f32 %0, %1;" : "=f"(y) : "f"(src[i])); + dst[i] = y; + } + } + + __global__ void sigmoidKernelInto(float *dst, const float *src, size_t n) + { + size_t i = (size_t)blockIdx.x * blockDim.x + threadIdx.x; + if (i < n) + { + float t; + asm("tanh.approx.f32 %0, %1;" : "=f"(t) : "f"(0.5f * src[i])); + dst[i] = fmaf(0.5f, t, 0.5f); + } + } + + __global__ void eSigmoidKernelInto(float *dst, const float *src, size_t n) + { + size_t i = (size_t)blockIdx.x * blockDim.x + threadIdx.x; + if (i < n) + { + float s = src[i]; + dst[i] = 0.5f * (s / (1.0f + fabsf(s)) + 1.0f); + } + } + + __global__ void linearKernelInto(float *dst, const float *src, size_t n) + { + size_t i = (size_t)blockIdx.x * blockDim.x + threadIdx.x; + if (i < n) + dst[i] = src[i]; + } + + __global__ void dReluKernelInto(float *dst, const float *src, size_t n) + { + size_t i = (size_t)blockIdx.x * blockDim.x + threadIdx.x; + if (i < n) + dst[i] = (float)(src[i] > 0.0f); + } + + __global__ void dTanhKernelInto(float *dst, const float *src, size_t n) + { + size_t i = (size_t)blockIdx.x * blockDim.x + threadIdx.x; + if (i < n) + { + float y = src[i]; + dst[i] = fmaf(-y, y, 1.0f); + } + } + + __global__ void dSigmoidKernelInto(float *dst, const float *src, size_t n) + { + size_t i = (size_t)blockIdx.x * blockDim.x + threadIdx.x; + if (i < n) + { + float t; + asm("tanh.approx.f32 %0, %1;" : "=f"(t) : "f"(0.5f * src[i])); + dst[i] = 0.25f * fmaf(-t, t, 1.0f); + } + } + + __global__ void dSigmoidActivatedKernelInto(float *dst, const float *src, size_t n) + { + size_t i = (size_t)blockIdx.x * blockDim.x + threadIdx.x; + if (i < n) + { + float s = src[i]; + dst[i] = fmaf(-s, s, s); + } + } + + __global__ void d_eSigmoidKernelInto(float *dst, const float *src, size_t n) + { + size_t i = (size_t)blockIdx.x * blockDim.x + threadIdx.x; + if (i < n) + { + float a = 1.0f + fabsf(src[i]); + dst[i] = 0.5f / (a * a); + } + } + + __global__ void d_eSigmoidActivatedKernelInto(float *dst, const float *src, size_t n) + { + size_t i = (size_t)blockIdx.x * blockDim.x + threadIdx.x; + if (i < n) + { + float s = src[i]; + dst[i] = 2.0f * fmaf(-s, s, s); + } + } + + __global__ void dLinearKernelInto(float *dst, const float *src, size_t n) + { + size_t i = (size_t)blockIdx.x * blockDim.x + threadIdx.x; + if (i < n) + dst[i] = 1.0f; + } + + __global__ void FusedStateUpdateKernel(float *z, const float *feedback, const float *deriv, + const float *e, size_t n, float ir) + { + size_t i = (size_t)blockIdx.x * blockDim.x + threadIdx.x; + if (i < n) + { + z[i] += ir * ((feedback[i] * deriv[i]) - e[i]); + } + } + + __global__ void ComputeErrorKernel(float *e, const float *z, const float *mu, size_t n) + { + size_t i = (size_t)blockIdx.x * blockDim.x + threadIdx.x; + if (i < n) + { + e[i] = z[i] - mu[i]; + } + } + + __global__ void AdamStepKernel(float *param, const float *grad, float *m, float *v, + size_t n, float step_size, float beta1, float beta2, float eps) + { + size_t i = (size_t)blockIdx.x * blockDim.x + threadIdx.x; + if (i < n) + { + float g = grad[i]; + m[i] = beta1 * m[i] + (1.0f - beta1) * g; + v[i] = beta2 * v[i] + (1.0f - beta2) * (g * g); + param[i] -= step_size * m[i] / (sqrtf(v[i]) + eps); + } + } + + __global__ void AdamWStepKernel(float *param, const float *grad, float *m, float *v, + size_t n, float lr, float weightDecay, + float step_size, float beta1, float beta2, float eps) + { + size_t i = (size_t)blockIdx.x * blockDim.x + threadIdx.x; + if (i < n) + { + float g = grad[i]; + m[i] = beta1 * m[i] + (1.0f - beta1) * g; + v[i] = beta2 * v[i] + (1.0f - beta2) * (g * g); + + param[i] -= lr * weightDecay * param[i]; + param[i] -= step_size * m[i] / (sqrtf(v[i]) + eps); + } + } + +#pragma endregion + + void CUDABackend::Activation(ActivationType type, float *buf, size_t n) noexcept + { + // For in-place, pass buf as both dst and src + ActivationInto(type, buf, buf, n); + } + + void CUDABackend::ActivationInto(ActivationType type, float *dst, const float *src, size_t n) noexcept + { + constexpr int BLOCK_SIZE = 256; + const int blocks = static_cast((n + BLOCK_SIZE - 1) / BLOCK_SIZE); + + switch (type) + { + case ActivationType::RELU: + ReluKernelInto<<>>(dst, src, n); + break; + case ActivationType::SIGMOID: + sigmoidKernelInto<<>>(dst, src, n); + break; + case ActivationType::eSIGMOID: + eSigmoidKernelInto<<>>(dst, src, n); + break; + case ActivationType::TANH: + tanhKernelInto<<>>(dst, src, n); + break; + case ActivationType::LINEAR: + linearKernelInto<<>>(dst, src, n); + break; + case ActivationType::NONE: + default: + break; + } + } + + void CUDABackend::ActivationDerivative(ActivationType type, float *buf, size_t n, bool activated) noexcept + { + ActivationDerivativeInto(type, buf, buf, n); + } + + void CUDABackend::ActivationDerivativeInto(ActivationType type, float *dst, const float *src, size_t n) noexcept + { + constexpr int BLOCK_SIZE = 256; + const int blocks = static_cast((n + BLOCK_SIZE - 1) / BLOCK_SIZE); + + switch (type) + { + case ActivationType::dRELU: + dReluKernelInto<<>>(dst, src, n); + break; + case ActivationType::dSIGMOID: + dSigmoidKernelInto<<>>(dst, src, n); + break; + case ActivationType::d_eSIGMOID: + d_eSigmoidKernelInto<<>>(dst, src, n); + break; + case ActivationType::dTANH: + dTanhKernelInto<<>>(dst, src, n); + break; + case ActivationType::dLINEAR: + dLinearKernelInto<<>>(dst, src, n); + break; + case ActivationType::NONE: + default: + break; + } + } + + void CUDABackend::FusedStateUpdate(float *z, const float *feedback, const float *deriv, + const float *e, size_t n, float ir) noexcept + { + constexpr int BLOCK_SIZE = 256; + const int blocks = static_cast((n + BLOCK_SIZE - 1) / BLOCK_SIZE); + + FusedStateUpdateKernel<<>>(z, feedback, deriv, e, n, ir); + } + + float CUDABackend::ComputeErrorAndEnergy(float *e, const float *z, const float *mu, size_t n) noexcept + { + constexpr int BLOCK_SIZE = 256; + const int blocks = static_cast((n + BLOCK_SIZE - 1) / BLOCK_SIZE); + + ComputeErrorKernel<<>>(e, z, mu, n); + + float sum_of_squares = 0.0f; + cublasSdot(handle, n, e, 1, e, 1, &sum_of_squares); + + return 0.5f * sum_of_squares; + } + + void CUDABackend::AdamStep(float *param, const float *grad, float *m, float *v, + size_t n, int t, float lr, + float beta1, float beta2, float eps) noexcept + { + // Compute scalars on host CPU + float beta1_t = 1.0f - Sleef_powf_u10(beta1, static_cast(t)); + float beta2_t = 1.0f - Sleef_powf_u10(beta2, static_cast(t)); + float step_size = lr * Sleef_sqrtf_u05(beta2_t) / beta1_t; + + constexpr int BLOCK_SIZE = 256; + const int blocks = static_cast((n + BLOCK_SIZE - 1) / BLOCK_SIZE); + + AdamStepKernel<<>>(param, grad, m, v, n, step_size, beta1, beta2, eps); + } + + void CUDABackend::AdamWStep(float *param, const float *grad, float *m, float *v, + size_t n, int t, float lr, float weightDecay, + float beta1, float beta2, float eps) noexcept + { + float beta1_t = 1.0f - Sleef_powf_u10(beta1, static_cast(t)); + float beta2_t = 1.0f - Sleef_powf_u10(beta2, static_cast(t)); + float step_size = lr * Sleef_sqrtf_u05(beta2_t) / beta1_t; + + constexpr int BLOCK_SIZE = 256; + const int blocks = static_cast((n + BLOCK_SIZE - 1) / BLOCK_SIZE); + + AdamWStepKernel<<>>(param, grad, m, v, n, lr, weightDecay, step_size, beta1, beta2, eps); + } } \ No newline at end of file From 545b137f946d1ad019cd451147b79fd97f34350e Mon Sep 17 00:00:00 2001 From: Ra4ster Date: Sun, 6 Sep 2026 02:07:08 +0000 Subject: [PATCH 04/11] CUDA: Fixing AXPY fails. --- CMakeLists.txt | 17 +- bindings/pybinding.cpp | 35 +- include/deepity/backend/CPUBackend.h | 5 + include/deepity/backend/CUDABackend.h | 5 + include/deepity/backend/DeviceType.h | 11 + include/deepity/backend/IComputeBackend.h | 20 ++ include/deepity/backend/Tensor.h | 7 +- include/deepity/layers/SimplePCLayer.h | 57 ++- include/deepity/networks/SimplePCNetwork.h | 58 +++- include/deepity/utils/Activations.h | 6 +- include/deepity/utils/DeviceMemoryArena.h | 191 ++++++---- logs/build.log | 240 +------------ pydeepity/SimplePCN.py | 8 +- .../__pycache__/SimplePCN.cpython-312.pyc | Bin 11703 -> 11790 bytes src/SimplePCLayer.cpp | 327 +++++++----------- src/SimplePCNetwork.cpp | 53 ++- src/backend/Backend.cpp | 8 +- src/backend/CPUBackend.cpp | 49 ++- src/backend/CUDABackend.cu | 84 ++++- temp.py | 151 ++++++++ tests/tCUDAFunctionsVerify.cpp | 237 +++++++++++++ 21 files changed, 999 insertions(+), 570 deletions(-) create mode 100644 include/deepity/backend/DeviceType.h create mode 100644 temp.py create mode 100644 tests/tCUDAFunctionsVerify.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index f65b485..626a429 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -144,6 +144,10 @@ endif() if(DEEPITY_ENABLE_CUDA) find_package(CUDAToolkit QUIET) if(CUDAToolkit_FOUND) + enable_language(CUDA) + set(CMAKE_CUDA_STANDARD 17) + set(CMAKE_CUDA_STANDARD_REQUIRED ON) + set(CMAKE_CUDA_ARCHITECTURES 86) message(STATUS "CUDA support enabled.") else() message(STATUS "CUDA toolkit not found. Building CPU-only.") @@ -273,7 +277,7 @@ add_library(Deepity src/StreamAlignedBatcher.cpp src/backend/Backend.cpp src/backend/CPUBackend.cpp - src/backend/CUDABackend.cpp + src/backend/CUDABackend.cu src/backend/Tensor.cpp ) @@ -304,6 +308,17 @@ endif() target_compile_definitions(Deepity PUBLIC SLEEF_STATIC_LIBS) +# --- Tests ----------------------------------------------------------- + +option(DEEPITY_BUILD_TESTS "Build test/verification executables" ON) + +add_executable(GaussSeidelVerify tests/tGaussSeidelVerify.cpp) +target_link_libraries(GaussSeidelVerify PRIVATE Deepity) +set_target_properties(GaussSeidelVerify PROPERTIES + RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin +) +add_test(NAME GaussSeidelVerify COMMAND GaussSeidelVerify) + # --- Compiler flags ------------------------------------------------- if(MSVC) diff --git a/bindings/pybinding.cpp b/bindings/pybinding.cpp index 144950c..5b45f7a 100644 --- a/bindings/pybinding.cpp +++ b/bindings/pybinding.cpp @@ -426,30 +426,39 @@ void bind_networks(nb::module_ &m) auto simpleNetCls = nb::class_(m, "SimplePCNetwork", "Predictive Coding Network built from SimplePCLayers."); BindCommonPCNetwork(simpleNetCls, "SimplePCNetwork"); + simpleNetCls.def("__init__", [](Deep::SimplePCNetwork *self, int batch_size, const std::string &device) + { + Deep::DeviceType dt = (device == "cuda" || device == "gpu") + ? Deep::DeviceType::DEVICE_GPU + : Deep::DeviceType::DEVICE_CPU; + new (self) Deep::SimplePCNetwork(batch_size, dt); }, nb::arg("batch_size"), nb::arg("device") = "cpu", "Construct a network with a fixed batch size and device (\"cpu\" or \"cuda\"/\"gpu\")."); simpleNetCls.def("add_layer", [](Deep::SimplePCNetwork &self, int size, int next_size, float lr, float ir, float lmbda, const std::string &activation, const std::string &activation_deriv) { self.AddLayer(size, next_size, lr, ir, lmbda, resolveActEnum(activation), resolveActEnum(activation_deriv)); }, nb::arg("size"), nb::arg("next_size"), nb::arg("lr") = 1e-6f, nb::arg("ir") = 0.1f, nb::arg("lmbda") = 1e-2f, nb::arg("activation") = "relu", nb::arg("activation_deriv") = "drelu", "Add a layer to the network.") .def("set_optimizer", [](Deep::SimplePCNetwork &self, const std::string &opt) { - if (opt == "ADAM") self.SetOptimizer(Deep::OptimizerType::ADAM); - else if (opt == "ADAMW") self.SetOptimizer(Deep::OptimizerType::ADAMW); - else self.SetOptimizer(Deep::OptimizerType::SGD); }, nb::arg("optimizer"), "Sets the optimizer: ADAM, ADAMW, or SGD.") - + if (opt == "ADAM") self.SetOptimizer(Deep::OptimizerType::ADAM); + else if (opt == "ADAMW") self.SetOptimizer(Deep::OptimizerType::ADAMW); + else self.SetOptimizer(Deep::OptimizerType::SGD); }, nb::arg("optimizer"), "Sets the optimizer: ADAM, ADAMW, or SGD.") .def("project_forward", &Deep::SimplePCNetwork::ProjectForward, "Seeds hidden layers from a genuine forward pass through current " "weights, instead of zero-init. Call AFTER clamp_input(), BEFORE " "the settling loop.") - .def("train_step_with_projection", [](Deep::SimplePCNetwork &self, FloatArray x, FloatArray y, int steps) { - std::vector xvec(x.data(), x.data() + x.size()); - std::vector yvec(y.data(), y.data() + y.size()); - return self.TrainStepWithProjection(xvec, yvec, steps); }, nb::arg("x"), nb::arg("y"), nb::arg("steps")) +std::vector xvec(x.data(), x.data() + x.size()); +std::vector yvec(y.data(), y.data() + y.size()); +return self.TrainStepWithProjection(xvec, yvec, steps); }, nb::arg("x"), nb::arg("y"), nb::arg("steps")) .def("predict_with_projection", [](Deep::SimplePCNetwork &self, FloatArray x, int steps) { - std::vector xvec(x.data(), x.data() + x.size()); - - std::vector out_beliefs = self.PredictWithProjection(xvec, steps); +std::vector xvec(x.data(), x.data() + x.size()); +std::vector out_beliefs = self.PredictWithProjection(xvec, steps); +return CopyToNewArray(out_beliefs.data(), {out_beliefs.size()}); }, nb::arg("x"), nb::arg("steps"), "Runs forward-projection init and settling loop entirely in C++, returning terminal beliefs.") + .def("randomize_weights", [](Deep::SimplePCNetwork &self, const std::string &distribution) + { + std::random_device rd; + std::mt19937 rng(rd()); + self.RandomizeWeights(rng, distribution.c_str()); }, nb::arg("distribution"), "Initialize every layer's weights using a distribution string, " + "e.g. \"normal(0, 1)\" or \"uniform(-0.3, 0.3)\"."); - return CopyToNewArray(out_beliefs.data(), {out_beliefs.size()}); }, nb::arg("x"), nb::arg("steps"), "Runs forward-projection init and settling loop entirely in C++, returning terminal beliefs."); nb::class_(m, "GaussSeidelPCNetwork", "Predictive Coding Network with Gauss-Seidel settling dynamics.") .def(nb::init(), nb::arg("batch_size")) .def("add_layer", [](Deep::GaussSeidelPCNetwork &self, int size, int next_size, float lr, float ir, float lmbda, const std::string &activation, const std::string &activation_deriv) @@ -706,9 +715,7 @@ void bind_utilities(nb::module_ &m) { Deep::dSigmoid(x.data(), x.size()); }); } -// ============================================================================ // Main Module Entry -// ============================================================================ NB_MODULE(pydeepity, m) { m.doc() = "Deepity: A high-performance Predictive Coding library."; diff --git a/include/deepity/backend/CPUBackend.h b/include/deepity/backend/CPUBackend.h index 033185c..cbebe52 100644 --- a/include/deepity/backend/CPUBackend.h +++ b/include/deepity/backend/CPUBackend.h @@ -1,5 +1,6 @@ #pragma once #include +#include namespace Deep { @@ -15,6 +16,8 @@ namespace Deep void Copy(float *dst, const float *src, size_t numFloats) noexcept override; void CopyFromHost(float *deviceDst, const float *hostSrc, size_t numFloats) noexcept override; void CopyToHost(float *hostDst, const float *deviceSrc, size_t numFloats) noexcept override; + void RandomizeNormal(float *buf, size_t n, float mean, float stddev, uint32_t seed) noexcept override; + void RandomizeUniform(float *buf, size_t n, float min, float max, uint32_t seed) noexcept override; void MatMul(bool transA, bool transB, int M, int N, int K, float alpha, const float *A, int lda, @@ -38,5 +41,7 @@ namespace Deep void AdamWStep(float *param, const float *grad, float *m, float *v, size_t n, int t, float lr, float weightDecay, float beta1 = 0.9f, float beta2 = 0.999f, float eps = 1e-8f) noexcept override; + + DeviceType GetDeviceType() const noexcept override { return DeviceType::DEVICE_CPU; }; }; } \ No newline at end of file diff --git a/include/deepity/backend/CUDABackend.h b/include/deepity/backend/CUDABackend.h index b6b53bc..a96cf6c 100644 --- a/include/deepity/backend/CUDABackend.h +++ b/include/deepity/backend/CUDABackend.h @@ -1,6 +1,7 @@ #pragma once #include #include +#include namespace Deep { @@ -16,6 +17,8 @@ namespace Deep void Copy(float *dst, const float *src, size_t numFloats) noexcept override; void CopyFromHost(float *deviceDst, const float *hostSrc, size_t numFloats) noexcept override; void CopyToHost(float *hostDst, const float *deviceSrc, size_t numFloats) noexcept override; + void RandomizeNormal(float *buf, size_t n, float mean, float stddev, uint32_t seed) noexcept override; + void RandomizeUniform(float *buf, size_t n, float min, float max, uint32_t seed) noexcept override; void MatMul(bool transA, bool transB, int M, int N, int K, float alpha, const float *A, int lda, @@ -40,6 +43,8 @@ namespace Deep size_t n, int t, float lr, float weightDecay, float beta1 = 0.9f, float beta2 = 0.999f, float eps = 1e-8f) noexcept override; + DeviceType GetDeviceType() const noexcept override { return DeviceType::DEVICE_GPU; } + private: cublasHandle_t handle; }; diff --git a/include/deepity/backend/DeviceType.h b/include/deepity/backend/DeviceType.h new file mode 100644 index 0000000..bfc730c --- /dev/null +++ b/include/deepity/backend/DeviceType.h @@ -0,0 +1,11 @@ +#pragma once + +/// @brief This file exists to provide both Tensor.h and IComputeBackend.h with device types without introducing circular dependencies. +namespace Deep +{ + enum class DeviceType + { + DEVICE_CPU, + DEVICE_GPU + }; +} \ No newline at end of file diff --git a/include/deepity/backend/IComputeBackend.h b/include/deepity/backend/IComputeBackend.h index 670b451..77d73f5 100644 --- a/include/deepity/backend/IComputeBackend.h +++ b/include/deepity/backend/IComputeBackend.h @@ -1,6 +1,7 @@ #pragma once #include #include +#include /** * @file IComputeBackend.h @@ -29,6 +30,7 @@ namespace Deep { + class IComputeBackend { public: @@ -53,6 +55,22 @@ namespace Deep /// @brief Device-to-host copy, e.g. for Save()/inspection. virtual void CopyToHost(float *hostDst, const float *deviceSrc, size_t numFloats) noexcept = 0; + /// @brief Randomizes floats using standard normal distribution. + /// @param buf Array of floats + /// @param n Size of buf + /// @param mean Mu parameter of Normal Dist. + /// @param stddev Sigma parameter of Normal Dist. + /// @param seed Random seed + virtual void RandomizeNormal(float *buf, size_t n, float mean, float stddev, uint32_t seed) noexcept = 0; + + /// @brief Randomizes floats using uniform distribution. + /// @param buf Array of floats + /// @param n Size of buf + /// @param mean Min value for random generation + /// @param stddev Max value for random generation + /// @param seed Random seed + virtual void RandomizeUniform(float *buf, size_t n, float min, float max, uint32_t seed) noexcept = 0; + // --- GEMM --------------------------------------------------------- virtual void MatMul(bool transA, bool transB, @@ -112,5 +130,7 @@ namespace Deep virtual void AdamWStep(float *param, const float *grad, float *m, float *v, size_t n, int t, float lr, float weightDecay, float beta1 = 0.9f, float beta2 = 0.999f, float eps = 1e-8f) noexcept = 0; + + virtual DeviceType GetDeviceType() const noexcept = 0; }; } \ No newline at end of file diff --git a/include/deepity/backend/Tensor.h b/include/deepity/backend/Tensor.h index 7fe3a5a..bfd849d 100644 --- a/include/deepity/backend/Tensor.h +++ b/include/deepity/backend/Tensor.h @@ -2,6 +2,7 @@ #include #include #include +#include /** * @file Tensor.h @@ -22,12 +23,6 @@ namespace Deep { - enum class DeviceType - { - DEVICE_CPU, - DEVICE_GPU - }; - class Tensor { public: diff --git a/include/deepity/layers/SimplePCLayer.h b/include/deepity/layers/SimplePCLayer.h index f980c8e..2a918b2 100644 --- a/include/deepity/layers/SimplePCLayer.h +++ b/include/deepity/layers/SimplePCLayer.h @@ -8,7 +8,9 @@ #include #include #include +#include #include +#include /** * @file SimplePCLayer.h @@ -52,8 +54,18 @@ * every step while clamped. This is an EXACT optimization, not an * approximation -- default OFF so both behaviors coexist for direct * correctness/timing comparison before trusting it. - * @version 1.0 - * @date 2026-06-30 + * + * @note As of this revision, all math is routed through an + * IComputeBackend rather than calling cblas_/Deep::* functions + * directly, so this layer can run on either CPUBackend or CUDABackend. + * `backend` defaults to nullptr, in which case the layer constructs and + * owns its own CPUBackend internally -- every existing call site + * (SimplePCNetwork::AddLayer, nanobind bindings, hand-written C++) + * keeps working completely unchanged, silently getting today's exact + * CPU-only behavior. Passing a real backend explicitly is only needed + * for new, GPU-aware call sites. + * @version 1.1 + * @date 2026-09-05 * @author Jack Rose */ @@ -74,10 +86,17 @@ namespace Deep /// @param lmbda Weight decay (L2 regularization) coefficient /// @param act Activation function /// @param dAct Derivative of activation function + /// @param backend Compute backend to route all math through. + /// Defaults to nullptr, in which case this layer + /// constructs and owns its own CPUBackend internally -- + /// existing callers don't need to change anything. Pass a + /// real backend (owned elsewhere, e.g. by the network) to + /// run this layer on GPU. SimplePCLayer(size_t size, size_t nextSize, size_t batchSize = 1, float learningRate = 1e-6f, float inferenceRate = 0.1f, float lmbda = 1e-2f, void (*act)(float *, size_t) = relu, - void (*dAct)(float *, size_t, bool) = dRelu); + void (*dAct)(float *, size_t, bool) = dRelu, + IComputeBackend *backend = nullptr); /// @brief Constructor for a SimplePCLayer, using a named /// ActivationType instead of raw function pointers. @@ -90,9 +109,13 @@ namespace Deep /// @param lmbda Weight decay (L2 regularization) coefficient /// @param aType Activation type /// @param dType Activation derivative type + /// @param backend Compute backend to route all math through. + /// See the other constructor's doc for the default-nullptr + /// behavior. SimplePCLayer(size_t size, size_t nextSize, size_t batchSize = 1, float learningRate = 1e-6f, float inferenceRate = 0.1f, float lmbda = 1e-2f, - ActivationType aType = ActivationType::RELU, ActivationType dType = ActivationType::dRELU); + ActivationType aType = ActivationType::RELU, ActivationType dType = ActivationType::dRELU, + IComputeBackend *backend = nullptr); /// @brief Calculates the total network energy state, and this /// layer's outgoing prediction (mu). @@ -233,7 +256,8 @@ namespace Deep /// @brief Randomizes this layer's weights (and biases) in place. /// @param twister The classic Mersenne Twister - void RandomizeWeights(std::mt19937 &twister) noexcept; + /// @param distribution The distribution of the randomization: (normal, uniform) + void RandomizeWeights(std::mt19937 &twister, const char *distribution = "normal") noexcept; /// @brief Returns this layer's configured activation type. ActivationType GetActivationType() const noexcept { return To_AType(activation); } @@ -247,11 +271,28 @@ namespace Deep size_t GetRequiredFloats() const noexcept; /// @brief Binds this layer's weight/state/scratch buffers into the /// supplied arena. Must be called before any other operation. - /// @param arena The MemoryArena to bind into. - void BindMemory(MemoryArena &arena); + /// Templated so either MemoryArena (CPU) or DeviceMemoryArena + /// (GPU) can be bound, resolved entirely at compile time -- see + /// the .cpp's explicit instantiations for the two concrete types + /// actually used. + /// @param arena The arena to bind into. + template + void BindMemory(ArenaT &arena); private: std::unique_ptr localArena; + + /// @brief The compute backend this layer routes all math + /// through. A unique_ptr with a swappable deleter, rather than + /// two separate owning/non-owning members: when constructed with + /// an explicit external backend (network-owned, GPU case), the + /// deleter is a no-op; when this layer had to construct its own + /// fallback CPUBackend (backend=nullptr was passed in), the + /// deleter actually frees it. Every call site still just uses + /// backend->Something(), identical to a raw pointer. + using BackendDeleter = void (*)(IComputeBackend *); + std::unique_ptr backend; + float *W; float *b; float *e; @@ -315,4 +356,4 @@ namespace Deep friend class SimplePCNDiagnostics; }; -} // namespace Deep +} // namespace Deep \ No newline at end of file diff --git a/include/deepity/networks/SimplePCNetwork.h b/include/deepity/networks/SimplePCNetwork.h index 628d6b7..29c1eea 100644 --- a/include/deepity/networks/SimplePCNetwork.h +++ b/include/deepity/networks/SimplePCNetwork.h @@ -4,6 +4,8 @@ #include #include #include +#include +#include /** * @file SimplePCNetwork.h @@ -22,8 +24,15 @@ * network.CalculateState(); * * @note All layers are stored in a vector. - * @version 1.0 - * @date 2026-06-30 + * + * @note As of this revision, the network owns a single IComputeBackend + * (CPU by default, GPU if requested at construction) and passes it down + * into every layer it creates. `device` defaults to DEVICE_CPU, so every + * existing caller (nanobind bindings, hand-written C++) keeps compiling + * and behaving exactly as before -- passing DEVICE_GPU explicitly is + * only needed for new, GPU-aware call sites. + * @version 1.1 + * @date 2026-09-05 * @author Jack Rose */ @@ -35,7 +44,11 @@ namespace Deep public: /// @brief Constructs an empty network with a predetermined batch size. /// @param batchSize Batch size - explicit SimplePCNetwork(int batchSize) noexcept; + /// @param device Which device this network's layers should run + /// on. Defaults to DEVICE_CPU, preserving existing + /// behavior exactly for anyone not explicitly requesting + /// DEVICE_GPU. + explicit SimplePCNetwork(int batchSize, DeviceType device = DeviceType::DEVICE_CPU) noexcept; SimplePCNetwork(const SimplePCNetwork &) = delete; SimplePCNetwork &operator=(const SimplePCNetwork &) = delete; @@ -56,6 +69,11 @@ namespace Deep void AddLayer(int size, int nextSize, float lr, float ir, float lmbda, ActivationType aType, ActivationType dType); + /// @brief Randomizes the weights of each layer + /// @param rng The classic Mersenne Twister + /// @param distribution A string representation of (normal, uniform) distribution. For example: "normal(0, 1)" for a standard normal. + void RandomizeWeights(std::mt19937 &rng, const char *distribution); + /// @brief Randomizes the weights of each layer /// @param rng The classic Mersenne Twister void RandomizeWeights(std::mt19937 &rng); @@ -95,6 +113,9 @@ namespace Deep /// @return size_t batchSize int GetBatchSize() const noexcept { return batchSize; } + /// @brief Returns which device this network's layers run on. + DeviceType GetDevice() const noexcept { return device; } + /// @brief Sets the optimizer used for weight updates on every layer. /// @param o The optimizer type to apply. void SetOptimizer(OptimizerType o) noexcept @@ -148,18 +169,41 @@ namespace Deep /// ONE crossing instead. float TrainStepWithProjection(const std::vector &x, const std::vector &y, int inferenceSteps); - std::vector PredictWithProjection(const std::vector &x, int inferenceSteps); + std::vector PredictWithProjection(const std::vector &x, int inferenceSteps); /// @brief Sets mu-cache threshold on every layer -- see /// SimplePCLayer::SetMuCacheThreshold() for semantics. Safe to call any /// time after Compile(). void SetMuCacheThreshold(float threshold) noexcept; - /// @brief Loads all layers into one contiguous block of memory. + /// @brief Loads all layers into one contiguous block of memory + /// (MemoryArena for DEVICE_CPU, DeviceMemoryArena for DEVICE_GPU). void Compile(); private: std::vector> layers; - std::unique_ptr arena; + + /// @brief The network's own compute backend, created once at + /// construction and shared by every layer added afterward. + std::unique_ptr backend; + /// @brief Which device `backend` actually is -- kept alongside + /// it since IComputeBackend itself doesn't expose its own type, + /// and Compile() needs to know which arena type to construct. + DeviceType device; + + /// @brief Used when device == DEVICE_CPU. Only one of + /// cpuArena/gpuArena is ever actually constructed for a given + /// network -- they're kept as two separate members (rather than + /// one unified type) because MemoryArena and DeviceMemoryArena + /// share no common base, matching the same reasoning already + /// applied when DeviceMemoryArena was designed. + std::unique_ptr cpuArena; + /// @brief Used when device == DEVICE_GPU. Only compiled at all + /// when DEEPITY_ENABLE_CUDA is defined, matching + /// DeviceMemoryArena.h's own guard. +#if defined(DEEPITY_ENABLE_CUDA) + std::unique_ptr gpuArena; +#endif + int batchSize; }; -} +} \ No newline at end of file diff --git a/include/deepity/utils/Activations.h b/include/deepity/utils/Activations.h index 1cfa2cb..648b039 100644 --- a/include/deepity/utils/Activations.h +++ b/include/deepity/utils/Activations.h @@ -69,11 +69,13 @@ namespace Deep static inline void dRelu(float *, size_t, bool) noexcept; static inline void dSigmoid(float *, size_t, bool) noexcept; + static inline void d_eSigmoid(float *, size_t, bool) noexcept; static inline void dTanh(float *, size_t, bool) noexcept; static inline void dLinear(float *, size_t, bool) noexcept; static inline void dReluInto(float *RESTRICT, const float *RESTRICT, size_t) noexcept; static inline void dSigmoidInto(float *RESTRICT, const float *RESTRICT, size_t) noexcept; + static inline void d_eSigmoidInto(float *RESTRICT, const float *RESTRICT, size_t) noexcept; static inline void dTanhInto(float *RESTRICT, const float *RESTRICT, size_t) noexcept; static inline void dLinearInto(float *RESTRICT, const float *RESTRICT, size_t) noexcept; @@ -1332,7 +1334,7 @@ namespace Deep static inline void dLinear(float *x, size_t n, [[maybe_unused]] bool activated = false) noexcept { - std::memset(x, 1.0f, n); + std::fill(x, x + n, 1.0f); } /// @brief Two-buffer variant of dLinear -- src is unused (the @@ -1340,6 +1342,6 @@ namespace Deep /// signature consistency with To_dFn2's dispatch table. static inline void dLinearInto(float *RESTRICT dst, [[maybe_unused]] const float *RESTRICT src, size_t n) noexcept { - std::memset(dst, 1.0f, n); + std::fill(dst, dst + n, 1.0f); } } diff --git a/include/deepity/utils/DeviceMemoryArena.h b/include/deepity/utils/DeviceMemoryArena.h index b42eaaa..7e28fbd 100644 --- a/include/deepity/utils/DeviceMemoryArena.h +++ b/include/deepity/utils/DeviceMemoryArena.h @@ -1,78 +1,141 @@ #pragma once #if defined(DEEPITY_ENABLE_CUDA) -#include +#include +#include +#include /** * @file DeviceMemoryArena.h - * @brief A simple bump-pointer allocator over a single cudaMalloc'd - * device buffer, mirroring this codebase's host-side MemoryArena but for - * GPU memory. + * @brief A 64-byte-aligned bump-pointer allocator over a single device + * buffer, mirroring this codebase's host-side MemoryArena, but backed by + * IComputeBackend::Allocate()/Free() rather than raw cudaMalloc/cudaFree + * directly -- the same allocation path Tensor already uses and + * CUDAFunctionsVerify already exercises, rather than a second, separate, + * unverified call site. * - * @warning No bounds checking: AllocateFloats() never verifies the - * running offset stays within capacity_bytes, so over-allocating past the - * arena's total_floats silently hands out a pointer past the end of the - * cudaMalloc'd buffer. Callers are responsible for summing every - * required allocation up front (mirroring how MemoryArena-based layers - * report GetRequiredFloats() before BindMemory()). + * Deliberately NOT routed through IComputeBackend on the CPU side + * (MemoryArena stays exactly as it is): CPUBackend::Allocate() has no + * huge-pages parameter, so wiring MemoryArena through it would mean + * either silently losing that real, measured feature, or bolting a + * CPU-only concept onto IComputeBackend's interface for no benefit, + * since nothing ever needs to swap MemoryArena's allocator + * polymorphically at runtime. The GPU side has no such feature to lose, + * and the virtual call only happens once per arena (construction and + * destruction), not per AllocateFloats() call -- a clean win with no + * real cost. * - * @warning cudaMalloc()'s return value is not checked in the constructor. - * If the allocation fails, base_ptr is left null/uninitialized and every - * subsequent AllocateFloats() call will silently return an invalid - * pointer rather than failing loudly. - * - * @version 1.0 - * @date 2026-06-30 - * @author Jack Rose + * @warning Individual chunks handed out by AllocateFloats() cannot be + * freed independently -- the entire arena is released at once in the + * destructor, matching MemoryArena's bump-pointer/no-reclaim design. + * @version 1.1 + * @date 2026-09-05 */ -/// @brief A bump-pointer allocator over a single contiguous block of CUDA -/// device memory. Only compiled when DEEPITY_ENABLE_CUDA is defined. -class DeviceMemoryArena +namespace Deep { -private: - /// @brief Base address of the underlying cudaMalloc'd device buffer. - float *base_ptr; - /// @brief Total capacity of the arena, in bytes. - size_t capacity_bytes; - /// @brief Current allocation offset from base_ptr, in bytes. Advances - /// monotonically with each AllocateFloats() call and is never reset - /// or reclaimed. - size_t offset_bytes; - -public: - /// @brief Allocates a single device buffer large enough to hold - /// `total_floats` floats. - /// @param total_floats Total number of floats this arena can hand - /// out across all future AllocateFloats() calls combined. - /// @warning Does not check cudaMalloc()'s return value -- see - /// file-level warning. - DeviceMemoryArena(size_t total_floats) + /// @brief A 64-byte-aligned bump-pointer allocator over a single + /// contiguous block of device memory, obtained via a supplied + /// IComputeBackend (expected to be a CUDABackend in practice, though + /// this class itself doesn't hard-require that specific type). + class DeviceMemoryArena { - capacity_bytes = total_floats * sizeof(float); - cudaMalloc(&base_ptr, capacity_bytes); - offset_bytes = 0; - } + private: + /// @brief Backend used for the underlying allocation and its + /// eventual release. Non-owning -- must outlive this arena. + IComputeBackend *backend; + /// @brief Base address of the underlying backend-allocated buffer. + float *base_ptr; + /// @brief Total capacity of the arena, in bytes, rounded up to a + /// multiple of 64. + size_t capacity_bytes; + /// @brief Current allocation offset from base_ptr, in bytes. + /// Advances monotonically with each AllocateFloats() call and is + /// never reset or reclaimed. + size_t offset_bytes; - /// @brief Frees the underlying device buffer via cudaFree(). - ~DeviceMemoryArena() - { - cudaFree(base_ptr); - } + public: + /// @brief Allocates a single 64-byte-aligned device buffer large + /// enough to hold `total_floats` floats, rounded up to the + /// nearest 64-byte boundary. + /// @param backend The backend to allocate from (and later free + /// through). Must outlive this DeviceMemoryArena. Passing a + /// CPUBackend here would allocate host memory under a class + /// named "device" -- that's a caller contract, not something + /// this class checks at runtime, mirroring Tensor's own + /// backend/device pairing contract. + /// @param total_floats Total number of floats this arena can + /// hand out across all future AllocateFloats() calls combined. + /// @throws std::bad_alloc if the underlying allocation fails. + DeviceMemoryArena(IComputeBackend *backend, size_t total_floats) + : backend(backend) + { + capacity_bytes = (total_floats * sizeof(float) + 63) & ~(size_t)63; - /// @brief Hands out a chunk of `num_floats` floats from the arena via - /// simple pointer-bump allocation. - /// @param num_floats Number of floats to allocate from the arena. - /// @return Pointer to the start of the allocated chunk, valid for the - /// lifetime of this DeviceMemoryArena. - /// @warning No bounds checking against capacity_bytes -- see - /// file-level warning. Individual chunks are also never freed - /// independently; the entire arena is released at once in the - /// destructor. - float *AllocateFloats(size_t num_floats) - { - float *chunk = base_ptr + (offset_bytes / sizeof(float)); - offset_bytes += num_floats * sizeof(float); - return chunk; - } -}; + base_ptr = backend->Allocate(capacity_bytes / sizeof(float)); + if (!base_ptr) + { + throw std::bad_alloc(); + } + offset_bytes = 0; + } + + /// @brief Frees the underlying device buffer via the same + /// backend it was allocated from. + ~DeviceMemoryArena() + { + if (base_ptr) + { + backend->Free(base_ptr); + } + } + + // Delete copy/move constructors to prevent double-free corruption + // -- the original version of this class had no such guard, a real + // gap relative to MemoryArena's own established protection. + + /// @brief Deleted: DeviceMemoryArena owns a single device + /// allocation, so copying would risk a double-free. + DeviceMemoryArena(const DeviceMemoryArena &) = delete; + /// @brief Deleted: DeviceMemoryArena owns a single device + /// allocation, so copy-assignment would risk a double-free. + DeviceMemoryArena &operator=(const DeviceMemoryArena &) = delete; + + /// @brief Allocates a 64-byte aligned chunk of floats from the + /// arena. + /// @param num_floats Number of floats to allocate from the + /// arena. The actual reservation is rounded up to the nearest + /// 64-byte boundary, matching MemoryArena's own guarantee (the + /// original version of this class had no such rounding at all). + /// @return Pointer to the start of the allocated chunk, valid + /// for the lifetime of this DeviceMemoryArena. 64-byte aligned. + /// @throws std::runtime_error if the requested allocation would + /// exceed the arena's total capacity -- the original version of + /// this class had no bounds checking at all. + /// @warning Individual chunks are never freed independently; the + /// entire arena is released at once in the destructor. + float *AllocateFloats(size_t num_floats) + { + size_t allocation_size = (num_floats * sizeof(float) + 63) & ~(size_t)63; + std::cerr << " DeviceMemoryArena chunk: requested=" << num_floats + << " floats, offset=" << offset_bytes << ", size=" << allocation_size << "\n"; + + if (offset_bytes + allocation_size > capacity_bytes) + { + throw std::runtime_error("Fatal: DeviceMemoryArena capacity exceeded during allocation."); + } + + float *chunk = reinterpret_cast( + reinterpret_cast(base_ptr) + offset_bytes); + + offset_bytes += allocation_size; + return chunk; + } + + /// @brief Returns how many bytes have been allocated from the + /// arena so far. + size_t GetUsedBytes() const { return offset_bytes; } + /// @brief Returns the arena's total capacity. + size_t GetCapacityBytes() const { return capacity_bytes; } + }; +} #endif \ No newline at end of file diff --git a/logs/build.log b/logs/build.log index 9978c1c..0dda1d5 100644 --- a/logs/build.log +++ b/logs/build.log @@ -1,241 +1,11 @@ --- Deepity Build Log (Release, arch=native) --- -=== CMake Configuration === --- The C compiler identification is GNU 11.4.0 --- The CXX compiler identification is GNU 11.4.0 --- Detecting C compiler ABI info --- Detecting C compiler ABI info - done --- Check for working C compiler: /usr/bin/cc - skipped --- Detecting C compile features --- Detecting C compile features - done --- Detecting CXX compiler ABI info --- Detecting CXX compiler ABI info - done --- Check for working CXX compiler: /usr/bin/g++ - skipped --- Detecting CXX compile features --- Detecting CXX compile features - done --- CXX compiler: /usr/bin/g++ --- Found OpenMP_C: -fopenmp (found version "4.5") --- Found OpenMP_CXX: -fopenmp (found version "4.5") --- Found OpenMP: TRUE (found version "4.5") --- Looking for pthread.h --- Looking for pthread.h - found --- Performing Test CMAKE_HAVE_LIBC_PTHREAD --- Performing Test CMAKE_HAVE_LIBC_PTHREAD - Success --- Found Threads: TRUE --- CUDA support enabled. --- Found Python: /usr/bin/python3.12 (found suitable version "3.12.13", minimum required is "3.8") found components: Interpreter Development.Module --- Performing Test NB_HAS_MTLS_GNU2 --- Performing Test NB_HAS_MTLS_GNU2 - Success -CMake Warning at CMakeLists.txt:213 (message): - DEEPITY_USE_MKL was set, but MKL was not found (is oneAPI/MKL installed and - sourced? e.g. 'source /opt/intel/oneapi/setvars.sh'). Falling back to - OpenBLAS. - - --- Looking for sgemm_ --- Looking for sgemm_ - found --- Found BLAS: /usr/lib/x86_64-linux-gnu/libopenblas.so --- Configuring SLEEF 3.9.0 --- Could NOT find OpenSSL, try to set the path to OpenSSL root folder in the system variable OPENSSL_ROOT_DIR (missing: OPENSSL_CRYPTO_LIBRARY OPENSSL_INCLUDE_DIR) --- Found PkgConfig: /usr/bin/pkg-config (found version "0.29.2") --- Found OpenMP_C: -fopenmp (found version "4.5") --- Found OpenMP_CXX: -fopenmp (found version "4.5") --- Configuring build for SLEEF-v3.9.0 - Target system: Linux-6.8.0-138-generic - Target processor: x86_64 - Host system: Linux-6.8.0-138-generic - Host processor: x86_64 - Detected C compiler: GNU @ /usr/bin/cc - CMake: 3.22.1 - Make program: /usr/bin/ninja - CMake build type: Release --- Using option `-Wall -Wno-unused-function -Wno-attributes -Wno-unused-result -Wno-psabi -ffp-contract=off -fno-math-errno -fno-trapping-math` to compile libsleef --- Building shared libs : OFF --- Building static test bins: OFF --- MPFR : LIB_MPFR-NOTFOUND --- GMP : LIBGMP-NOTFOUND --- RT : /usr/lib/x86_64-linux-gnu/librt.a --- FFTW3 : LIBFFTW3-NOTFOUND --- FFTW3F : LIBFFTW3F-NOTFOUND --- OPENSSL : --- SDE : SDE_COMMAND-NOTFOUND --- COMPILER_SUPPORTS_OPENMP : 1 --- A version of SLEEF compatible with libm and libmvec in GNU libc will be produced (sleefgnuabi.so) --- Configuring done --- Generating done -CMake Warning: - Manually-specified variables were not used by the project: - - DEEPITY_BUILD_TESTS - - --- Build files have been written to: /root/Deepity/build/Release - - === Compilation === -[1/147] Building C object _deps/sleef-build/src/libm/CMakeFiles/mkmasked_gnuabi.dir/mkmasked_gnuabi.c.o -[2/147] Building C object _deps/sleef-build/src/common/CMakeFiles/common.dir/common.c.o -[3/147] Building C object _deps/sleef-build/src/common/CMakeFiles/testerutil_obj.dir/testerutil.c.o -[4/147] Building C object _deps/sleef-build/src/libm/CMakeFiles/mkrename_gnuabi.dir/mkrename_gnuabi.c.o -[5/147] Building C object _deps/sleef-build/src/libm/CMakeFiles/mkdisp.dir/mkdisp.c.o -[6/147] Linking C executable _deps/sleef-build/bin/mkmasked_gnuabi -[7/147] Generating include/masked_avx512f_sp_gnuabi.h -[8/147] Generating include/masked_avx512f_dp_gnuabi.h -[9/147] Linking C executable _deps/sleef-build/bin/mkrename_gnuabi -[10/147] Generating include/renameavx512f_gnuabi.h -Generating renameavx512f_gnuabi.h: mkrename_gnuabi avx512f e 8 16 __m512d __m512 __m256i __m512i __AVX512F__ -[11/147] Generating include/renameavx2_gnuabi.h -Generating renameavx2_gnuabi.h: mkrename_gnuabi avx2 d 4 8 __m256d __m256 __m128i __m256i __AVX2__ -[12/147] Generating include/renameavx_gnuabi.h -Generating renameavx_gnuabi.h: mkrename_gnuabi avx c 4 8 __m256d __m256 __m128i struct\ {\ __m128i\ x,\ y;\ } __AVX__ -[13/147] Generating include/renamesse2_gnuabi.h -Generating renamesse2_gnuabi.h: mkrename_gnuabi sse2 b 2 4 _mm128d _mm128 _mm128i _mm128i __SSE2__ -[14/147] Building C object _deps/sleef-build/src/libm/CMakeFiles/mkalias.dir/mkalias.c.o -[15/147] Linking C executable _deps/sleef-build/bin/mkdisp -[16/147] Building C object _deps/sleef-build/src/common/CMakeFiles/qtesterutil_obj.dir/qtesterutil.c.o -[17/147] Building C object _deps/sleef-build/src/common/CMakeFiles/addSuffix.dir/addSuffix.c.o -[18/147] Generating dispscalar.c.body -[19/147] Linking C executable _deps/sleef-build/bin/mkalias -[20/147] Generating alias_AVX512F_dp.h.tmp -[21/147] Generating alias_AVX512F_sp.h.tmp -[22/147] Generating dispscalar.c -[23/147] Generating dispsse.c.tmp -[24/147] Generating include/alias_avx512f.h -[25/147] Generating dispavx.c.tmp -[26/147] Generating dispsse.c -[27/147] Generating dispavx.c -[28/147] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefgnuabi.dir/rempitab.c.o -[29/147] Building CXX object _deps/sleef-build/src/common/CMakeFiles/psha_obj.dir/psha2_capi.cpp.o -[30/147] Linking C executable _deps/sleef-build/bin/addSuffix -[31/147] Building C object _deps/sleef-build/src/libm/CMakeFiles/mkrename.dir/mkrename.c.o -[32/147] Linking C executable _deps/sleef-build/bin/mkrename -[33/147] Generating include/renameavx512fnofma.h -Generating renameavx512fnofma.h: mkrename cinz_ 8 16 avx512fnofma -[34/147] Generating include/renameavx512f.h -Generating renameavx512f.h: mkrename finz_ 8 16 avx512f -[35/147] Generating include/renameavx2.h -Generating renameavx2.h: mkrename finz_ 4 8 avx2 -[36/147] Generating include/renameavx2128.h -Generating renameavx2128.h: mkrename finz_ 2 4 avx2128 -[37/147] Generating include/renamefma4.h -Generating renamefma4.h: mkrename finz_ 4 8 fma4 -[38/147] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetavx512fnofma.dir/sleefsimdsp.c.o -[39/147] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetavx512fnofma.dir/sleefsimddp.c.o -[40/147] Building CXX object CMakeFiles/nanobind-static.dir/usr/local/lib/python3.12/dist-packages/nanobind/src/nb_static_property.cpp.o -[41/147] Building CXX object CMakeFiles/nanobind-static.dir/usr/local/lib/python3.12/dist-packages/nanobind/src/implicit.cpp.o -[42/147] Building CXX object CMakeFiles/nanobind-static.dir/usr/local/lib/python3.12/dist-packages/nanobind/src/nb_datetime.cpp.o -[43/147] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefgnuabiavx2sp.dir/sleefsimdsp.c.o -[44/147] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefgnuabiavx2dp.dir/sleefsimddp.c.o -[45/147] Building CXX object CMakeFiles/nanobind-static.dir/usr/local/lib/python3.12/dist-packages/nanobind/src/trampoline.cpp.o -[46/147] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefgnuabiavx512fsp.dir/sleefsimdsp.c.o -[47/147] Building CXX object CMakeFiles/nanobind-static.dir/usr/local/lib/python3.12/dist-packages/nanobind/src/error.cpp.o -[48/147] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefavx512f.dir/sleefsimdsp.c.o -[49/147] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefgnuabiavx512fdp.dir/sleefsimddp.c.o -[50/147] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetavx512f.dir/sleefsimdsp.c.o -[51/147] Building CXX object CMakeFiles/nanobind-static.dir/usr/local/lib/python3.12/dist-packages/nanobind/src/common.cpp.o -[52/147] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefavx512fnofma.dir/sleefsimdsp.c.o -[53/147] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefgnuabisse2sp.dir/sleefsimdsp.c.o -[54/147] Generating include/renameavx.h -Generating renameavx.h: mkrename cinz_ 4 8 avx -[55/147] Building CXX object CMakeFiles/nanobind-static.dir/usr/local/lib/python3.12/dist-packages/nanobind/src/nb_ndarray.cpp.o -[56/147] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetavx2.dir/sleefsimdsp.c.o -[57/147] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefgnuabiavxdp.dir/sleefsimddp.c.o -[58/147] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefavx512fnofma.dir/sleefsimddp.c.o -[59/147] Generating include/renamesse4.h -Generating renamesse4.h: mkrename cinz_ 2 4 sse4 -[60/147] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetavx512f.dir/sleefsimddp.c.o -[61/147] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetavx2128.dir/sleefsimdsp.c.o -[62/147] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetavx2.dir/sleefsimddp.c.o -[63/147] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefgnuabisse2dp.dir/sleefsimddp.c.o -[64/147] Generating include/renamesse2.h -Generating renamesse2.h: mkrename cinz_ 2 4 sse2 -[65/147] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefavx512f.dir/sleefsimddp.c.o -[66/147] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefgnuabiavxsp.dir/sleefsimdsp.c.o -[67/147] Building CXX object CMakeFiles/nanobind-static.dir/usr/local/lib/python3.12/dist-packages/nanobind/src/nb_enum.cpp.o -[68/147] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetavx2128.dir/sleefsimddp.c.o -[69/147] Building CXX object CMakeFiles/nanobind-static.dir/usr/local/lib/python3.12/dist-packages/nanobind/src/nb_internals.cpp.o -[70/147] Linking C static library _deps/sleef-build/lib/libsleefgnuabi.a -[71/147] Generating include/renamepurec_scalar.h -Generating renamepurec_scalar.h: mkrename cinz_ 1 1 purec -[72/147] Generating include/renamepurecfma_scalar.h -Generating renamepurecfma_scalar.h: mkrename finz_ 1 1 purecfma -[73/147] Building CXX object CMakeFiles/nanobind-static.dir/usr/local/lib/python3.12/dist-packages/nanobind/src/nb_func.cpp.o -[74/147] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefavx2.dir/sleefsimdsp.c.o -[75/147] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefavx2128.dir/sleefsimdsp.c.o -[76/147] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefavx2.dir/sleefsimddp.c.o -[77/147] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetfma4.dir/sleefsimdsp.c.o -[78/147] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetfma4.dir/sleefsimddp.c.o -[79/147] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefavx2128.dir/sleefsimddp.c.o -[80/147] Generating include/renamecuda.h -Generating renamecuda.h: mkrename finz_ 1 1 cuda -[81/147] Building CXX object CMakeFiles/nanobind-static.dir/usr/local/lib/python3.12/dist-packages/nanobind/src/nb_type.cpp.o -[82/147] Generating sleeflibm_AVX.h.tmp -[83/147] Generating sleeflibm_AVX2.h.tmp -[84/147] Generating sleeflibm_AVX2128.h.tmp -[85/147] Generating sleeflibm_AVX512F.h.tmp -[86/147] Generating sleeflibm_AVX512FNOFMA.h.tmp -[87/147] Generating sleeflibm_AVX512F_.h.tmp -[88/147] Generating sleeflibm_AVX_.h.tmp -[89/147] Generating sleeflibm_DSP_SCALAR.h.tmp -[90/147] Generating sleeflibm_FMA4.h.tmp -[91/147] Generating sleeflibm_PURECFMA_SCALAR.h.tmp -[92/147] Generating sleeflibm_PUREC_SCALAR.h.tmp -[93/147] Linking CXX static library bin/libnanobind-static.a -[94/147] Generating sleeflibm_SSE4.h.tmp -[95/147] Generating sleeflibm_SSE2.h.tmp -[96/147] Generating sleeflibm_SSE_.h.tmp -[97/147] Generating include/renamedspscalar.h -[98/147] Generating include/renamedsp128.h -[99/147] Generating include/renamedsp256.h -[100/147] Generating ../../include/sleef.h -[101/147] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetsse4.dir/sleefsimdsp.c.o -[102/147] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetpurec_scalar.dir/sleefsimdsp.c.o -[103/147] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetavx.dir/sleefsimddp.c.o -[104/147] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetpurecfma_scalar.dir/sleefsimdsp.c.o -[105/147] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetavx.dir/sleefsimdsp.c.o -[106/147] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetsse2.dir/sleefsimdsp.c.o -[107/147] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleeffma4.dir/sleefsimdsp.c.o -[108/147] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetsse4.dir/sleefsimddp.c.o -[109/147] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleeffma4.dir/sleefsimddp.c.o -[110/147] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetpurecfma_scalar.dir/sleefsimddp.c.o -[111/147] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetpurec_scalar.dir/sleefsimddp.c.o -[112/147] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetsse2.dir/sleefsimddp.c.o -[113/147] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefsse4.dir/sleefsimdsp.c.o -[114/147] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefavx.dir/sleefsimddp.c.o -[115/147] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefsse4.dir/sleefsimddp.c.o -[116/147] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleef.dir/rempitab.c.o -[117/147] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefpurecfma_scalar.dir/sleefsimdsp.c.o -[118/147] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleef.dir/sleefld.c.o -[119/147] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefpurecfma_scalar.dir/sleefsimddp.c.o -[120/147] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleef.dir/sleefqp.c.o -[121/147] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefpurec_scalar.dir/sleefsimdsp.c.o -[122/147] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefsse2.dir/sleefsimdsp.c.o -[123/147] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefpurec_scalar.dir/sleefsimddp.c.o -[124/147] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefavx.dir/sleefsimdsp.c.o -[125/147] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefsse2.dir/sleefsimddp.c.o -[126/147] Building C object _deps/sleef-build/src/libm/CMakeFiles/dispscalar_obj.dir/dispscalar.c.o -[127/147] Building CXX object CMakeFiles/Deepity.dir/src/DirectKPPCLayer.cpp.o -[128/147] Building CXX object CMakeFiles/Deepity.dir/src/RBLayer.cpp.o -[129/147] Building CXX object CMakeFiles/Deepity.dir/src/DirectKPPCNetwork.cpp.o -[130/147] Building CXX object CMakeFiles/Deepity.dir/src/ConvPCNetwork.cpp.o -[131/147] Building CXX object CMakeFiles/Deepity.dir/src/StreamAlignedBatcher.cpp.o -[132/147] Building CXX object CMakeFiles/Deepity.dir/src/ConvPCLayer.cpp.o -[133/147] Building CXX object CMakeFiles/Deepity.dir/src/SimpleConvPCLayer.cpp.o -[134/147] Building CXX object CMakeFiles/Deepity.dir/src/GaussSeidelPCLayer.cpp.o -[135/147] Building CXX object CMakeFiles/Deepity.dir/src/DiscriminativePCLayer.cpp.o -[136/147] Building CXX object CMakeFiles/Deepity.dir/src/SimplePCLayer.cpp.o -[137/147] Building CXX object CMakeFiles/Deepity.dir/src/SimplePCNetwork.cpp.o -[138/147] Building CXX object CMakeFiles/Deepity.dir/src/GaussSeidelPCNetwork.cpp.o -[139/147] Building CXX object CMakeFiles/Deepity.dir/src/SimpleConvPCNetwork.cpp.o -[140/147] Building CXX object CMakeFiles/Deepity.dir/src/DiscriminativePCNetwork.cpp.o -[141/147] Building C object _deps/sleef-build/src/libm/CMakeFiles/dispsse_obj.dir/dispsse.c.o -[142/147] Building CXX object CMakeFiles/Deepity.dir/src/ModelIO.cpp.o -[143/147] Building C object _deps/sleef-build/src/libm/CMakeFiles/dispavx_obj.dir/dispavx.c.o -[144/147] Linking C static library _deps/sleef-build/lib/libsleef.a -[145/147] Linking CXX static library bin/libDeepity.a -[146/147] Building CXX object CMakeFiles/pydeepity.dir/bindings/pybinding.cpp.o -[147/147] Linking CXX shared module bin/pydeepity.cpython-312-x86_64-linux-gnu.so -lto-wrapper: warning: using serial compilation of 6 LTRANS jobs +[1/4] Building CXX object CMakeFiles/Deepity.dir/src/SimplePCNetwork.cpp.o +[2/4] Linking CXX static library bin/libDeepity.a +[3/4] Linking CXX executable bin/GaussSeidelVerify +[4/4] Linking CXX shared module bin/pydeepity.cpython-312-x86_64-linux-gnu.so +lto-wrapper: warning: using serial compilation of 5 LTRANS jobs === Tests === diff --git a/pydeepity/SimplePCN.py b/pydeepity/SimplePCN.py index 9a4c9e4..b23d409 100644 --- a/pydeepity/SimplePCN.py +++ b/pydeepity/SimplePCN.py @@ -19,9 +19,9 @@ class SimplePCN(dy.SimplePCNetwork): """ A Sequential Predictive Coding Network built from precision-stripped SimplePCLayers. """ - def __init__(self, batch_size: Optional[int] = None) -> None: + def __init__(self, batch_size: Optional[int] = None, device: str = "cpu") -> None: bsz = dy.auto_batch_size() if batch_size is None else batch_size - super().__init__(bsz) + super().__init__(bsz, device) def add_layer( self, @@ -44,8 +44,8 @@ def set_learning_rate(self, lr: float) -> None: def compile(self) -> None: super().compile() - def randomize_weights(self) -> None: - super().randomize_weights() + def randomize_weights(self, dist: str = "") -> None: + super().randomize_weights(dist) def train_step(self, X: npt.NDArray[np.float32], Y: npt.NDArray[np.float32], steps: int) -> float: self.reset_state() diff --git a/pydeepity/__pycache__/SimplePCN.cpython-312.pyc b/pydeepity/__pycache__/SimplePCN.cpython-312.pyc index 843336b63f51d6af292c410c98b68ddff46a4215..6872bb18af1163ec7e829ddf03f6b87cae6f596b 100644 GIT binary patch delta 1776 zcmZuxOKclO7@k@0tk;d5m(w(HV>eEmwA&_WN#D>%Xegx6rlloCa-nA98K(~G2eTVW zqKNbmL4$aS5h@1`K?qUNLqS36wFgeX0T4Z~q6l$A97u#J^#bDm*9k3%UClTD`~T;k z$BlDuJ`?*Y7F$Q)o4N7+-*Uh6 z{te)arfpDHJB);gLw%&s$T!ZjYzyyLtykwsh_U|vV$c+%X;%$;+tIaXn;>E*jI`Jw z+5{7DdYS%P%itVjuzU~UAAe%ugMGlo!t^uEXNsk& zBTfyE4&-vha?#7>T0!n^Rx7NBy;rzp_NaE_t-3Tu=t9V-B_k7W1JaM!oaY>ST1d;3 z3H7^?OlHue1Hd%_I&0r0nMARrGT{??3P|-=gB>EWRs9g|-jqNc#`jqe%MAeyjy0z~ zwb2}*sdpkn16Y`)$slz&Uny0KjsQOJ4^qO(B%Q0twyW>V{y9wM>%8cCc~q`_m;uKh zSnj*@3h~x7WI=;2LX&3cHChu=G7Y5yy_UO4+GSoT6=y{5q9{&W@LVfYt6~RO)mC3y zgSlGFpn4(F)7l8yyO+q1W^5t+t(jiT+%h-I)Y2!BuW4hjh@I%Z3t_iqNR5e}hTD_4DHOKDI0;c?*<~92qysU-(ZYzyJ<_1oMyL15uh>`hL?0r9vgsv#tBU)}qo0CZBO{x+>&ebv>P+ zW%Xrxke1Y+>DzO-W}oF90SoO5XenIb`H>0X<;HCvOP?XVCv%3!g%E4B;%oy^cPO>?FcQ^?CPp<2fXcs9(DmXuonZ$Dv6#Gq34gmNx2# z^3^GePV^6V{PxS9#9=s=EnqzgU3(s8HExa0`61Vn5^uQAf>lck17fG-c7#Dxh00aW z505_bFdi0(t&+0{!wARFLLaYG9C;G?)5x=885-#84%{f8c~kIo$SbIb<#Kt)cHO*i zoLufW)Y1WB7q8Gz?IJe;DG~57a`8IeohK`r#>^E%W4(*}t`2{2aG3ykGagLeJUMm; zS>A)UQT^Ok dO`ON<_!7%Q6y90yde?C_f6kBN*sdMhrOscQG$alEONdhv+NOjA4hFPxJ!3FzVrzCI znh+^HSfW7%q>cofP%9+Fk2s(l-~b0C#04%0^|A^`^uVPYD5#=v;=Of3g2b-o+c)oh z@4cOGX5M)AvuBKN4WmJV=h(0Byg2u%@iN)GBTLe&(%TY~@`OrxnUcJZ_EH~HnA#0Y zs!+M4T$4(Ksx>3;4@fjc{Y<4A44r8-0HcBFGzeehHhJ7e;0C<7mrsqFl5e1CFV<9R z%m;UEtr;ebv_;(X4U<;H);1`tFs-E15G1wMW2)zc__4N&kkJGiQ~r6hZ8NA0Jpa{#X*->R6ZTD0`tT z--h(MAxRtZ$3(oKjwcmYv6)rkZNPASm+f^oV46j%VB4l?W19Fz?T?4SSZ`Y_FIO%0 z#Ps}-X_i;Yj%jv+-_D3;f5vxPkBJHYo52LQcoHEce(U)Wz8t4r1qrkEcQisL+flPZ7 zxF1}Vc64J)`(97p)>AyT{d3?u(&GK$2!#1@gi-NjFai7hQ*fKawr@0MNM;QD*#JL* z;Ryg&D_I4{VJrM3x=taS7CS~HzF%t;ol^kbUZ6-kD%zW_L(N-F&17t!>2;k$x|j1f zTIUfiVD=X*OR}60b}cg6T%HS!#mj3MAHcJ))s6}oD%GyD_r)!>|hk$(VC CIA7fW diff --git a/src/SimplePCLayer.cpp b/src/SimplePCLayer.cpp index 8f8d798..b7c513a 100644 --- a/src/SimplePCLayer.cpp +++ b/src/SimplePCLayer.cpp @@ -1,26 +1,48 @@ #include +#include #include #include #include #include -#ifdef DEEPITY_USE_MKL -#include -#else -#include -#endif #include -#include #include -#include #include namespace Deep { + namespace + { + ActivationType ToDerivativeType(ActivationType fwd) + { + switch (fwd) + { + case ActivationType::RELU: + return ActivationType::dRELU; + case ActivationType::SIGMOID: + return ActivationType::dSIGMOID; + case ActivationType::eSIGMOID: + return ActivationType::d_eSIGMOID; + case ActivationType::TANH: + return ActivationType::dTANH; + case ActivationType::LINEAR: + return ActivationType::dLINEAR; + default: + return ActivationType::NONE; + } + } + + void DeleteBackend(IComputeBackend *p) { delete p; } + void NoOpDeleter(IComputeBackend *) {} + } + SimplePCLayer::SimplePCLayer(size_t size, size_t nextSize, size_t batchSize, float learningRate, float inferenceRate, float lmbda, void (*act)(float *, size_t), - void (*dAct)(float *, size_t, bool)) - : batchSize(batchSize), lr(learningRate), ir(inferenceRate), lmbda(lmbda), isClamped(false), + void (*dAct)(float *, size_t, bool), + IComputeBackend *backend) + : backend(backend ? backend : new CPUBackend(), + backend ? NoOpDeleter : DeleteBackend), + batchSize(batchSize), lr(learningRate), ir(inferenceRate), lmbda(lmbda), isClamped(false), layerAbove(nullptr), layerBelow(nullptr), activation(act), activationDerivative(dAct), activationType(To_AType(act)), opt(OptimizerType::SGD) { this->size = size; @@ -34,8 +56,11 @@ namespace Deep SimplePCLayer::SimplePCLayer(size_t size, size_t nextSize, size_t batchSize, float learningRate, float inferenceRate, float lmbda, - ActivationType aType, ActivationType dType) - : batchSize(batchSize), lr(learningRate), ir(inferenceRate), lmbda(lmbda), isClamped(false), + ActivationType aType, ActivationType dType, + IComputeBackend *backend) + : backend(backend ? backend : new CPUBackend(), + backend ? NoOpDeleter : DeleteBackend), + batchSize(batchSize), lr(learningRate), ir(inferenceRate), lmbda(lmbda), isClamped(false), layerAbove(nullptr), layerBelow(nullptr), activationType(aType) { this->activation = To_Fn(aType); @@ -49,25 +74,32 @@ namespace Deep BindMemory(*localArena); } - void SimplePCLayer::RandomizeWeights(std::mt19937 &seedGenerator) noexcept + void SimplePCLayer::RandomizeWeights(std::mt19937 &seedGenerator, const char *distribution) noexcept { - std::uniform_int_distribution seedDist; - size_t Wsz = size * nextSize; - float limit = std::sqrt(2.0f / (size + nextSize)); + char name[16] = {0}; + float a = 0.0f, b = 1.0f; - std::vector seeds(omp_get_max_threads()); - for (auto &s : seeds) - s = seedDist(seedGenerator); - -#pragma omp parallel if (!omp_in_parallel()) + if (sscanf(distribution, "%15[^(](%f,%f)", name, &a, &b) != 3) { - std::mt19937 rng(seeds[omp_get_thread_num()]); - std::normal_distribution dist(0.0f, limit); - -#pragma omp for - for (ptrdiff_t i = 0; i < (ptrdiff_t)Wsz; ++i) - W[i] = dist(rng); + // Malformed string -- fall back to this class's original, + // validated default (He/Xavier-style normal init) rather than + // silently doing nothing. + name[0] = '\0'; + strcpy(name, "normal"); + a = 0.0f; + b = std::sqrt(2.0f / (size + nextSize)); } + + std::uniform_int_distribution seedDist; + size_t Wsz = size * nextSize; + uint32_t seed = seedDist(seedGenerator); + + if (strcmp(name, "normal") == 0) + backend->RandomizeNormal(W, Wsz, a, b, seed); + else if (strcmp(name, "uniform") == 0) + backend->RandomizeUniform(W, Wsz, a, b, seed); + else + backend->RandomizeNormal(W, Wsz, 0.0f, std::sqrt(2.0f / (size + nextSize)), seed); } float SimplePCLayer::CalculateState() noexcept @@ -76,7 +108,7 @@ namespace Deep if (layerBelow == nullptr) { - std::memset(e, 0, N * sizeof(float)); + backend->Zero(e, N); if (nextSize > 0) { ComputeMuOnly(); @@ -84,65 +116,8 @@ namespace Deep return 0.0f; } - cblas_scopy(N, z, 1, e, 1); - cblas_saxpy(N, -1.0f, layerBelow->mu, 1, e, 1); + float totalEnergy = backend->ComputeErrorAndEnergy(e, z, layerBelow->mu, N); - float totalEnergy = 0.0f; - -#pragma omp parallel for schedule(static) reduction(+ : totalEnergy) if (batchSize > 4 && !omp_in_parallel()) - for (int batch = 0; batch < batchSize; ++batch) - { - const size_t offset = (size_t)batch * size; - size_t i = 0; - -#if defined(__AVX512F__) - __m512 half = _mm512_set1_ps(0.5f); - __m512 energy = _mm512_setzero_ps(); - size_t r = size % 16; - size_t simd_end = size - r; - for (; i < simd_end; i += 16) - { - __m512 e512 = _mm512_loadu_ps(&e[offset + i]); - energy = _mm512_fmadd_ps(half, _mm512_mul_ps(e512, e512), energy); - } - totalEnergy += _mm512_reduce_add_ps(energy); -#elif defined(__AVX2__) || defined(__AVX__) - __m256 half = _mm256_set1_ps(0.5f); - __m256 energy = _mm256_setzero_ps(); - size_t r = size % 8; - size_t simd_end = size - r; - for (; i < simd_end; i += 8) - { - __m256 e256 = _mm256_loadu_ps(&e[offset + i]); -#ifdef __FMA__ - energy = _mm256_fmadd_ps(half, _mm256_mul_ps(e256, e256), energy); -#else - energy = _mm256_add_ps(energy, _mm256_mul_ps(half, _mm256_mul_ps(e256, e256))); -#endif - } - totalEnergy += hsum256_ps(energy); -#elif defined(__SSE__) || defined(_M_AMD64) || defined(_M_X64) - __m128 half = _mm_set1_ps(0.5f); - __m128 energy = _mm_setzero_ps(); - size_t r = size % 4; - size_t simd_end = size - r; - for (; i < simd_end; i += 4) - { - __m128 e128 = _mm_loadu_ps(&e[offset + i]); -#ifdef __FMA__ - energy = _mm_fmadd_ps(half, _mm_mul_ps(e128, e128), energy); -#else - energy = _mm_add_ps(energy, _mm_mul_ps(half, _mm_mul_ps(e128, e128))); -#endif - } - totalEnergy += hsum128_ps(energy); -#endif - for (; i < size; ++i) - { - float err = e[offset + i]; - totalEnergy += 0.5f * err * err; - } - } if (nextSize > 0) { ComputeMuOnly(); @@ -161,47 +136,27 @@ namespace Deep if (isClamped && muCacheValid) { - cblas_scopy((int)Nout, cachedMu, 1, mu, 1); + backend->Copy(mu, cachedMu, Nout); return; } - cblas_scopy((int)N, z, 1, zF, 1); - switch (activationType) - { - case ActivationType::RELU: - Deep::relu(zF, N); - break; - case ActivationType::SIGMOID: - Deep::sigmoid(zF, N); - break; - case ActivationType::eSIGMOID: - Deep::e_sigmoid(zF, N); - break; - case ActivationType::TANH: - Deep::tanh(zF, N); - break; - case ActivationType::LINEAR: - Deep::linear(zF, N); - break; - default: - activation(zF, N); - break; // custom/unrecognized function pointer - } + backend->ActivationInto(activationType, zF, z, N); - cblas_sgemm( - CblasRowMajor, CblasNoTrans, CblasTrans, - batchSize, nextSize, size, - 1.0f, zF, size, W, size, 0.0f, mu, nextSize); + backend->MatMul( + false, true, + (int)batchSize, (int)nextSize, (int)size, + 1.0f, zF, (int)size, W, (int)size, 0.0f, mu, (int)nextSize); -#pragma omp parallel for schedule(static) if (batchSize > 4 && !omp_in_parallel()) + bool parallelOk = backend->GetDeviceType() == DeviceType::DEVICE_CPU; +#pragma omp parallel for schedule(static) if (batchSize > 4 && parallelOk && !omp_in_parallel()) for (int batch = 0; batch < batchSize; ++batch) { - cblas_saxpy(nextSize, 1.0f, b, 1, mu + batch * nextSize, 1); + backend->AxpyInto(mu + batch * nextSize, b, nextSize, 1.0f); } if (isClamped) { - cblas_scopy((int)Nout, mu, 1, cachedMu, 1); + backend->Copy(cachedMu, mu, Nout); muCacheValid = true; } } @@ -217,61 +172,21 @@ namespace Deep { const float *e_above = layerAbove->GetErrors(); - switch (activationType) - { - case ActivationType::RELU: - Deep::dReluInto(zFDeriv, z, N); - break; - case ActivationType::SIGMOID: - Deep::dSigmoidInto(zFDeriv, z, N); - break; - case ActivationType::TANH: - Deep::dTanhInto(zFDeriv, z, N); - break; - case ActivationType::LINEAR: - Deep::dLinearInto(zFDeriv, z, N); - break; - default: - activationDerivativeInto(zFDeriv, z, N); - break; // custom/unrecognized, or eSIGMOID (no dedicated derivative variant exists) - } + backend->ActivationDerivativeInto(ToDerivativeType(activationType), zFDeriv, z, N); - cblas_sgemm( - CblasRowMajor, CblasNoTrans, CblasNoTrans, - batchSize, size, nextSize, - 1.0f, e_above, nextSize, W, size, - 0.0f, feedbackScratch, size); + backend->MatMul( + /*transA=*/false, /*transB=*/false, + (int)batchSize, (int)size, (int)nextSize, + 1.0f, e_above, (int)nextSize, W, (int)size, + 0.0f, feedbackScratch, (int)size); -#pragma omp parallel for schedule(static) if (batchSize > 4 && !omp_in_parallel()) - for (int batch = 0; batch < batchSize; ++batch) - { - float *RESTRICT zPtr = z; - const float *RESTRICT feedbackPtr = feedbackScratch; - const float *RESTRICT zFDerivPtr = zFDeriv; - const float *RESTRICT ePtr = e; - size_t offset = (size_t)batch * size; - for (size_t i = 0; i < size; ++i) - { - size_t idx = offset + i; - // Fused dz_dt directly into the z update to save memory writes - zPtr[idx] += ir * ((feedbackPtr[idx] * zFDerivPtr[idx]) - ePtr[idx]); - } - } + backend->FusedStateUpdate(z, feedbackScratch, zFDeriv, e, N, ir); } else // Output Layer { -#pragma omp parallel for schedule(static) if (batchSize > 4 && !omp_in_parallel()) - for (int batch = 0; batch < batchSize; ++batch) - { - float *RESTRICT zPtr = z; - const float *RESTRICT ePtr = e; - size_t offset = (size_t)batch * size; - for (size_t i = 0; i < size; ++i) - { - size_t idx = offset + i; - zPtr[idx] += ir * -ePtr[idx]; - } - } + // z[i] += ir * (-e[i]) == z += (-ir) * e, i.e. AxpyInto + // with alpha = -ir. + backend->AxpyInto(z, e, N, -ir); } } @@ -287,17 +202,17 @@ namespace Deep case OptimizerType::SGD: { if (lmbda > 0.0f) - cblas_sscal((size_t)nextSize * size, 1.0f - lmbda, W, 1); + backend->Scale(W, (size_t)nextSize * size, 1.0f - lmbda); - cblas_sgemm( - CblasRowMajor, CblasTrans, CblasNoTrans, - nextSize, size, batchSize, - lr / batchSize, local_grad, nextSize, zF, size, - 1.0f, W, size); + backend->MatMul( + true, false, + (int)nextSize, (int)size, (int)batchSize, + lr / batchSize, local_grad, (int)nextSize, zF, (int)size, + 1.0f, W, (int)size); float lr_batch = lr / batchSize; for (int batch = 0; batch < batchSize; batch++) - cblas_saxpy(nextSize, lr_batch, local_grad + batch * nextSize, 1, b, 1); + backend->AxpyInto(b, local_grad + batch * nextSize, nextSize, lr_batch); break; } @@ -309,26 +224,26 @@ namespace Deep size_t num_weights = (size_t)nextSize * size; float grad_scale = -1.0f; - cblas_sgemm( - CblasRowMajor, CblasTrans, CblasNoTrans, - nextSize, size, batchSize, - grad_scale, local_grad, nextSize, zF, size, - 0.0f, grad_W, size); + backend->MatMul( + true, false, + (int)nextSize, (int)size, (int)batchSize, + grad_scale, local_grad, (int)nextSize, zF, (int)size, + 0.0f, grad_W, (int)size); - std::memset(grad_b, 0, nextSize * sizeof(float)); + backend->Zero(grad_b, nextSize); for (int batch = 0; batch < batchSize; batch++) - cblas_saxpy(nextSize, grad_scale, local_grad + batch * nextSize, 1, grad_b, 1); + backend->AxpyInto(grad_b, local_grad + batch * nextSize, nextSize, grad_scale); if (opt == OptimizerType::ADAMW) { - Deep::AdamWUpdate(W, grad_W, m_W, v_W, num_weights, t, lr, lmbda); + backend->AdamWStep(W, grad_W, m_W, v_W, num_weights, t, lr, lmbda); } else { - Deep::AdamUpdate(W, grad_W, m_W, v_W, num_weights, t, lr); + backend->AdamStep(W, grad_W, m_W, v_W, num_weights, t, lr); } - Deep::AdamUpdate(b, grad_b, m_b, v_b, nextSize, t, lr); + backend->AdamStep(b, grad_b, m_b, v_b, nextSize, t, lr); break; } } @@ -337,13 +252,13 @@ namespace Deep void SimplePCLayer::ResetState() noexcept { size_t N = (size_t)batchSize * size; - std::memset(z, 0, N * sizeof(float)); + backend->Zero(z, N); } void SimplePCLayer::ClampState(const std::vector &inputData) noexcept { - size_t copySize = (std::min)(inputData.size(), (size_t)(batchSize * size)) * sizeof(float); - memcpy(z, inputData.data(), copySize); + size_t copyFloats = (std::min)(inputData.size(), (size_t)(batchSize * size)); + backend->CopyFromHost(z, inputData.data(), copyFloats); isClamped = true; muCacheValid = false; } @@ -361,7 +276,6 @@ namespace Deep size_t total = 0; size_t own_state_size = (size_t)batchSize * size; - // dz_dt removed: state size drops from 3 arrays to 2 total += pad16(own_state_size) * 2; if (nextSize > 0) @@ -372,8 +286,7 @@ namespace Deep total += pad16(w_size); total += pad16(nextSize); total += pad16(out_state_size) * 2; - // E, prevZ, and bottom_up removed - total += pad16(own_state_size) * 3; // zF, zFDeriv, feedbackScratch + total += pad16(own_state_size) * 3; if (opt == OptimizerType::ADAM || opt == OptimizerType::ADAMW) { @@ -385,7 +298,8 @@ namespace Deep return total; } - void SimplePCLayer::BindMemory(MemoryArena &arena) + template + void SimplePCLayer::BindMemory(ArenaT &arena) { size_t own_state_size = (size_t)batchSize * size; size_t out_state_size = (size_t)batchSize * nextSize; @@ -393,8 +307,8 @@ namespace Deep z = arena.AllocateFloats(own_state_size); e = arena.AllocateFloats(own_state_size); - std::memset(z, 0, own_state_size * sizeof(float)); - std::memset(e, 0, own_state_size * sizeof(float)); + backend->Zero(z, own_state_size); + backend->Zero(e, own_state_size); if (nextSize > 0) { @@ -408,12 +322,12 @@ namespace Deep zFDeriv = arena.AllocateFloats(own_state_size); feedbackScratch = arena.AllocateFloats(own_state_size); - std::memset(b, 0, nextSize * sizeof(float)); - std::memset(mu, 0, out_state_size * sizeof(float)); - std::memset(cachedMu, 0, out_state_size * sizeof(float)); - std::memset(zF, 0, own_state_size * sizeof(float)); - std::memset(zFDeriv, 0, own_state_size * sizeof(float)); - std::memset(feedbackScratch, 0, own_state_size * sizeof(float)); + backend->Zero(b, nextSize); + backend->Zero(mu, out_state_size); + backend->Zero(cachedMu, out_state_size); + backend->Zero(zF, own_state_size); + backend->Zero(zFDeriv, own_state_size); + backend->Zero(feedbackScratch, own_state_size); if (opt == OptimizerType::ADAM || opt == OptimizerType::ADAMW) { @@ -425,13 +339,13 @@ namespace Deep m_b = arena.AllocateFloats(nextSize); v_b = arena.AllocateFloats(nextSize); - std::memset(m_W, 0, w_size * sizeof(float)); - std::memset(v_W, 0, w_size * sizeof(float)); - std::memset(m_b, 0, nextSize * sizeof(float)); - std::memset(v_b, 0, nextSize * sizeof(float)); + backend->Zero(m_W, w_size); + backend->Zero(v_W, w_size); + backend->Zero(m_b, nextSize); + backend->Zero(v_b, nextSize); - std::memset(grad_W, 0, w_size * sizeof(float)); - std::memset(grad_b, 0, nextSize * sizeof(float)); + backend->Zero(grad_W, w_size); + backend->Zero(grad_b, nextSize); } } @@ -440,4 +354,9 @@ namespace Deep localArena.reset(); } } -} + + template void SimplePCLayer::BindMemory(MemoryArena &arena); +#if defined(DEEPITY_ENABLE_CUDA) + template void SimplePCLayer::BindMemory(DeviceMemoryArena &arena); +#endif +} \ No newline at end of file diff --git a/src/SimplePCNetwork.cpp b/src/SimplePCNetwork.cpp index 960c620..c8af039 100644 --- a/src/SimplePCNetwork.cpp +++ b/src/SimplePCNetwork.cpp @@ -2,15 +2,22 @@ #include #include #include +#include namespace Deep { - SimplePCNetwork::SimplePCNetwork(int batchSize) noexcept : batchSize(batchSize) {} + SimplePCNetwork::SimplePCNetwork(int batchSize, DeviceType device) noexcept + : device(device), batchSize(batchSize) + { + std::cerr << "SimplePCNetwork ctor: device=" << (device == DeviceType::DEVICE_GPU ? "GPU" : "CPU") << "\n"; + backend = CreateBackend(device); + } void SimplePCNetwork::AddLayer(int size, int nextSize, float lr, float ir, float lmbda, void (*act)(float *, size_t), void (*dAct)(float *, size_t, bool)) { - std::unique_ptr l = std::make_unique(size, nextSize, batchSize, lr, ir, lmbda, act, dAct); + std::unique_ptr l = std::make_unique( + size, nextSize, batchSize, lr, ir, lmbda, act, dAct, backend.get()); if (!layers.empty()) { @@ -23,7 +30,8 @@ namespace Deep void SimplePCNetwork::AddLayer(int size, int nextSize, float lr, float ir, float lmbda, ActivationType aType, ActivationType dType) { - std::unique_ptr l = std::make_unique(size, nextSize, batchSize, lr, ir, lmbda, aType, dType); + std::unique_ptr l = std::make_unique( + size, nextSize, batchSize, lr, ir, lmbda, aType, dType, backend.get()); if (!layers.empty()) { @@ -33,6 +41,12 @@ namespace Deep layers.push_back(std::move(l)); } + void SimplePCNetwork::RandomizeWeights(std::mt19937 &rng, const char *distribution) + { + for (auto &l : layers) + l->RandomizeWeights(rng, distribution); + } + void SimplePCNetwork::RandomizeWeights(std::mt19937 &rng) { for (auto &l : layers) @@ -105,7 +119,9 @@ namespace Deep const float *beliefs = terminal->GetBeliefs(); size_t count = terminal->GetBatchSize() * terminal->GetInputSize(); - return std::vector(beliefs, beliefs + count); + std::vector result(count); + backend->CopyToHost(result.data(), beliefs, count); + return result; } void SimplePCNetwork::ProjectForward() noexcept @@ -118,7 +134,7 @@ namespace Deep float *nextZ = layers[i + 1]->GetBeliefs(); size_t n = layers[i]->GetBatchSize() * layers[i]->GetOutputSize(); - std::memcpy(nextZ, mu, n * sizeof(float)); + backend->Copy(nextZ, mu, n); } } @@ -147,7 +163,7 @@ namespace Deep { ResetState(); Clamp(x); - ProjectForward(); // Add your forward projection initialization here + ProjectForward(); for (int t = 0; t < inferenceSteps; t++) { @@ -159,7 +175,9 @@ namespace Deep const float *beliefs = terminal->GetBeliefs(); size_t count = terminal->GetBatchSize() * terminal->GetInputSize(); - return std::vector(beliefs, beliefs + count); + std::vector result(count); + backend->CopyToHost(result.data(), beliefs, count); + return result; } void SimplePCNetwork::SetMuCacheThreshold(float threshold) noexcept @@ -179,8 +197,21 @@ namespace Deep for (auto &layer : layers) total_floats_needed += layer->GetRequiredFloats(); - arena = std::make_unique(total_floats_needed, true); - for (auto &layer : layers) - layer->BindMemory(*arena); + if (device == DeviceType::DEVICE_CPU) + { + cpuArena = std::make_unique(total_floats_needed, true); + for (auto &layer : layers) + layer->BindMemory(*cpuArena); + } +#if defined(DEEPITY_ENABLE_CUDA) + else + { + std::cerr << "GPU arena: total_floats_needed=" << total_floats_needed + << " (" << total_floats_needed * sizeof(float) << " bytes)\n"; + gpuArena = std::make_unique(backend.get(), total_floats_needed); + for (auto &layer : layers) + layer->BindMemory(*gpuArena); + } +#endif } -} +} \ No newline at end of file diff --git a/src/backend/Backend.cpp b/src/backend/Backend.cpp index 96d6001..ca7dc40 100644 --- a/src/backend/Backend.cpp +++ b/src/backend/Backend.cpp @@ -2,7 +2,7 @@ #include #include -#ifdef DEEPITY_ENABLE_CUDA +#ifdef DEEPITY_USE_CUDA #include #endif @@ -16,13 +16,13 @@ namespace Deep } else if (device == DeviceType::DEVICE_GPU) { -#ifdef DEEPITY_ENABLE_CUDA +#ifdef DEEPITY_USE_CUDA return std::make_unique(); #else - throw std::runtime_error("Deepity was compiled without CUDA support (-DDEEPITY_ENABLE_CUDA=OFF). Cannot create CUDABackend."); + throw std::runtime_error("Deepity was compiled without CUDA support (-DDEEPITY_USE_CUDA=OFF). Cannot create CUDABackend."); #endif } - + throw std::invalid_argument("Unknown DeviceType requested from CreateBackend."); } } \ No newline at end of file diff --git a/src/backend/CPUBackend.cpp b/src/backend/CPUBackend.cpp index ff902cf..f1038ba 100644 --- a/src/backend/CPUBackend.cpp +++ b/src/backend/CPUBackend.cpp @@ -9,15 +9,16 @@ #else #include #endif +#include namespace Deep { - float *CPUBackend::Allocate(size_t numFloats) noexcept + float *CPUBackend::Allocate(size_t numFloats) { #ifdef _WIN32 return (float *)_aligned_alloc(numFloats * sizeof(float), 16); #else - return (float *)aligned_alloc(16, numFloats); + return (float *)aligned_alloc(16, numFloats * sizeof(float)); #endif } @@ -42,12 +43,52 @@ namespace Deep void CPUBackend::CopyFromHost(float *deviceDst, const float *hostSrc, size_t numFloats) noexcept { - memcpy(deviceDst, hostSrc, numFloats); + memcpy(deviceDst, hostSrc, numFloats * sizeof(float)); } void CPUBackend::CopyToHost(float *hostDst, const float *deviceSrc, size_t numFloats) noexcept { - memcpy(hostDst, deviceSrc, numFloats); + memcpy(hostDst, deviceSrc, numFloats * sizeof(float)); + } + + void CPUBackend::RandomizeNormal(float *buf, size_t n, float mean, float stddev, uint32_t seed) noexcept + { + std::mt19937 seedGenerator(seed); + std::uniform_int_distribution seedDist; + + std::vector seeds(omp_get_max_threads()); + for (auto &s : seeds) + s = seedDist(seedGenerator); + +#pragma omp parallel if (!omp_in_parallel()) + { + std::mt19937 rng(seeds[omp_get_thread_num()]); + std::normal_distribution dist(mean, stddev); + +#pragma omp for + for (ptrdiff_t i = 0; i < (ptrdiff_t)n; ++i) + buf[i] = dist(rng); + } + } + + void CPUBackend::RandomizeUniform(float *buf, size_t n, float min, float max, uint32_t seed) noexcept + { + std::mt19937 seedGenerator(seed); + std::uniform_int_distribution seedDist; + + std::vector seeds(omp_get_max_threads()); + for (auto &s : seeds) + s = seedDist(seedGenerator); + +#pragma omp parallel if (!omp_in_parallel()) + { + std::mt19937 rng(seeds[omp_get_thread_num()]); + std::uniform_real_distribution dist(min, max); + +#pragma omp for + for (ptrdiff_t i = 0; i < (ptrdiff_t)n; ++i) + buf[i] = dist(rng); + } } void CPUBackend::MatMul(bool transA, bool transB, int M, int N, int K, diff --git a/src/backend/CUDABackend.cu b/src/backend/CUDABackend.cu index a10c9b5..fae5603 100644 --- a/src/backend/CUDABackend.cu +++ b/src/backend/CUDABackend.cu @@ -2,6 +2,7 @@ #include #include #include +#include namespace Deep { @@ -52,14 +53,84 @@ namespace Deep cudaMemcpy(hostDst, deviceSrc, numFloats * sizeof(float), cudaMemcpyDeviceToHost); } + __global__ void normal_generation(curandState *state, float *random_numbers, + size_t n, float mean, float stddev, uint32_t seed) + { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i < n) + { + curand_init(seed, static_cast(i), 0, &state[i]); + random_numbers[i] = mean + stddev * curand_normal(&state[i]); + } + } + + __global__ void uniform_generation(curandState *state, float *random_numbers, size_t n, float min, float range, uint32_t seed) + { + int i = blockIdx.x * blockDim.x + threadIdx.x; + if (i == 0) + printf("[kernel] state ptr = %p, random_numbers ptr = %p, n = %llu\n", (void *)state, (void *)random_numbers, (unsigned long long)n); + if (i < n) + { + curand_init(seed, static_cast(i), 0, &state[i]); + random_numbers[i] = min + curand_uniform(&state[i]) * range; + } + } + + void CUDABackend::RandomizeNormal(float *buf, size_t n, float mean, float stddev, uint32_t seed) noexcept + { + if (n == 0) + return; + + curandState *state = nullptr; + if (cudaMalloc(&state, n * sizeof(curandState)) != cudaSuccess) + return; + + constexpr int BLOCK_SIZE = 256; + const int blocks = static_cast((n + BLOCK_SIZE - 1) / BLOCK_SIZE); + normal_generation<<>>(state, buf, n, mean, stddev, seed); + cudaFree(state); + } + + void CUDABackend::RandomizeUniform(float *buf, size_t n, float min, float max, uint32_t seed) noexcept + { + if (n == 0) + return; + + curandState *state = nullptr; + cudaError_t mallocErr = cudaMalloc(&state, n * sizeof(curandState)); + if (mallocErr != cudaSuccess) + { + std::cerr << "curandState cudaMalloc failed for n=" << n + << " (" << n * sizeof(curandState) << " bytes): " + << cudaGetErrorString(mallocErr) << "\n"; + return; + } + std::cerr << "curandState allocated OK: n=" << n << ", ptr=" << (void *)state << "\n"; + + constexpr int BLOCK_SIZE = 256; + const int blocks = static_cast((n + BLOCK_SIZE - 1) / BLOCK_SIZE); + uniform_generation<<>>(state, buf, n, min, max - min, seed); + + cudaError_t launchErr = cudaGetLastError(); + if (launchErr != cudaSuccess) + std::cerr << "uniform_generation kernel launch failed: " << cudaGetErrorString(launchErr) << "\n"; + + cudaDeviceSynchronize(); // force the kernel to finish and surface any async error HERE, before cudaFree + cudaError_t syncErr = cudaGetLastError(); + if (syncErr != cudaSuccess) + std::cerr << "uniform_generation kernel execution failed: " << cudaGetErrorString(syncErr) << "\n"; + + cudaFree(state); + } + void CUDABackend::MatMul(bool transA, bool transB, int M, int N, int K, float alpha, const float *A, int lda, const float *B, int ldb, float beta, float *C, int ldc) noexcept { // CUDA is *column-major*. - cublasOperation_t cuTransA = transA ? CUBLAS_OP_N : CUBLAS_OP_T; - cublasOperation_t cuTransB = transB ? CUBLAS_OP_N : CUBLAS_OP_T; + cublasOperation_t cuTransA = transA ? CUBLAS_OP_T : CUBLAS_OP_N; + cublasOperation_t cuTransB = transB ? CUBLAS_OP_T : CUBLAS_OP_N; if (cublasSgemm(handle, cuTransB, cuTransA, N, M, K, &alpha, B, ldb, A, lda, &beta, C, ldc) != CUBLAS_STATUS_SUCCESS) @@ -138,8 +209,9 @@ namespace Deep size_t i = (size_t)blockIdx.x * blockDim.x + threadIdx.x; if (i < n) { - float y = src[i]; - dst[i] = fmaf(-y, y, 1.0f); + float t; + asm("tanh.approx.f32 %0, %1;" : "=f"(t) : "f"(src[i])); + dst[i] = fmaf(-t, t, 1.0f); } } @@ -337,7 +409,7 @@ namespace Deep // Compute scalars on host CPU float beta1_t = 1.0f - Sleef_powf_u10(beta1, static_cast(t)); float beta2_t = 1.0f - Sleef_powf_u10(beta2, static_cast(t)); - float step_size = lr * Sleef_sqrtf_u05(beta2_t) / beta1_t; + float step_size = lr * Sleef_sqrtf(beta2_t) / beta1_t; constexpr int BLOCK_SIZE = 256; const int blocks = static_cast((n + BLOCK_SIZE - 1) / BLOCK_SIZE); @@ -351,7 +423,7 @@ namespace Deep { float beta1_t = 1.0f - Sleef_powf_u10(beta1, static_cast(t)); float beta2_t = 1.0f - Sleef_powf_u10(beta2, static_cast(t)); - float step_size = lr * Sleef_sqrtf_u05(beta2_t) / beta1_t; + float step_size = lr * Sleef_sqrtf(beta2_t) / beta1_t; constexpr int BLOCK_SIZE = 256; const int blocks = static_cast((n + BLOCK_SIZE - 1) / BLOCK_SIZE); diff --git a/temp.py b/temp.py new file mode 100644 index 0000000..031ccdc --- /dev/null +++ b/temp.py @@ -0,0 +1,151 @@ +import numpy as np +import os +import sys +from time import perf_counter +from pydeepity import SimplePCN + +def load_full_mnist(): + import gzip + import urllib.request + print("Fetching canonical MNIST dataset (idx-ubyte, matching ngc-learn exactly)...") + base_url = "https://storage.googleapis.com/cvdf-datasets/mnist/" + files: dict[str, str] = { + "x_train": "train-images-idx3-ubyte.gz", + "y_train": "train-labels-idx1-ubyte.gz", + "x_test": "t10k-images-idx3-ubyte.gz", + "y_test": "t10k-labels-idx1-ubyte.gz" + } + data_dir = "./data" + os.makedirs(data_dir, exist_ok=True) + paths: dict[str, str] = {} + for key, fname in files.items(): + filepath = os.path.join(data_dir, fname) + paths[key] = filepath + if not os.path.exists(filepath): + print(f"Downloading {fname}...") + urllib.request.urlretrieve(base_url + fname, filepath) + with gzip.open(paths["x_train"], 'rb') as f: + X_train_raw = np.frombuffer(f.read(), np.uint8, offset=16).reshape(-1, 784) + with gzip.open(paths["x_test"], 'rb') as f: + X_test_raw = np.frombuffer(f.read(), np.uint8, offset=16).reshape(-1, 784) + with gzip.open(paths["y_train"], 'rb') as f: + y_train_labels = np.frombuffer(f.read(), np.uint8, offset=8) + with gzip.open(paths["y_test"], 'rb') as f: + y_test_labels = np.frombuffer(f.read(), np.uint8, offset=8) + X_train = X_train_raw.astype(np.float32) / 255.0 + X_test = X_test_raw.astype(np.float32) / 255.0 + eps = 0.001 + Y_train = np.full((y_train_labels.shape[0], 10), eps, dtype=np.float32) + Y_train[np.arange(y_train_labels.shape[0]), y_train_labels] = 1.0 - eps + return X_train, Y_train, X_test, y_test_labels + +def main() -> None: + SEED = int(sys.argv[1]) if len(sys.argv) > 1 else 7 + EPOCHS = int(sys.argv[2]) if len(sys.argv) > 2 else 15 + + X_train, Y_train, X_test, y_test_labels = load_full_mnist() + + BATCH_SIZE = 250 + STEPS = 30 + LR = 0.00373 + DECAY_RATE = 0.94 + LMBDA = 0.0 # Keep at 0.0! Weight decay breaks the PCN's symmetric feedback + + print(f"\nBuilding network (784->512->512->10), seed={SEED}...") + net = SimplePCN(batch_size=BATCH_SIZE, device="gpu") + + net.add_layer(784, 512, lr=LR, ir=0.091, act="linear", lmbda=LMBDA) + net.add_layer(512, 512, lr=LR, ir=0.091, act="sigmoid", lmbda=LMBDA) + net.add_layer(512, 10, lr=LR, ir=0.091, act="sigmoid", lmbda=LMBDA) + net.add_layer(10, 0, lr=LR, ir=0.091, act="linear", lmbda=LMBDA) + + # 1. Engage decoupled AdamW + net.set_optimizer("ADAMW") + net.compile() + + # 2. Engage C++ zero-energy bypass and mu caching + # net.set_mu_cache_threshold(0) + + net.randomize_weights("uniform(-0.3,0.3)") + + print(f"\nTraining with FORWARD-PROJECTION init: {EPOCHS} epochs, {STEPS} steps, " + f"lr={LR}, decay_rate={DECAY_RATE}...\n") + print("Reference (ngc-learn, real run): 26.91, 42.96, 60.12, 75.20, 84.68, 89.52,") + print(" 91.90, 93.45, 94.30, 94.80, 95.13, 95.38, 95.63, 95.74, 95.95 -- test 95.09%\n") + + rng = np.random.default_rng(SEED) + n_batches = len(X_train) // BATCH_SIZE + start_time = perf_counter() + epoch_accs = [] + + for epoch in range(EPOCHS): + current_lr = LR * (DECAY_RATE ** epoch) + net.set_learning_rate(current_lr) + + # Standard randomized batches + indices = rng.permutation(len(X_train)) + X_shuf, Y_shuf = X_train[indices], Y_train[indices] + + correct = 0 + total = 0 + epoch_energy = 0.0 + + for b in range(n_batches): + X_batch = X_shuf[b * BATCH_SIZE:(b + 1) * BATCH_SIZE] + Y_batch = Y_shuf[b * BATCH_SIZE:(b + 1) * BATCH_SIZE] + + # 3. Call C++ Native Loop (avoid Nanobind overhead) + energy = net.train_step_with_projection(X_batch, Y_batch, STEPS) + epoch_energy += energy + + + N_ACC_BATCHES = 10 + for b in range(min(N_ACC_BATCHES, n_batches)): + X_batch = X_shuf[b * BATCH_SIZE:(b + 1) * BATCH_SIZE] + Y_batch = Y_shuf[b * BATCH_SIZE:(b + 1) * BATCH_SIZE] + + net.reset_state() + net.clamp_input(X_batch) + flat_beliefs = net.predict_with_projection(X_batch, STEPS) + terminal_beliefs = flat_beliefs.reshape(BATCH_SIZE, 10) + + pred = np.argmax(terminal_beliefs, axis=1) + true = np.argmax(Y_batch, axis=1) + correct += np.sum(pred == true) + total += BATCH_SIZE + + epoch_acc = 100.0 * correct / total + epoch_accs.append(epoch_acc) + avg_energy = epoch_energy / n_batches + elapsed = perf_counter() - start_time + print(f"Epoch {epoch+1}/{EPOCHS} | Time: {elapsed:.1f}s | Acc: {epoch_acc:.2f}% | Avg energy: {avg_energy:.4f}") + + train_time = perf_counter() - start_time + print(f"\nTraining complete in {train_time:.1f}s.") + + print("\nRunning final test evaluation (with forward-projection init)...") + correct = 0 + total = 0 + for i in range(0, len(X_test), BATCH_SIZE): + X_batch = X_test[i:i + BATCH_SIZE] + y_labels_batch = y_test_labels[i:i + BATCH_SIZE] + if len(X_batch) != BATCH_SIZE: + continue + + net.reset_state() + net.clamp_input(X_batch) + flat_beliefs = net.predict_with_projection(X_batch, STEPS) + terminal_beliefs = flat_beliefs.reshape(BATCH_SIZE, 10) + + pred_classes = np.argmax(terminal_beliefs, axis=1) + correct += np.sum(pred_classes == y_labels_batch) + total += BATCH_SIZE + + test_acc = 100.0 * correct / total + print(f"\n=== Result ===") + print(f"Deepity Peak Test Accuracy: {test_acc:.2f}% (ngc-learn: 95.09%)") + print(f"Train time: {train_time:.1f}s") + print(f"\nDeepity per-epoch: {[round(a,2) for a in epoch_accs]}") + +if __name__ == "__main__": + main() diff --git a/tests/tCUDAFunctionsVerify.cpp b/tests/tCUDAFunctionsVerify.cpp new file mode 100644 index 0000000..d10b0fa --- /dev/null +++ b/tests/tCUDAFunctionsVerify.cpp @@ -0,0 +1,237 @@ +/** + * @file tCUDAFunctionsVerify.cpp + * @brief Verifies every IComputeBackend method not already covered by + * tMatMulVerify.cpp -- activations, derivatives, Scale, AxpyInto, + * FusedStateUpdate, ComputeErrorAndEnergy, AdamStep, AdamWStep -- on + * both CPUBackend and CUDABackend, against expected values computed + * independently (via a separate Python script, not derived from this + * codebase's own formulas). + * + * IMPORTANT PRECISION NOTE: CUDABackend's tanh/sigmoid kernels use the + * hardware-approximate `tanh.approx.f32` PTX instruction, trading + * precision for speed -- CPUBackend uses SLEEF's high-precision + * (u10 = <=1.0 ULP error) implementation. These will NOT match to tight + * precision even when both are correct. Functions using this instruction + * (SIGMOID, TANH, and their derivatives) use a loose tolerance (1e-3); + * everything else (exact arithmetic: RELU, LINEAR, eSIGMOID, Scale, + * AxpyInto, FusedStateUpdate, ComputeErrorAndEnergy, Adam/AdamW) uses a + * tight one (1e-4), since a loose match there would hide a real bug. + */ +#include +#include +#include +#include +#include + +using namespace Deep; + +namespace +{ + int g_failures = 0; + + void Check(const std::vector &actual, const std::vector &expected, + const char *testName, const char *backendName, float tolerance) + { + bool ok = true; + for (size_t i = 0; i < expected.size(); ++i) + { + if (std::fabs(actual[i] - expected[i]) > tolerance) + { + printf(" [%s / %s] MISMATCH at index %zu: got %.6f, expected %.6f (tol %.1e)\n", + backendName, testName, i, actual[i], expected[i], tolerance); + ok = false; + } + } + if (ok) + printf(" [%s / %s] PASSED\n", backendName, testName); + else + g_failures++; + } + + void CheckScalar(float actual, float expected, const char *testName, + const char *backendName, float tolerance) + { + Check({actual}, {expected}, testName, backendName, tolerance); + } + + void RunAllTests(DeviceType device, const char *backendName) + { + auto backend = CreateBackend(device); + const std::vector xs = {-2.0f, -1.0f, -0.5f, 0.0f, 0.5f, 1.0f, 2.0f}; + const size_t n = xs.size(); + + // --- Activations (independently computed via Python) --------- + { + Tensor t(backend.get(), device, xs); + backend->Activation(ActivationType::RELU, t.Data(), n); + std::vector out; + t.CopyToHost(out); + Check(out, {0, 0, 0, 0, 0.5f, 1.0f, 2.0f}, "relu", backendName, 1e-4f); + } + { + Tensor t(backend.get(), device, xs); + backend->Activation(ActivationType::SIGMOID, t.Data(), n); + std::vector out; + t.CopyToHost(out); + Check(out, {0.11920292f, 0.26894142f, 0.37754067f, 0.5f, 0.62245933f, 0.73105858f, 0.88079708f}, + "sigmoid", backendName, 1e-3f); // approx tanh-based on GPU + } + { + Tensor t(backend.get(), device, xs); + backend->Activation(ActivationType::eSIGMOID, t.Data(), n); + std::vector out; + t.CopyToHost(out); + Check(out, {0.16666667f, 0.25f, 0.33333333f, 0.5f, 0.66666667f, 0.75f, 0.83333333f}, + "e_sigmoid", backendName, 1e-4f); // exact arithmetic, no approx instruction + } + { + Tensor t(backend.get(), device, xs); + backend->Activation(ActivationType::TANH, t.Data(), n); + std::vector out; + t.CopyToHost(out); + Check(out, {-0.96402758f, -0.76159416f, -0.46211716f, 0.0f, 0.46211716f, 0.76159416f, 0.96402758f}, + "tanh", backendName, 1e-3f); // approx instruction on GPU + } + { + Tensor t(backend.get(), device, xs); + backend->Activation(ActivationType::LINEAR, t.Data(), n); + std::vector out; + t.CopyToHost(out); + Check(out, {-2, -1, -0.5f, 0, 0.5f, 1, 2}, "linear (identity)", backendName, 1e-4f); + } + + // --- Derivatives, raw-input (...Into) variants ---------------- + { + Tensor src(backend.get(), device, xs); + Tensor dst(backend.get(), device, n); + backend->ActivationDerivativeInto(ActivationType::dRELU, dst.Data(), src.Data(), n); + std::vector out; + dst.CopyToHost(out); + Check(out, {0, 0, 0, 0, 1.0f, 1.0f, 1.0f}, "dRelu", backendName, 1e-4f); + } + { + Tensor src(backend.get(), device, xs); + Tensor dst(backend.get(), device, n); + backend->ActivationDerivativeInto(ActivationType::dSIGMOID, dst.Data(), src.Data(), n); + std::vector out; + dst.CopyToHost(out); + Check(out, {0.10499359f, 0.19661193f, 0.23500371f, 0.25f, 0.23500371f, 0.19661193f, 0.10499359f}, + "dSigmoid", backendName, 1e-3f); + } + { + Tensor src(backend.get(), device, xs); + Tensor dst(backend.get(), device, n); + backend->ActivationDerivativeInto(ActivationType::d_eSIGMOID, dst.Data(), src.Data(), n); + std::vector out; + dst.CopyToHost(out); + Check(out, {0.05555556f, 0.125f, 0.22222222f, 0.5f, 0.22222222f, 0.125f, 0.05555556f}, + "d_eSigmoid", backendName, 1e-4f); + } + { + Tensor src(backend.get(), device, xs); + Tensor dst(backend.get(), device, n); + backend->ActivationDerivativeInto(ActivationType::dTANH, dst.Data(), src.Data(), n); + std::vector out; + dst.CopyToHost(out); + Check(out, {0.07065082f, 0.41997434f, 0.78644773f, 1.0f, 0.78644773f, 0.41997434f, 0.07065082f}, + "dTanh", backendName, 1e-3f); + } + { + Tensor src(backend.get(), device, xs); + Tensor dst(backend.get(), device, n); + backend->ActivationDerivativeInto(ActivationType::dLINEAR, dst.Data(), src.Data(), n); + std::vector out; + dst.CopyToHost(out); + Check(out, {1, 1, 1, 1, 1, 1, 1}, "dLinear", backendName, 1e-4f); + } + + // --- Scale: [1,2,3,4] *= 2.0 ----------------------------------- + { + Tensor t(backend.get(), device, std::vector{1, 2, 3, 4}); + backend->Scale(t.Data(), 4, 2.0f); + std::vector out; + t.CopyToHost(out); + Check(out, {2, 4, 6, 8}, "Scale", backendName, 1e-4f); + } + + // --- AxpyInto: y=[1,1,1,1] += 2.0 * x=[1,2,3,4] ----------------- + { + Tensor y(backend.get(), device, std::vector{1, 1, 1, 1}); + Tensor x(backend.get(), device, std::vector{1, 2, 3, 4}); + backend->AxpyInto(y.Data(), x.Data(), 4, 2.0f); + std::vector out; + y.CopyToHost(out); + Check(out, {3, 5, 7, 9}, "AxpyInto", backendName, 1e-4f); + } + + // --- FusedStateUpdate: z += ir*(feedback*deriv - e) ------------- + { + Tensor z(backend.get(), device, std::vector{1.0f, 2.0f}); + Tensor feedback(backend.get(), device, std::vector{0.5f, 1.0f}); + Tensor deriv(backend.get(), device, std::vector{2.0f, 0.5f}); + Tensor e(backend.get(), device, std::vector{0.1f, 0.2f}); + backend->FusedStateUpdate(z.Data(), feedback.Data(), deriv.Data(), e.Data(), 2, 0.1f); + std::vector out; + z.CopyToHost(out); + Check(out, {1.09f, 2.03f}, "FusedStateUpdate", backendName, 1e-4f); + } + + // --- ComputeErrorAndEnergy: e=z-mu, returns 0.5*sum(e^2) -------- + { + Tensor z(backend.get(), device, std::vector{3.0f, 5.0f}); + Tensor mu(backend.get(), device, std::vector{1.0f, 2.0f}); + Tensor e(backend.get(), device, 2); + float energy = backend->ComputeErrorAndEnergy(e.Data(), z.Data(), mu.Data(), 2); + std::vector eOut; + e.CopyToHost(eOut); + Check(eOut, {2.0f, 3.0f}, "ComputeErrorAndEnergy (e)", backendName, 1e-4f); + CheckScalar(energy, 6.5f, "ComputeErrorAndEnergy (energy)", backendName, 1e-4f); + } + + // --- AdamStep: param=1.0, grad=0.5, m=v=0, t=1 ------------------ + { + Tensor param(backend.get(), device, std::vector{1.0f}); + Tensor grad(backend.get(), device, std::vector{0.5f}); + Tensor m(backend.get(), device, 1); + Tensor v(backend.get(), device, 1); + backend->AdamStep(param.Data(), grad.Data(), m.Data(), v.Data(), 1, 1, 0.1f); + std::vector out; + param.CopyToHost(out); + CheckScalar(out[0], 0.9000000632455132f, "AdamStep", backendName, 1e-4f); + } + + // --- AdamWStep: same, plus weightDecay=0.01 --------------------- + { + Tensor param(backend.get(), device, std::vector{1.0f}); + Tensor grad(backend.get(), device, std::vector{0.5f}); + Tensor m(backend.get(), device, 1); + Tensor v(backend.get(), device, 1); + backend->AdamWStep(param.Data(), grad.Data(), m.Data(), v.Data(), 1, 1, 0.1f, 0.01f); + std::vector out; + param.CopyToHost(out); + CheckScalar(out[0], 0.8990000632455132f, "AdamWStep", backendName, 1e-4f); + } + } +} + +int main() +{ + printf("--- CPUBackend ---\n"); + RunAllTests(DeviceType::DEVICE_CPU, "CPU"); + +#ifdef DEEPITY_USE_CUDA + printf("\n--- CUDABackend ---\n"); + RunAllTests(DeviceType::DEVICE_GPU, "CUDA"); +#else + printf("\n--- CUDABackend skipped (DEEPITY_USE_CUDA not defined) ---\n"); +#endif + + if (g_failures > 0) + { + printf("\nFAILED: %d check(s) mismatched.\n", g_failures); + return 1; + } + + printf("\nPASSED: all backend functions verified correct.\n"); + return 0; +} \ No newline at end of file From 9570137d49e0fc142292624fd80d2a2c13e89388 Mon Sep 17 00:00:00 2001 From: Ra4ster Date: Sat, 5 Sep 2026 22:17:13 -0400 Subject: [PATCH 05/11] Replaced deprecated DEEPITY flag --- include/deepity/networks/SimplePCNetwork.h | 4 ++-- include/deepity/utils/DeviceMemoryArena.h | 2 +- src/SimplePCLayer.cpp | 2 +- src/SimplePCNetwork.cpp | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/include/deepity/networks/SimplePCNetwork.h b/include/deepity/networks/SimplePCNetwork.h index 29c1eea..f6b7adb 100644 --- a/include/deepity/networks/SimplePCNetwork.h +++ b/include/deepity/networks/SimplePCNetwork.h @@ -198,9 +198,9 @@ namespace Deep /// applied when DeviceMemoryArena was designed. std::unique_ptr cpuArena; /// @brief Used when device == DEVICE_GPU. Only compiled at all - /// when DEEPITY_ENABLE_CUDA is defined, matching + /// when DEEPITY_USE_CUDA is defined, matching /// DeviceMemoryArena.h's own guard. -#if defined(DEEPITY_ENABLE_CUDA) +#if defined(DEEPITY_USE_CUDA) std::unique_ptr gpuArena; #endif diff --git a/include/deepity/utils/DeviceMemoryArena.h b/include/deepity/utils/DeviceMemoryArena.h index 7e28fbd..b8ab440 100644 --- a/include/deepity/utils/DeviceMemoryArena.h +++ b/include/deepity/utils/DeviceMemoryArena.h @@ -1,5 +1,5 @@ #pragma once -#if defined(DEEPITY_ENABLE_CUDA) +#if defined(DEEPITY_USE_CUDA) #include #include #include diff --git a/src/SimplePCLayer.cpp b/src/SimplePCLayer.cpp index b7c513a..4f9bdc6 100644 --- a/src/SimplePCLayer.cpp +++ b/src/SimplePCLayer.cpp @@ -356,7 +356,7 @@ namespace Deep } template void SimplePCLayer::BindMemory(MemoryArena &arena); -#if defined(DEEPITY_ENABLE_CUDA) +#if defined(DEEPITY_USE_CUDA) template void SimplePCLayer::BindMemory(DeviceMemoryArena &arena); #endif } \ No newline at end of file diff --git a/src/SimplePCNetwork.cpp b/src/SimplePCNetwork.cpp index c8af039..6bd1911 100644 --- a/src/SimplePCNetwork.cpp +++ b/src/SimplePCNetwork.cpp @@ -203,7 +203,7 @@ namespace Deep for (auto &layer : layers) layer->BindMemory(*cpuArena); } -#if defined(DEEPITY_ENABLE_CUDA) +#if defined(DEEPITY_USE_CUDA) else { std::cerr << "GPU arena: total_floats_needed=" << total_floats_needed From 3d5559a57dcf02ef782b96d41f59999b5c864b65 Mon Sep 17 00:00:00 2001 From: Ra4ster Date: Sun, 6 Sep 2026 05:33:31 +0000 Subject: [PATCH 06/11] Now tested: current CUDA implementation evaluates MNIST SimplePCN in 22.4s. Now moving to CUDA graphs. After all classes are ported, we'll train ImageNet. --- CMakeLists.txt | 8 +- bindings/pybinding.cpp | 2 +- include/deepity/backend/CPUBackend.h | 3 + include/deepity/backend/CUDABackend.h | 3 + include/deepity/backend/IComputeBackend.h | 17 ++ include/deepity/layers/SimplePCLayer.h | 7 +- include/deepity/networks/SimplePCNetwork.h | 5 +- include/deepity/utils/DeviceMemoryArena.h | 3 +- logs/build.log | 246 ++++++++++++++++++++- src/SimplePCLayer.cpp | 29 ++- src/SimplePCNetwork.cpp | 20 +- src/backend/CPUBackend.cpp | 14 ++ src/backend/CUDABackend.cu | 24 +- tests/tAddBiasBroadcastVerify.cpp | 83 +++++++ 14 files changed, 419 insertions(+), 45 deletions(-) create mode 100644 tests/tAddBiasBroadcastVerify.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 626a429..629d5c0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -312,12 +312,12 @@ target_compile_definitions(Deepity PUBLIC SLEEF_STATIC_LIBS) option(DEEPITY_BUILD_TESTS "Build test/verification executables" ON) -add_executable(GaussSeidelVerify tests/tGaussSeidelVerify.cpp) -target_link_libraries(GaussSeidelVerify PRIVATE Deepity) -set_target_properties(GaussSeidelVerify PROPERTIES +add_executable(AddBiasBroadcastVerify tests/tAddBiasBroadcastVerify.cpp) +target_link_libraries(AddBiasBroadcastVerify PRIVATE Deepity) +set_target_properties(AddBiasBroadcastVerify PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin ) -add_test(NAME GaussSeidelVerify COMMAND GaussSeidelVerify) +add_test(NAME AddBiasBroadcastVerify COMMAND AddBiasBroadcastVerify) # --- Compiler flags ------------------------------------------------- diff --git a/bindings/pybinding.cpp b/bindings/pybinding.cpp index 5b45f7a..d3ff0cc 100644 --- a/bindings/pybinding.cpp +++ b/bindings/pybinding.cpp @@ -142,7 +142,7 @@ namespace template void BindCommonPCLayer(nb::class_ &cls, const char *className) { - cls.def("calculate_state", &LayerT::CalculateState) + cls.def("calculate_state", static_cast(&LayerT::CalculateState)) .def("update_state", &LayerT::UpdateState) .def("update_weights", &LayerT::UpdateWeights) .def("flush", &LayerT::Flush) diff --git a/include/deepity/backend/CPUBackend.h b/include/deepity/backend/CPUBackend.h index cbebe52..1a279a7 100644 --- a/include/deepity/backend/CPUBackend.h +++ b/include/deepity/backend/CPUBackend.h @@ -26,6 +26,8 @@ namespace Deep void Scale(float *buf, size_t n, float alpha) noexcept override; void AxpyInto(float *y, const float *x, size_t n, float alpha) noexcept override; + void AddBiasBroadcast(float *buf, const float *bias, size_t batchSize, size_t width) noexcept override; + void Activation(ActivationType type, float *buf, size_t n) noexcept override; void ActivationInto(ActivationType type, float *dst, const float *src, size_t n) noexcept override; void ActivationDerivative(ActivationType type, float *buf, size_t n, bool activated) noexcept override; @@ -34,6 +36,7 @@ namespace Deep void FusedStateUpdate(float *z, const float *feedback, const float *deriv, const float *e, size_t n, float ir) noexcept override; float ComputeErrorAndEnergy(float *e, const float *z, const float *mu, size_t n) noexcept override; + void ComputeError(float *e, const float *z, const float *mu, size_t n) noexcept override; void AdamStep(float *param, const float *grad, float *m, float *v, size_t n, int t, float lr, diff --git a/include/deepity/backend/CUDABackend.h b/include/deepity/backend/CUDABackend.h index a96cf6c..2f35c4c 100644 --- a/include/deepity/backend/CUDABackend.h +++ b/include/deepity/backend/CUDABackend.h @@ -27,6 +27,8 @@ namespace Deep void Scale(float *buf, size_t n, float alpha) noexcept override; void AxpyInto(float *y, const float *x, size_t n, float alpha) noexcept override; + void AddBiasBroadcast(float *buf, const float *bias, size_t batchSize, size_t width) noexcept override; + void Activation(ActivationType type, float *buf, size_t n) noexcept override; void ActivationInto(ActivationType type, float *dst, const float *src, size_t n) noexcept override; void ActivationDerivative(ActivationType type, float *buf, size_t n, bool activated) noexcept override; @@ -35,6 +37,7 @@ namespace Deep void FusedStateUpdate(float *z, const float *feedback, const float *deriv, const float *e, size_t n, float ir) noexcept override; float ComputeErrorAndEnergy(float *e, const float *z, const float *mu, size_t n) noexcept override; + void ComputeError(float *e, const float *z, const float *mu, size_t n) noexcept override; void AdamStep(float *param, const float *grad, float *m, float *v, size_t n, int t, float lr, diff --git a/include/deepity/backend/IComputeBackend.h b/include/deepity/backend/IComputeBackend.h index 77d73f5..1b649d0 100644 --- a/include/deepity/backend/IComputeBackend.h +++ b/include/deepity/backend/IComputeBackend.h @@ -89,6 +89,14 @@ namespace Deep /// adds and Adam/AdamW's own internal accumulation today. virtual void AxpyInto(float *y, const float *x, size_t n, float alpha) noexcept = 0; + /// @brief buf[b, :] += bias[:] for every row b in [0, batchSize). Replaces + /// a batchSize-iteration loop of individual AxpyInto calls with one + /// launch -- on GPU, 250 separate kernel launches per call (each with + /// real, fixed host-side dispatch overhead regardless of how little work + /// it does) was the actual dominant cost in the settling loop, not the + /// energy-reduction syncs this was originally suspected to be. + virtual void AddBiasBroadcast(float *buf, const float *bias, size_t batchSize, size_t width) noexcept = 0; + // --- Activation ----------------------------------------------- /// @brief In-place activation, matching Deep::relu/sigmoid/etc's @@ -122,6 +130,15 @@ namespace Deep /// computation exactly (the fused AVX loop from earlier tonight). virtual float ComputeErrorAndEnergy(float *e, const float *z, const float *mu, size_t n) noexcept = 0; + /// @brief e[i] = z[i] - mu[i], for all i in [0, n). Same computation as + /// ComputeErrorAndEnergy but WITHOUT the energy reduction -- no cuBLAS + /// dot-product call, no host/device sync. Use this during a settling + /// loop's discarded intermediate iterations, where only the final + /// energy value (from ComputeErrorAndEnergy) is ever actually used; + /// every earlier call was needlessly forcing a full GPU pipeline stall + /// just to compute a number nobody reads. + virtual void ComputeError(float *e, const float *z, const float *mu, size_t n) noexcept = 0; + // --- Optimizer --------------------------------------------------- virtual void AdamStep(float *param, const float *grad, float *m, float *v, diff --git a/include/deepity/layers/SimplePCLayer.h b/include/deepity/layers/SimplePCLayer.h index 2a918b2..b388e08 100644 --- a/include/deepity/layers/SimplePCLayer.h +++ b/include/deepity/layers/SimplePCLayer.h @@ -128,8 +128,11 @@ namespace Deep /// ngc-learn's documented convention exactly (was previously /// mu = phi(W@z+b), activation AFTER the transform). /// (No precision weighting -- see file-level note.) - /// @return This layer's energy contribution at the current state. - float CalculateState() noexcept override; + /// @param needEnergy Asks for energy + /// @return This layer's energy contribution at the current state, if asked for. + float CalculateState(bool needEnergy = true) noexcept; + + float CalculateState() noexcept override { return CalculateState(true); } /// @brief Computes the state derivatives for inference. /// diff --git a/include/deepity/networks/SimplePCNetwork.h b/include/deepity/networks/SimplePCNetwork.h index f6b7adb..5e70858 100644 --- a/include/deepity/networks/SimplePCNetwork.h +++ b/include/deepity/networks/SimplePCNetwork.h @@ -86,8 +86,9 @@ namespace Deep void Clamp(const std::vector &input); /// @brief Calculates the state of each layer - /// @return Returns total energy - float CalculateState(); + /// @param needEnergy ask for energy after + /// @return Returns total energy if asked for + float CalculateState(bool needEnergy = true); /// @brief Updates each layer's state void UpdateState(); diff --git a/include/deepity/utils/DeviceMemoryArena.h b/include/deepity/utils/DeviceMemoryArena.h index b8ab440..00ee67c 100644 --- a/include/deepity/utils/DeviceMemoryArena.h +++ b/include/deepity/utils/DeviceMemoryArena.h @@ -3,6 +3,7 @@ #include #include #include +#include /** * @file DeviceMemoryArena.h @@ -116,8 +117,6 @@ namespace Deep float *AllocateFloats(size_t num_floats) { size_t allocation_size = (num_floats * sizeof(float) + 63) & ~(size_t)63; - std::cerr << " DeviceMemoryArena chunk: requested=" << num_floats - << " floats, offset=" << offset_bytes << ", size=" << allocation_size << "\n"; if (offset_bytes + allocation_size > capacity_bytes) { diff --git a/logs/build.log b/logs/build.log index 0dda1d5..2487a0d 100644 --- a/logs/build.log +++ b/logs/build.log @@ -1,11 +1,247 @@ --- Deepity Build Log (Release, arch=native) --- +=== CMake Configuration === +-- The C compiler identification is GNU 11.4.0 +-- The CXX compiler identification is GNU 11.4.0 +-- Detecting C compiler ABI info +-- Detecting C compiler ABI info - done +-- Check for working C compiler: /usr/bin/cc - skipped +-- Detecting C compile features +-- Detecting C compile features - done +-- Detecting CXX compiler ABI info +-- Detecting CXX compiler ABI info - done +-- Check for working CXX compiler: /usr/bin/g++ - skipped +-- Detecting CXX compile features +-- Detecting CXX compile features - done +-- CXX compiler: /usr/bin/g++ +-- Found OpenMP_C: -fopenmp (found version "4.5") +-- Found OpenMP_CXX: -fopenmp (found version "4.5") +-- Found OpenMP: TRUE (found version "4.5") +-- Looking for pthread.h +-- Looking for pthread.h - found +-- Performing Test CMAKE_HAVE_LIBC_PTHREAD +-- Performing Test CMAKE_HAVE_LIBC_PTHREAD - Success +-- Found Threads: TRUE +-- The CUDA compiler identification is NVIDIA 12.4.131 +-- Detecting CUDA compiler ABI info +-- Detecting CUDA compiler ABI info - done +-- Check for working CUDA compiler: /usr/local/cuda/bin/nvcc - skipped +-- Detecting CUDA compile features +-- Detecting CUDA compile features - done +-- CUDA support enabled. +-- Found Python: /usr/bin/python3.12 (found suitable version "3.12.13", minimum required is "3.8") found components: Interpreter Development.Module +-- Performing Test NB_HAS_MTLS_GNU2 +-- Performing Test NB_HAS_MTLS_GNU2 - Success +CMake Warning at CMakeLists.txt:213 (message): + DEEPITY_USE_MKL was set, but MKL was not found (is oneAPI/MKL installed and + sourced? e.g. 'source /opt/intel/oneapi/setvars.sh'). Falling back to + OpenBLAS. + + +-- Looking for sgemm_ +-- Looking for sgemm_ - found +-- Found BLAS: /usr/lib/x86_64-linux-gnu/libopenblas.so +-- Configuring SLEEF 3.9.0 +-- Could NOT find OpenSSL, try to set the path to OpenSSL root folder in the system variable OPENSSL_ROOT_DIR (missing: OPENSSL_CRYPTO_LIBRARY OPENSSL_INCLUDE_DIR) +-- Found PkgConfig: /usr/bin/pkg-config (found version "0.29.2") +-- Found OpenMP_C: -fopenmp (found version "4.5") +-- Found OpenMP_CXX: -fopenmp (found version "4.5") +-- Configuring build for SLEEF-v3.9.0 + Target system: Linux-6.8.0-138-generic + Target processor: x86_64 + Host system: Linux-6.8.0-138-generic + Host processor: x86_64 + Detected C compiler: GNU @ /usr/bin/cc + CMake: 3.22.1 + Make program: /usr/bin/ninja + CMake build type: Release +-- Using option `-Wall -Wno-unused-function -Wno-attributes -Wno-unused-result -Wno-psabi -ffp-contract=off -fno-math-errno -fno-trapping-math` to compile libsleef +-- Building shared libs : OFF +-- Building static test bins: OFF +-- MPFR : LIB_MPFR-NOTFOUND +-- GMP : LIBGMP-NOTFOUND +-- RT : /usr/lib/x86_64-linux-gnu/librt.a +-- FFTW3 : LIBFFTW3-NOTFOUND +-- FFTW3F : LIBFFTW3F-NOTFOUND +-- OPENSSL : +-- SDE : SDE_COMMAND-NOTFOUND +-- COMPILER_SUPPORTS_OPENMP : 1 +-- A version of SLEEF compatible with libm and libmvec in GNU libc will be produced (sleefgnuabi.so) +-- Configuring done +-- Generating done +-- Build files have been written to: /root/Deepity/build/Release + + === Compilation === -[1/4] Building CXX object CMakeFiles/Deepity.dir/src/SimplePCNetwork.cpp.o -[2/4] Linking CXX static library bin/libDeepity.a -[3/4] Linking CXX executable bin/GaussSeidelVerify -[4/4] Linking CXX shared module bin/pydeepity.cpython-312-x86_64-linux-gnu.so -lto-wrapper: warning: using serial compilation of 5 LTRANS jobs +[1/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/mkmasked_gnuabi.dir/mkmasked_gnuabi.c.o +[2/153] Building C object _deps/sleef-build/src/common/CMakeFiles/common.dir/common.c.o +[3/153] Linking C executable _deps/sleef-build/bin/mkmasked_gnuabi +[4/153] Generating include/masked_avx512f_sp_gnuabi.h +[5/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/mkrename_gnuabi.dir/mkrename_gnuabi.c.o +[6/153] Generating include/masked_avx512f_dp_gnuabi.h +[7/153] Linking C executable _deps/sleef-build/bin/mkrename_gnuabi +[8/153] Generating include/renameavx2_gnuabi.h +Generating renameavx2_gnuabi.h: mkrename_gnuabi avx2 d 4 8 __m256d __m256 __m128i __m256i __AVX2__ +[9/153] Generating include/renameavx512f_gnuabi.h +Generating renameavx512f_gnuabi.h: mkrename_gnuabi avx512f e 8 16 __m512d __m512 __m256i __m512i __AVX512F__ +[10/153] Generating include/renameavx_gnuabi.h +Generating renameavx_gnuabi.h: mkrename_gnuabi avx c 4 8 __m256d __m256 __m128i struct\ {\ __m128i\ x,\ y;\ } __AVX__ +[11/153] Generating include/renamesse2_gnuabi.h +Generating renamesse2_gnuabi.h: mkrename_gnuabi sse2 b 2 4 _mm128d _mm128 _mm128i _mm128i __SSE2__ +[12/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/mkalias.dir/mkalias.c.o +[13/153] Building C object _deps/sleef-build/src/common/CMakeFiles/qtesterutil_obj.dir/qtesterutil.c.o +[14/153] Building CXX object _deps/sleef-build/src/common/CMakeFiles/psha_obj.dir/psha2_capi.cpp.o +[15/153] Building C object _deps/sleef-build/src/common/CMakeFiles/testerutil_obj.dir/testerutil.c.o +[16/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/mkdisp.dir/mkdisp.c.o +[17/153] Linking C executable _deps/sleef-build/bin/mkalias +[18/153] Generating alias_AVX512F_dp.h.tmp +[19/153] Generating alias_AVX512F_sp.h.tmp +[20/153] Building C object _deps/sleef-build/src/common/CMakeFiles/addSuffix.dir/addSuffix.c.o +[21/153] Generating include/alias_avx512f.h +[22/153] Linking C executable _deps/sleef-build/bin/mkdisp +[23/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefgnuabi.dir/rempitab.c.o +[24/153] Generating dispscalar.c.body +[25/153] Generating dispsse.c.tmp +[26/153] Generating dispavx.c.tmp +[27/153] Generating dispsse.c +[28/153] Linking C executable _deps/sleef-build/bin/addSuffix +[29/153] Generating dispavx.c +[30/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/mkrename.dir/mkrename.c.o +[31/153] Generating dispscalar.c +[32/153] Linking C executable _deps/sleef-build/bin/mkrename +[33/153] Generating include/renamepurec_scalar.h +Generating renamepurec_scalar.h: mkrename cinz_ 1 1 purec +[34/153] Generating include/renameavx512fnofma.h +Generating renameavx512fnofma.h: mkrename cinz_ 8 16 avx512fnofma +[35/153] Generating include/renameavx512f.h +Generating renameavx512f.h: mkrename finz_ 8 16 avx512f +[36/153] Generating include/renameavx2.h +Generating renameavx2.h: mkrename finz_ 4 8 avx2 +[37/153] Generating include/renameavx2128.h +Generating renameavx2128.h: mkrename finz_ 2 4 avx2128 +[38/153] Generating include/renamefma4.h +Generating renamefma4.h: mkrename finz_ 4 8 fma4 +[39/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetavx512fnofma.dir/sleefsimdsp.c.o +[40/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetavx512f.dir/sleefsimddp.c.o +[41/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetavx512fnofma.dir/sleefsimddp.c.o +[42/153] Building CXX object CMakeFiles/nanobind-static.dir/usr/local/lib/python3.12/dist-packages/nanobind/src/nb_static_property.cpp.o +[43/153] Building CXX object CMakeFiles/nanobind-static.dir/usr/local/lib/python3.12/dist-packages/nanobind/src/nb_datetime.cpp.o +[44/153] Building CXX object CMakeFiles/nanobind-static.dir/usr/local/lib/python3.12/dist-packages/nanobind/src/implicit.cpp.o +[45/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefgnuabiavx2sp.dir/sleefsimdsp.c.o +[46/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefgnuabiavx512fsp.dir/sleefsimdsp.c.o +[47/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetavx512f.dir/sleefsimdsp.c.o +[48/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefgnuabiavx2dp.dir/sleefsimddp.c.o +[49/153] Building CXX object CMakeFiles/nanobind-static.dir/usr/local/lib/python3.12/dist-packages/nanobind/src/error.cpp.o +[50/153] Building CXX object CMakeFiles/nanobind-static.dir/usr/local/lib/python3.12/dist-packages/nanobind/src/trampoline.cpp.o +[51/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefavx512fnofma.dir/sleefsimdsp.c.o +[52/153] Building CXX object CMakeFiles/nanobind-static.dir/usr/local/lib/python3.12/dist-packages/nanobind/src/common.cpp.o +[53/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefgnuabisse2sp.dir/sleefsimdsp.c.o +[54/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefgnuabiavx512fdp.dir/sleefsimddp.c.o +[55/153] Generating include/renameavx.h +Generating renameavx.h: mkrename cinz_ 4 8 avx +[56/153] Building CXX object CMakeFiles/nanobind-static.dir/usr/local/lib/python3.12/dist-packages/nanobind/src/nb_ndarray.cpp.o +[57/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefgnuabiavxdp.dir/sleefsimddp.c.o +[58/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetavx2.dir/sleefsimdsp.c.o +[59/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefavx512fnofma.dir/sleefsimddp.c.o +[60/153] Generating include/renamesse4.h +Generating renamesse4.h: mkrename cinz_ 2 4 sse4 +[61/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefgnuabisse2dp.dir/sleefsimddp.c.o +[62/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetavx2128.dir/sleefsimdsp.c.o +[63/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefavx512f.dir/sleefsimdsp.c.o +[64/153] Building CXX object CMakeFiles/nanobind-static.dir/usr/local/lib/python3.12/dist-packages/nanobind/src/nb_internals.cpp.o +[65/153] Generating include/renamesse2.h +Generating renamesse2.h: mkrename cinz_ 2 4 sse2 +[66/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetavx2.dir/sleefsimddp.c.o +[67/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefgnuabiavxsp.dir/sleefsimdsp.c.o +[68/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefavx2.dir/sleefsimdsp.c.o +[69/153] Building CXX object CMakeFiles/nanobind-static.dir/usr/local/lib/python3.12/dist-packages/nanobind/src/nb_enum.cpp.o +[70/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetavx2128.dir/sleefsimddp.c.o +[71/153] Linking C static library _deps/sleef-build/lib/libsleefgnuabi.a +[72/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefavx2.dir/sleefsimddp.c.o +[73/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefavx512f.dir/sleefsimddp.c.o +[74/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetfma4.dir/sleefsimdsp.c.o +[75/153] Generating include/renamepurecfma_scalar.h +Generating renamepurecfma_scalar.h: mkrename finz_ 1 1 purecfma +[76/153] Building CXX object CMakeFiles/nanobind-static.dir/usr/local/lib/python3.12/dist-packages/nanobind/src/nb_func.cpp.o +[77/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetfma4.dir/sleefsimddp.c.o +[78/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefavx2128.dir/sleefsimdsp.c.o +[79/153] Building CXX object CMakeFiles/nanobind-static.dir/usr/local/lib/python3.12/dist-packages/nanobind/src/nb_type.cpp.o +[80/153] Linking CXX static library bin/libnanobind-static.a +[81/153] Generating include/renamecuda.h +Generating renamecuda.h: mkrename finz_ 1 1 cuda +[82/153] Generating sleeflibm_AVX.h.tmp +[83/153] Generating sleeflibm_AVX2.h.tmp +[84/153] Generating sleeflibm_AVX2128.h.tmp +[85/153] Generating sleeflibm_AVX512F.h.tmp +[86/153] Generating sleeflibm_AVX512FNOFMA.h.tmp +[87/153] Generating sleeflibm_AVX512F_.h.tmp +[88/153] Generating sleeflibm_AVX_.h.tmp +[89/153] Generating sleeflibm_DSP_SCALAR.h.tmp +[90/153] Generating sleeflibm_FMA4.h.tmp +[91/153] Generating sleeflibm_PURECFMA_SCALAR.h.tmp +[92/153] Generating sleeflibm_PUREC_SCALAR.h.tmp +[93/153] Generating sleeflibm_SSE2.h.tmp +[94/153] Generating sleeflibm_SSE4.h.tmp +[95/153] Generating sleeflibm_SSE_.h.tmp +[96/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefavx2128.dir/sleefsimddp.c.o +[97/153] Generating include/renamedspscalar.h +[98/153] Generating include/renamedsp128.h +[99/153] Generating include/renamedsp256.h +[100/153] Generating ../../include/sleef.h +[101/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetpurec_scalar.dir/sleefsimdsp.c.o +[102/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetavx.dir/sleefsimdsp.c.o +[103/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetsse4.dir/sleefsimdsp.c.o +[104/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetpurecfma_scalar.dir/sleefsimdsp.c.o +[105/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetavx.dir/sleefsimddp.c.o +[106/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetsse2.dir/sleefsimdsp.c.o +[107/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleeffma4.dir/sleefsimdsp.c.o +[108/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetsse4.dir/sleefsimddp.c.o +[109/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleeffma4.dir/sleefsimddp.c.o +[110/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetpurec_scalar.dir/sleefsimddp.c.o +[111/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetpurecfma_scalar.dir/sleefsimddp.c.o +[112/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetsse2.dir/sleefsimddp.c.o +[113/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefsse4.dir/sleefsimdsp.c.o +[114/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefavx.dir/sleefsimddp.c.o +[115/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefsse4.dir/sleefsimddp.c.o +[116/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefpurecfma_scalar.dir/sleefsimdsp.c.o +[117/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefpurecfma_scalar.dir/sleefsimddp.c.o +[118/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefsse2.dir/sleefsimdsp.c.o +[119/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefavx.dir/sleefsimdsp.c.o +[120/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefsse2.dir/sleefsimddp.c.o +[121/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefpurec_scalar.dir/sleefsimddp.c.o +[122/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleef.dir/rempitab.c.o +[123/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleef.dir/sleefld.c.o +[124/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefpurec_scalar.dir/sleefsimdsp.c.o +[125/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleef.dir/sleefqp.c.o +[126/153] Building CXX object CMakeFiles/Deepity.dir/src/RBLayer.cpp.o +[127/153] Building CXX object CMakeFiles/Deepity.dir/src/GaussSeidelPCLayer.cpp.o +[128/153] Building CXX object CMakeFiles/Deepity.dir/src/SimpleConvPCLayer.cpp.o +[129/153] Building CXX object CMakeFiles/Deepity.dir/src/StreamAlignedBatcher.cpp.o +[130/153] Building CXX object CMakeFiles/Deepity.dir/src/ConvPCLayer.cpp.o +[131/153] Building CXX object CMakeFiles/Deepity.dir/src/DirectKPPCLayer.cpp.o +[132/153] Building CXX object CMakeFiles/Deepity.dir/src/SimplePCLayer.cpp.o +[133/153] Building CXX object CMakeFiles/Deepity.dir/src/ConvPCNetwork.cpp.o +[134/153] Building CXX object CMakeFiles/Deepity.dir/src/DiscriminativePCLayer.cpp.o +[135/153] Building CXX object CMakeFiles/Deepity.dir/src/SimpleConvPCNetwork.cpp.o +[136/153] Building CXX object CMakeFiles/Deepity.dir/src/GaussSeidelPCNetwork.cpp.o +[137/153] Building CXX object CMakeFiles/Deepity.dir/src/DiscriminativePCNetwork.cpp.o +[138/153] Building CXX object CMakeFiles/Deepity.dir/src/DirectKPPCNetwork.cpp.o +[139/153] Building CXX object CMakeFiles/Deepity.dir/src/SimplePCNetwork.cpp.o +[140/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/dispscalar_obj.dir/dispscalar.c.o +[141/153] Building CXX object CMakeFiles/Deepity.dir/src/backend/CPUBackend.cpp.o +[142/153] Building CXX object CMakeFiles/Deepity.dir/src/backend/Tensor.cpp.o +[143/153] Building CXX object CMakeFiles/Deepity.dir/src/ModelIO.cpp.o +[144/153] Building CXX object CMakeFiles/AddBiasBroadcastVerify.dir/tests/tAddBiasBroadcastVerify.cpp.o +[145/153] Building CXX object CMakeFiles/Deepity.dir/src/backend/Backend.cpp.o +[146/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/dispsse_obj.dir/dispsse.c.o +[147/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/dispavx_obj.dir/dispavx.c.o +[148/153] Linking C static library _deps/sleef-build/lib/libsleef.a +[149/153] Building CUDA object CMakeFiles/Deepity.dir/src/backend/CUDABackend.cu.o +[150/153] Linking CXX static library bin/libDeepity.a +[151/153] Building CXX object CMakeFiles/pydeepity.dir/bindings/pybinding.cpp.o +[152/153] Linking CXX executable bin/AddBiasBroadcastVerify +[153/153] Linking CXX shared module bin/pydeepity.cpython-312-x86_64-linux-gnu.so +lto-wrapper: warning: using serial compilation of 6 LTRANS jobs === Tests === diff --git a/src/SimplePCLayer.cpp b/src/SimplePCLayer.cpp index 4f9bdc6..7faa975 100644 --- a/src/SimplePCLayer.cpp +++ b/src/SimplePCLayer.cpp @@ -2,11 +2,11 @@ #include #include #include -#include #include #include #include #include +#include namespace Deep { @@ -102,7 +102,7 @@ namespace Deep backend->RandomizeNormal(W, Wsz, 0.0f, std::sqrt(2.0f / (size + nextSize)), seed); } - float SimplePCLayer::CalculateState() noexcept + float SimplePCLayer::CalculateState(bool needEnergy) noexcept { const size_t N = batchSize * size; @@ -110,18 +110,18 @@ namespace Deep { backend->Zero(e, N); if (nextSize > 0) - { ComputeMuOnly(); - } return 0.0f; } - float totalEnergy = backend->ComputeErrorAndEnergy(e, z, layerBelow->mu, N); + float totalEnergy = 0.0f; + if (needEnergy) + totalEnergy = backend->ComputeErrorAndEnergy(e, z, layerBelow->mu, N); + else + backend->ComputeError(e, z, layerBelow->mu, N); if (nextSize > 0) - { ComputeMuOnly(); - } return totalEnergy; } @@ -147,12 +147,7 @@ namespace Deep (int)batchSize, (int)nextSize, (int)size, 1.0f, zF, (int)size, W, (int)size, 0.0f, mu, (int)nextSize); - bool parallelOk = backend->GetDeviceType() == DeviceType::DEVICE_CPU; -#pragma omp parallel for schedule(static) if (batchSize > 4 && parallelOk && !omp_in_parallel()) - for (int batch = 0; batch < batchSize; ++batch) - { - backend->AxpyInto(mu + batch * nextSize, b, nextSize, 1.0f); - } + backend->AddBiasBroadcast(mu, b, batchSize, nextSize); if (isClamped) { @@ -348,8 +343,12 @@ namespace Deep backend->Zero(grad_b, nextSize); } } - - if (localArena && localArena.get() != &arena) + if constexpr (std::is_same_v) + { + if (localArena && localArena.get() != &arena) + localArena.reset(); + } + else { localArena.reset(); } diff --git a/src/SimplePCNetwork.cpp b/src/SimplePCNetwork.cpp index 6bd1911..2c92849 100644 --- a/src/SimplePCNetwork.cpp +++ b/src/SimplePCNetwork.cpp @@ -2,14 +2,12 @@ #include #include #include -#include namespace Deep { SimplePCNetwork::SimplePCNetwork(int batchSize, DeviceType device) noexcept : device(device), batchSize(batchSize) { - std::cerr << "SimplePCNetwork ctor: device=" << (device == DeviceType::DEVICE_GPU ? "GPU" : "CPU") << "\n"; backend = CreateBackend(device); } @@ -64,12 +62,12 @@ namespace Deep layers.front()->ClampState(input); } - float SimplePCNetwork::CalculateState() + float SimplePCNetwork::CalculateState(bool needEnergy) { float e = 0.0f; for (size_t i = 0; i < layers.size(); i++) - e += layers[i]->CalculateState(); - return e; + e += layers[i]->CalculateState(needEnergy); + return needEnergy ? e : 0.0f; } void SimplePCNetwork::UpdateState() @@ -92,11 +90,11 @@ namespace Deep for (int t = 0; t < inferenceSteps; t++) { - CalculateState(); + CalculateState(false); UpdateState(); } - float finalEnergy = CalculateState(); + float finalEnergy = CalculateState(true); UpdateWeights(); GetTerminalLayer()->UnclampState(); @@ -148,11 +146,11 @@ namespace Deep float finalEnergy = 0.0f; for (int t = 0; t < inferenceSteps; ++t) { - CalculateState(); + CalculateState(false); UpdateState(); } - finalEnergy = CalculateState(); + finalEnergy = CalculateState(true); UpdateWeights(); GetTerminalLayer()->UnclampState(); @@ -167,7 +165,7 @@ namespace Deep for (int t = 0; t < inferenceSteps; t++) { - CalculateState(); + CalculateState(false); UpdateState(); } @@ -206,8 +204,6 @@ namespace Deep #if defined(DEEPITY_USE_CUDA) else { - std::cerr << "GPU arena: total_floats_needed=" << total_floats_needed - << " (" << total_floats_needed * sizeof(float) << " bytes)\n"; gpuArena = std::make_unique(backend.get(), total_floats_needed); for (auto &layer : layers) layer->BindMemory(*gpuArena); diff --git a/src/backend/CPUBackend.cpp b/src/backend/CPUBackend.cpp index f1038ba..d6ae6f7 100644 --- a/src/backend/CPUBackend.cpp +++ b/src/backend/CPUBackend.cpp @@ -110,6 +110,13 @@ namespace Deep cblas_saxpy(n, alpha, x, 1, y, 1); } + void CPUBackend::AddBiasBroadcast(float *buf, const float *bias, size_t batchSize, size_t width) noexcept + { +#pragma omp parallel for schedule(static) if (batchSize > 4 && !omp_in_parallel()) + for (size_t b = 0; b < batchSize; ++b) + cblas_saxpy(width, 1.0f, bias, 1, buf + b * width, 1); + } + void CPUBackend::Activation(ActivationType type, float *buf, size_t n) noexcept { To_Fn(type)(buf, n); @@ -156,6 +163,13 @@ namespace Deep return 0.5f * energy; } + void CPUBackend::ComputeError(float *e, const float *z, const float *mu, size_t n) noexcept + { +#pragma omp parallel for schedule(static) if (n > 256 && !omp_in_parallel()) + for (size_t i = 0; i < n; ++i) + e[i] = z[i] - mu[i]; + } + void CPUBackend::AdamStep(float *param, const float *grad, float *m, float *v, size_t n, int t, float lr, float beta1, float beta2, float eps) noexcept diff --git a/src/backend/CUDABackend.cu b/src/backend/CUDABackend.cu index fae5603..138e112 100644 --- a/src/backend/CUDABackend.cu +++ b/src/backend/CUDABackend.cu @@ -67,8 +67,6 @@ namespace Deep __global__ void uniform_generation(curandState *state, float *random_numbers, size_t n, float min, float range, uint32_t seed) { int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i == 0) - printf("[kernel] state ptr = %p, random_numbers ptr = %p, n = %llu\n", (void *)state, (void *)random_numbers, (unsigned long long)n); if (i < n) { curand_init(seed, static_cast(i), 0, &state[i]); @@ -149,6 +147,21 @@ namespace Deep std::cerr << "Failed to perform CUDA Axpy.\n"; } + __global__ void AddBiasBroadcastKernel(float *buf, const float *bias, size_t batchSize, size_t width) + { + size_t i = (size_t)blockIdx.x * blockDim.x + threadIdx.x; + if (i < batchSize * width) + buf[i] += bias[i % width]; + } + + void CUDABackend::AddBiasBroadcast(float *buf, const float *bias, size_t batchSize, size_t width) noexcept + { + constexpr int BLOCK_SIZE = 256; + size_t total = batchSize * width; + const int blocks = static_cast((total + BLOCK_SIZE - 1) / BLOCK_SIZE); + AddBiasBroadcastKernel<<>>(buf, bias, batchSize, width); + } + #pragma region ACTIVATIONS_AND_KERNELS __global__ void ReluKernelInto(float *dst, const float *src, size_t n) @@ -402,6 +415,13 @@ namespace Deep return 0.5f * sum_of_squares; } + void CUDABackend::ComputeError(float *e, const float *z, const float *mu, size_t n) noexcept + { + constexpr int BLOCK_SIZE = 256; + const int blocks = static_cast((n + BLOCK_SIZE - 1) / BLOCK_SIZE); + ComputeErrorKernel<<>>(e, z, mu, n); + } + void CUDABackend::AdamStep(float *param, const float *grad, float *m, float *v, size_t n, int t, float lr, float beta1, float beta2, float eps) noexcept diff --git a/tests/tAddBiasBroadcastVerify.cpp b/tests/tAddBiasBroadcastVerify.cpp new file mode 100644 index 0000000..4cac8e5 --- /dev/null +++ b/tests/tAddBiasBroadcastVerify.cpp @@ -0,0 +1,83 @@ +/** + * @file tAddBiasBroadcastVerify.cpp + * @brief Verifies IComputeBackend::AddBiasBroadcast against an + * independently, hand-computed expected result -- written specifically + * because a live MNIST run regressed from ~97% accuracy to ~10% (exactly + * chance) immediately after this function replaced a per-batch-row loop + * of individual AxpyInto calls. Something in the new implementation is + * mathematically wrong; this isolates exactly what. + */ +#include +#include +#include +#include + +using namespace Deep; + +namespace +{ + bool CheckClose(const std::vector &actual, const std::vector &expected, + const char *backendName) + { + bool ok = true; + for (size_t i = 0; i < expected.size(); ++i) + { + if (std::fabs(actual[i] - expected[i]) > 1e-4f) + { + printf(" [%s] MISMATCH at index %zu: got %.4f, expected %.4f\n", + backendName, i, actual[i], expected[i]); + ok = false; + } + } + return ok; + } + + bool RunTest(DeviceType device, const char *backendName) + { + auto backend = CreateBackend(device); + + const size_t batchSize = 3, width = 4; + std::vector hBuf = {1, 2, 3, 4, 10, 20, 30, 40, 100, 200, 300, 400}; + std::vector hBias = {0.1f, 0.2f, 0.3f, 0.4f}; + std::vector expected = {1.1f, 2.2f, 3.3f, 4.4f, + 10.1f, 20.2f, 30.3f, 40.4f, + 100.1f, 200.2f, 300.3f, 400.4f}; + + Tensor buf(backend.get(), device, hBuf); + Tensor bias(backend.get(), device, hBias); + + backend->AddBiasBroadcast(buf.Data(), bias.Data(), batchSize, width); + + std::vector out; + buf.CopyToHost(out); + + bool ok = CheckClose(out, expected, backendName); + printf(" [%s] AddBiasBroadcast: %s (got [%.1f, %.1f, %.1f, %.1f, ...])\n", + backendName, ok ? "PASSED" : "FAILED", out[0], out[1], out[2], out[3]); + return ok; + } +} + +int main() +{ + bool allPassed = true; + + printf("--- CPUBackend ---\n"); + allPassed &= RunTest(DeviceType::DEVICE_CPU, "CPU"); + +#ifdef DEEPITY_USE_CUDA + printf("\n--- CUDABackend ---\n"); + allPassed &= RunTest(DeviceType::DEVICE_GPU, "CUDA"); +#else + printf("\n--- CUDABackend skipped (DEEPITY_USE_CUDA not defined) ---\n"); +#endif + + if (!allPassed) + { + printf("\nFAILED: AddBiasBroadcast produced incorrect results on at least one backend.\n"); + return 1; + } + + printf("\nPASSED: AddBiasBroadcast correct on all tested backends.\n"); + return 0; +} \ No newline at end of file From f030627d6402cf7393e310f1b3721558b548d21d Mon Sep 17 00:00:00 2001 From: Ra4ster Date: Sun, 6 Sep 2026 06:01:13 +0000 Subject: [PATCH 07/11] Preparing for CUDA graphs. --- include/deepity/backend/CUDABackend.h | 4 + include/deepity/backend/IComputeBackend.h | 18 +- logs/build.log | 245 +--------------------- src/backend/CUDABackend.cu | 50 ++++- 4 files changed, 70 insertions(+), 247 deletions(-) diff --git a/include/deepity/backend/CUDABackend.h b/include/deepity/backend/CUDABackend.h index 2f35c4c..1bfdb8f 100644 --- a/include/deepity/backend/CUDABackend.h +++ b/include/deepity/backend/CUDABackend.h @@ -10,6 +10,9 @@ namespace Deep public: CUDABackend(); ~CUDABackend() override; + void BeginGraphCapture() noexcept override; + void EndGraphCapture() noexcept override; + void ReplayGraph() noexcept override; float *Allocate(size_t numFloats) override; void Free(float *ptr) noexcept override; @@ -50,5 +53,6 @@ namespace Deep private: cublasHandle_t handle; + cudaStream_t stream; }; } \ No newline at end of file diff --git a/include/deepity/backend/IComputeBackend.h b/include/deepity/backend/IComputeBackend.h index 1b649d0..7b905e9 100644 --- a/include/deepity/backend/IComputeBackend.h +++ b/include/deepity/backend/IComputeBackend.h @@ -36,7 +36,13 @@ namespace Deep public: virtual ~IComputeBackend() = default; - // --- Memory ----------------------------------------------------- + // Graphs + + virtual void BeginGraphCapture() noexcept = 0; + virtual void EndGraphCapture() noexcept = 0; + virtual void ReplayGraph() noexcept = 0; + + // Memory virtual float *Allocate(size_t numFloats) = 0; virtual void Free(float *ptr) noexcept = 0; @@ -71,7 +77,7 @@ namespace Deep /// @param seed Random seed virtual void RandomizeUniform(float *buf, size_t n, float min, float max, uint32_t seed) noexcept = 0; - // --- GEMM --------------------------------------------------------- + // GEMM virtual void MatMul(bool transA, bool transB, int M, int N, int K, @@ -79,7 +85,7 @@ namespace Deep const float *B, int ldb, float beta, float *C, int ldc) noexcept = 0; - // --- Elementwise scalar ops --------------------------------------- + // Elementwise scalar ops /// @brief buf *= alpha (cblas_sscal equivalent). Used for weight /// decay (W *= 1-lambda) today. @@ -97,7 +103,7 @@ namespace Deep /// energy-reduction syncs this was originally suspected to be. virtual void AddBiasBroadcast(float *buf, const float *bias, size_t batchSize, size_t width) noexcept = 0; - // --- Activation ----------------------------------------------- + // Activation /// @brief In-place activation, matching Deep::relu/sigmoid/etc's /// existing single-buffer signature. @@ -117,7 +123,7 @@ namespace Deep /// to Activations.h this session. virtual void ActivationDerivativeInto(ActivationType type, float *dst, const float *src, size_t n) noexcept = 0; - // --- Fused PC-specific ops ------- + // Fused PC-specific ops /// @brief z[i] += ir * (feedback[i] * deriv[i] - e[i]), for all i /// in [0, n). Matches SimplePCLayer/DirectKPPCLayer's UpdateState() @@ -139,7 +145,7 @@ namespace Deep /// just to compute a number nobody reads. virtual void ComputeError(float *e, const float *z, const float *mu, size_t n) noexcept = 0; - // --- Optimizer --------------------------------------------------- + // Optimizer virtual void AdamStep(float *param, const float *grad, float *m, float *v, size_t n, int t, float lr, diff --git a/logs/build.log b/logs/build.log index 2487a0d..0a034f9 100644 --- a/logs/build.log +++ b/logs/build.log @@ -1,246 +1,11 @@ --- Deepity Build Log (Release, arch=native) --- -=== CMake Configuration === --- The C compiler identification is GNU 11.4.0 --- The CXX compiler identification is GNU 11.4.0 --- Detecting C compiler ABI info --- Detecting C compiler ABI info - done --- Check for working C compiler: /usr/bin/cc - skipped --- Detecting C compile features --- Detecting C compile features - done --- Detecting CXX compiler ABI info --- Detecting CXX compiler ABI info - done --- Check for working CXX compiler: /usr/bin/g++ - skipped --- Detecting CXX compile features --- Detecting CXX compile features - done --- CXX compiler: /usr/bin/g++ --- Found OpenMP_C: -fopenmp (found version "4.5") --- Found OpenMP_CXX: -fopenmp (found version "4.5") --- Found OpenMP: TRUE (found version "4.5") --- Looking for pthread.h --- Looking for pthread.h - found --- Performing Test CMAKE_HAVE_LIBC_PTHREAD --- Performing Test CMAKE_HAVE_LIBC_PTHREAD - Success --- Found Threads: TRUE --- The CUDA compiler identification is NVIDIA 12.4.131 --- Detecting CUDA compiler ABI info --- Detecting CUDA compiler ABI info - done --- Check for working CUDA compiler: /usr/local/cuda/bin/nvcc - skipped --- Detecting CUDA compile features --- Detecting CUDA compile features - done --- CUDA support enabled. --- Found Python: /usr/bin/python3.12 (found suitable version "3.12.13", minimum required is "3.8") found components: Interpreter Development.Module --- Performing Test NB_HAS_MTLS_GNU2 --- Performing Test NB_HAS_MTLS_GNU2 - Success -CMake Warning at CMakeLists.txt:213 (message): - DEEPITY_USE_MKL was set, but MKL was not found (is oneAPI/MKL installed and - sourced? e.g. 'source /opt/intel/oneapi/setvars.sh'). Falling back to - OpenBLAS. - - --- Looking for sgemm_ --- Looking for sgemm_ - found --- Found BLAS: /usr/lib/x86_64-linux-gnu/libopenblas.so --- Configuring SLEEF 3.9.0 --- Could NOT find OpenSSL, try to set the path to OpenSSL root folder in the system variable OPENSSL_ROOT_DIR (missing: OPENSSL_CRYPTO_LIBRARY OPENSSL_INCLUDE_DIR) --- Found PkgConfig: /usr/bin/pkg-config (found version "0.29.2") --- Found OpenMP_C: -fopenmp (found version "4.5") --- Found OpenMP_CXX: -fopenmp (found version "4.5") --- Configuring build for SLEEF-v3.9.0 - Target system: Linux-6.8.0-138-generic - Target processor: x86_64 - Host system: Linux-6.8.0-138-generic - Host processor: x86_64 - Detected C compiler: GNU @ /usr/bin/cc - CMake: 3.22.1 - Make program: /usr/bin/ninja - CMake build type: Release --- Using option `-Wall -Wno-unused-function -Wno-attributes -Wno-unused-result -Wno-psabi -ffp-contract=off -fno-math-errno -fno-trapping-math` to compile libsleef --- Building shared libs : OFF --- Building static test bins: OFF --- MPFR : LIB_MPFR-NOTFOUND --- GMP : LIBGMP-NOTFOUND --- RT : /usr/lib/x86_64-linux-gnu/librt.a --- FFTW3 : LIBFFTW3-NOTFOUND --- FFTW3F : LIBFFTW3F-NOTFOUND --- OPENSSL : --- SDE : SDE_COMMAND-NOTFOUND --- COMPILER_SUPPORTS_OPENMP : 1 --- A version of SLEEF compatible with libm and libmvec in GNU libc will be produced (sleefgnuabi.so) --- Configuring done --- Generating done --- Build files have been written to: /root/Deepity/build/Release - - === Compilation === -[1/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/mkmasked_gnuabi.dir/mkmasked_gnuabi.c.o -[2/153] Building C object _deps/sleef-build/src/common/CMakeFiles/common.dir/common.c.o -[3/153] Linking C executable _deps/sleef-build/bin/mkmasked_gnuabi -[4/153] Generating include/masked_avx512f_sp_gnuabi.h -[5/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/mkrename_gnuabi.dir/mkrename_gnuabi.c.o -[6/153] Generating include/masked_avx512f_dp_gnuabi.h -[7/153] Linking C executable _deps/sleef-build/bin/mkrename_gnuabi -[8/153] Generating include/renameavx2_gnuabi.h -Generating renameavx2_gnuabi.h: mkrename_gnuabi avx2 d 4 8 __m256d __m256 __m128i __m256i __AVX2__ -[9/153] Generating include/renameavx512f_gnuabi.h -Generating renameavx512f_gnuabi.h: mkrename_gnuabi avx512f e 8 16 __m512d __m512 __m256i __m512i __AVX512F__ -[10/153] Generating include/renameavx_gnuabi.h -Generating renameavx_gnuabi.h: mkrename_gnuabi avx c 4 8 __m256d __m256 __m128i struct\ {\ __m128i\ x,\ y;\ } __AVX__ -[11/153] Generating include/renamesse2_gnuabi.h -Generating renamesse2_gnuabi.h: mkrename_gnuabi sse2 b 2 4 _mm128d _mm128 _mm128i _mm128i __SSE2__ -[12/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/mkalias.dir/mkalias.c.o -[13/153] Building C object _deps/sleef-build/src/common/CMakeFiles/qtesterutil_obj.dir/qtesterutil.c.o -[14/153] Building CXX object _deps/sleef-build/src/common/CMakeFiles/psha_obj.dir/psha2_capi.cpp.o -[15/153] Building C object _deps/sleef-build/src/common/CMakeFiles/testerutil_obj.dir/testerutil.c.o -[16/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/mkdisp.dir/mkdisp.c.o -[17/153] Linking C executable _deps/sleef-build/bin/mkalias -[18/153] Generating alias_AVX512F_dp.h.tmp -[19/153] Generating alias_AVX512F_sp.h.tmp -[20/153] Building C object _deps/sleef-build/src/common/CMakeFiles/addSuffix.dir/addSuffix.c.o -[21/153] Generating include/alias_avx512f.h -[22/153] Linking C executable _deps/sleef-build/bin/mkdisp -[23/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefgnuabi.dir/rempitab.c.o -[24/153] Generating dispscalar.c.body -[25/153] Generating dispsse.c.tmp -[26/153] Generating dispavx.c.tmp -[27/153] Generating dispsse.c -[28/153] Linking C executable _deps/sleef-build/bin/addSuffix -[29/153] Generating dispavx.c -[30/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/mkrename.dir/mkrename.c.o -[31/153] Generating dispscalar.c -[32/153] Linking C executable _deps/sleef-build/bin/mkrename -[33/153] Generating include/renamepurec_scalar.h -Generating renamepurec_scalar.h: mkrename cinz_ 1 1 purec -[34/153] Generating include/renameavx512fnofma.h -Generating renameavx512fnofma.h: mkrename cinz_ 8 16 avx512fnofma -[35/153] Generating include/renameavx512f.h -Generating renameavx512f.h: mkrename finz_ 8 16 avx512f -[36/153] Generating include/renameavx2.h -Generating renameavx2.h: mkrename finz_ 4 8 avx2 -[37/153] Generating include/renameavx2128.h -Generating renameavx2128.h: mkrename finz_ 2 4 avx2128 -[38/153] Generating include/renamefma4.h -Generating renamefma4.h: mkrename finz_ 4 8 fma4 -[39/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetavx512fnofma.dir/sleefsimdsp.c.o -[40/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetavx512f.dir/sleefsimddp.c.o -[41/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetavx512fnofma.dir/sleefsimddp.c.o -[42/153] Building CXX object CMakeFiles/nanobind-static.dir/usr/local/lib/python3.12/dist-packages/nanobind/src/nb_static_property.cpp.o -[43/153] Building CXX object CMakeFiles/nanobind-static.dir/usr/local/lib/python3.12/dist-packages/nanobind/src/nb_datetime.cpp.o -[44/153] Building CXX object CMakeFiles/nanobind-static.dir/usr/local/lib/python3.12/dist-packages/nanobind/src/implicit.cpp.o -[45/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefgnuabiavx2sp.dir/sleefsimdsp.c.o -[46/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefgnuabiavx512fsp.dir/sleefsimdsp.c.o -[47/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetavx512f.dir/sleefsimdsp.c.o -[48/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefgnuabiavx2dp.dir/sleefsimddp.c.o -[49/153] Building CXX object CMakeFiles/nanobind-static.dir/usr/local/lib/python3.12/dist-packages/nanobind/src/error.cpp.o -[50/153] Building CXX object CMakeFiles/nanobind-static.dir/usr/local/lib/python3.12/dist-packages/nanobind/src/trampoline.cpp.o -[51/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefavx512fnofma.dir/sleefsimdsp.c.o -[52/153] Building CXX object CMakeFiles/nanobind-static.dir/usr/local/lib/python3.12/dist-packages/nanobind/src/common.cpp.o -[53/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefgnuabisse2sp.dir/sleefsimdsp.c.o -[54/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefgnuabiavx512fdp.dir/sleefsimddp.c.o -[55/153] Generating include/renameavx.h -Generating renameavx.h: mkrename cinz_ 4 8 avx -[56/153] Building CXX object CMakeFiles/nanobind-static.dir/usr/local/lib/python3.12/dist-packages/nanobind/src/nb_ndarray.cpp.o -[57/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefgnuabiavxdp.dir/sleefsimddp.c.o -[58/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetavx2.dir/sleefsimdsp.c.o -[59/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefavx512fnofma.dir/sleefsimddp.c.o -[60/153] Generating include/renamesse4.h -Generating renamesse4.h: mkrename cinz_ 2 4 sse4 -[61/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefgnuabisse2dp.dir/sleefsimddp.c.o -[62/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetavx2128.dir/sleefsimdsp.c.o -[63/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefavx512f.dir/sleefsimdsp.c.o -[64/153] Building CXX object CMakeFiles/nanobind-static.dir/usr/local/lib/python3.12/dist-packages/nanobind/src/nb_internals.cpp.o -[65/153] Generating include/renamesse2.h -Generating renamesse2.h: mkrename cinz_ 2 4 sse2 -[66/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetavx2.dir/sleefsimddp.c.o -[67/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefgnuabiavxsp.dir/sleefsimdsp.c.o -[68/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefavx2.dir/sleefsimdsp.c.o -[69/153] Building CXX object CMakeFiles/nanobind-static.dir/usr/local/lib/python3.12/dist-packages/nanobind/src/nb_enum.cpp.o -[70/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetavx2128.dir/sleefsimddp.c.o -[71/153] Linking C static library _deps/sleef-build/lib/libsleefgnuabi.a -[72/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefavx2.dir/sleefsimddp.c.o -[73/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefavx512f.dir/sleefsimddp.c.o -[74/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetfma4.dir/sleefsimdsp.c.o -[75/153] Generating include/renamepurecfma_scalar.h -Generating renamepurecfma_scalar.h: mkrename finz_ 1 1 purecfma -[76/153] Building CXX object CMakeFiles/nanobind-static.dir/usr/local/lib/python3.12/dist-packages/nanobind/src/nb_func.cpp.o -[77/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetfma4.dir/sleefsimddp.c.o -[78/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefavx2128.dir/sleefsimdsp.c.o -[79/153] Building CXX object CMakeFiles/nanobind-static.dir/usr/local/lib/python3.12/dist-packages/nanobind/src/nb_type.cpp.o -[80/153] Linking CXX static library bin/libnanobind-static.a -[81/153] Generating include/renamecuda.h -Generating renamecuda.h: mkrename finz_ 1 1 cuda -[82/153] Generating sleeflibm_AVX.h.tmp -[83/153] Generating sleeflibm_AVX2.h.tmp -[84/153] Generating sleeflibm_AVX2128.h.tmp -[85/153] Generating sleeflibm_AVX512F.h.tmp -[86/153] Generating sleeflibm_AVX512FNOFMA.h.tmp -[87/153] Generating sleeflibm_AVX512F_.h.tmp -[88/153] Generating sleeflibm_AVX_.h.tmp -[89/153] Generating sleeflibm_DSP_SCALAR.h.tmp -[90/153] Generating sleeflibm_FMA4.h.tmp -[91/153] Generating sleeflibm_PURECFMA_SCALAR.h.tmp -[92/153] Generating sleeflibm_PUREC_SCALAR.h.tmp -[93/153] Generating sleeflibm_SSE2.h.tmp -[94/153] Generating sleeflibm_SSE4.h.tmp -[95/153] Generating sleeflibm_SSE_.h.tmp -[96/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefavx2128.dir/sleefsimddp.c.o -[97/153] Generating include/renamedspscalar.h -[98/153] Generating include/renamedsp128.h -[99/153] Generating include/renamedsp256.h -[100/153] Generating ../../include/sleef.h -[101/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetpurec_scalar.dir/sleefsimdsp.c.o -[102/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetavx.dir/sleefsimdsp.c.o -[103/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetsse4.dir/sleefsimdsp.c.o -[104/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetpurecfma_scalar.dir/sleefsimdsp.c.o -[105/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetavx.dir/sleefsimddp.c.o -[106/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetsse2.dir/sleefsimdsp.c.o -[107/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleeffma4.dir/sleefsimdsp.c.o -[108/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetsse4.dir/sleefsimddp.c.o -[109/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleeffma4.dir/sleefsimddp.c.o -[110/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetpurec_scalar.dir/sleefsimddp.c.o -[111/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetpurecfma_scalar.dir/sleefsimddp.c.o -[112/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetsse2.dir/sleefsimddp.c.o -[113/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefsse4.dir/sleefsimdsp.c.o -[114/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefavx.dir/sleefsimddp.c.o -[115/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefsse4.dir/sleefsimddp.c.o -[116/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefpurecfma_scalar.dir/sleefsimdsp.c.o -[117/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefpurecfma_scalar.dir/sleefsimddp.c.o -[118/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefsse2.dir/sleefsimdsp.c.o -[119/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefavx.dir/sleefsimdsp.c.o -[120/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefsse2.dir/sleefsimddp.c.o -[121/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefpurec_scalar.dir/sleefsimddp.c.o -[122/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleef.dir/rempitab.c.o -[123/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleef.dir/sleefld.c.o -[124/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefpurec_scalar.dir/sleefsimdsp.c.o -[125/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleef.dir/sleefqp.c.o -[126/153] Building CXX object CMakeFiles/Deepity.dir/src/RBLayer.cpp.o -[127/153] Building CXX object CMakeFiles/Deepity.dir/src/GaussSeidelPCLayer.cpp.o -[128/153] Building CXX object CMakeFiles/Deepity.dir/src/SimpleConvPCLayer.cpp.o -[129/153] Building CXX object CMakeFiles/Deepity.dir/src/StreamAlignedBatcher.cpp.o -[130/153] Building CXX object CMakeFiles/Deepity.dir/src/ConvPCLayer.cpp.o -[131/153] Building CXX object CMakeFiles/Deepity.dir/src/DirectKPPCLayer.cpp.o -[132/153] Building CXX object CMakeFiles/Deepity.dir/src/SimplePCLayer.cpp.o -[133/153] Building CXX object CMakeFiles/Deepity.dir/src/ConvPCNetwork.cpp.o -[134/153] Building CXX object CMakeFiles/Deepity.dir/src/DiscriminativePCLayer.cpp.o -[135/153] Building CXX object CMakeFiles/Deepity.dir/src/SimpleConvPCNetwork.cpp.o -[136/153] Building CXX object CMakeFiles/Deepity.dir/src/GaussSeidelPCNetwork.cpp.o -[137/153] Building CXX object CMakeFiles/Deepity.dir/src/DiscriminativePCNetwork.cpp.o -[138/153] Building CXX object CMakeFiles/Deepity.dir/src/DirectKPPCNetwork.cpp.o -[139/153] Building CXX object CMakeFiles/Deepity.dir/src/SimplePCNetwork.cpp.o -[140/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/dispscalar_obj.dir/dispscalar.c.o -[141/153] Building CXX object CMakeFiles/Deepity.dir/src/backend/CPUBackend.cpp.o -[142/153] Building CXX object CMakeFiles/Deepity.dir/src/backend/Tensor.cpp.o -[143/153] Building CXX object CMakeFiles/Deepity.dir/src/ModelIO.cpp.o -[144/153] Building CXX object CMakeFiles/AddBiasBroadcastVerify.dir/tests/tAddBiasBroadcastVerify.cpp.o -[145/153] Building CXX object CMakeFiles/Deepity.dir/src/backend/Backend.cpp.o -[146/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/dispsse_obj.dir/dispsse.c.o -[147/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/dispavx_obj.dir/dispavx.c.o -[148/153] Linking C static library _deps/sleef-build/lib/libsleef.a -[149/153] Building CUDA object CMakeFiles/Deepity.dir/src/backend/CUDABackend.cu.o -[150/153] Linking CXX static library bin/libDeepity.a -[151/153] Building CXX object CMakeFiles/pydeepity.dir/bindings/pybinding.cpp.o -[152/153] Linking CXX executable bin/AddBiasBroadcastVerify -[153/153] Linking CXX shared module bin/pydeepity.cpython-312-x86_64-linux-gnu.so +[1/5] Building CXX object CMakeFiles/Deepity.dir/src/backend/Backend.cpp.o +[2/5] Building CUDA object CMakeFiles/Deepity.dir/src/backend/CUDABackend.cu.o +[3/5] Linking CXX static library bin/libDeepity.a +[4/5] Linking CXX executable bin/AddBiasBroadcastVerify +[5/5] Linking CXX shared module bin/pydeepity.cpython-312-x86_64-linux-gnu.so lto-wrapper: warning: using serial compilation of 6 LTRANS jobs diff --git a/src/backend/CUDABackend.cu b/src/backend/CUDABackend.cu index 138e112..c3c487d 100644 --- a/src/backend/CUDABackend.cu +++ b/src/backend/CUDABackend.cu @@ -8,12 +8,60 @@ namespace Deep { CUDABackend::CUDABackend() { + cudaStreamCreate(&this->stream); cublasCreate(&this->handle); + cublasSetStream(this->handle, this->stream); } CUDABackend::~CUDABackend() { cublasDestroy(this->handle); + cudaStreamDestroy(this->stream); + } + + void CUDABackend::BeginGraphCapture() noexcept + { + cudaStreamBeginCapture(stream, cudaStreamCaptureModeThreadLocal); + } + + void CUDABackend::EndGraphCapture() noexcept + { + cudaGraph_t newGraph; + cudaError_t err = cudaStreamEndCapture(stream, &newGraph); + if (err != cudaSuccess) + { + std::cerr << "cudaStreamEndCapture failed: " << cudaGetErrorString(err) << "\n"; + return; + } + + if (hasGraph) + { + // Re-capturing (e.g. batch size or settling-step count changed) + // -- destroy the previous instantiated graph first. + cudaGraphExecDestroy(graphExec); + cudaGraphDestroy(graph); + } + + graph = newGraph; + err = cudaGraphInstantiate(&graphExec, graph, nullptr, nullptr, 0); + if (err != cudaSuccess) + { + std::cerr << "cudaGraphInstantiate failed: " << cudaGetErrorString(err) << "\n"; + hasGraph = false; + return; + } + + hasGraph = true; + } + + void CUDABackend::ReplayGraph() noexcept + { + if (!hasGraph) + { + std::cerr << "ReplayGraph() called before any graph was captured.\n"; + return; + } + cudaGraphLaunch(graphExec, stream); } float *CUDABackend::Allocate(size_t numFloats) @@ -113,7 +161,7 @@ namespace Deep if (launchErr != cudaSuccess) std::cerr << "uniform_generation kernel launch failed: " << cudaGetErrorString(launchErr) << "\n"; - cudaDeviceSynchronize(); // force the kernel to finish and surface any async error HERE, before cudaFree + cudaDeviceSynchronize(); // force the kernel to finish and surface any async error cudaError_t syncErr = cudaGetLastError(); if (syncErr != cudaSuccess) std::cerr << "uniform_generation kernel execution failed: " << cudaGetErrorString(syncErr) << "\n"; From 4b991eb2ec68ef791b87e5592711a378f3d2c453 Mon Sep 17 00:00:00 2001 From: Ra4ster Date: Sun, 6 Sep 2026 06:58:08 +0000 Subject: [PATCH 08/11] currently fixing sgemm pre-graph errors. --- include/deepity/backend/CPUBackend.h | 11 ++- include/deepity/backend/CUDABackend.h | 9 +- include/deepity/backend/IComputeBackend.h | 17 +++- include/deepity/layers/SimplePCLayer.h | 5 +- include/deepity/networks/SimplePCNetwork.h | 3 + src/SimplePCLayer.cpp | 26 ++++-- src/SimplePCNetwork.cpp | 43 +++++++-- src/backend/CPUBackend.cpp | 20 +++-- src/backend/CUDABackend.cu | 100 +++++++++++---------- 9 files changed, 157 insertions(+), 77 deletions(-) diff --git a/include/deepity/backend/CPUBackend.h b/include/deepity/backend/CPUBackend.h index 1a279a7..7a57e1b 100644 --- a/include/deepity/backend/CPUBackend.h +++ b/include/deepity/backend/CPUBackend.h @@ -10,6 +10,11 @@ namespace Deep CPUBackend() = default; ~CPUBackend() override = default; + // @remark these are no-ops for backend purposes + void BeginGraphCapture() noexcept override {} + void EndGraphCapture() noexcept override {} + void ReplayGraph() noexcept override {} + float *Allocate(size_t numFloats) override; void Free(float *ptr) noexcept override; void Zero(float *ptr, size_t numFloats) noexcept override; @@ -38,11 +43,13 @@ namespace Deep float ComputeErrorAndEnergy(float *e, const float *z, const float *mu, size_t n) noexcept override; void ComputeError(float *e, const float *z, const float *mu, size_t n) noexcept override; + void IncrementCounter(int *counter) noexcept override; + void AdamStep(float *param, const float *grad, float *m, float *v, - size_t n, int t, float lr, + size_t n, const int *t, const float *lr, float beta1 = 0.9f, float beta2 = 0.999f, float eps = 1e-8f) noexcept override; void AdamWStep(float *param, const float *grad, float *m, float *v, - size_t n, int t, float lr, float weightDecay, + size_t n, const int *t, const float *lr, float weightDecay, float beta1 = 0.9f, float beta2 = 0.999f, float eps = 1e-8f) noexcept override; DeviceType GetDeviceType() const noexcept override { return DeviceType::DEVICE_CPU; }; diff --git a/include/deepity/backend/CUDABackend.h b/include/deepity/backend/CUDABackend.h index 1bfdb8f..6d87b9e 100644 --- a/include/deepity/backend/CUDABackend.h +++ b/include/deepity/backend/CUDABackend.h @@ -42,11 +42,13 @@ namespace Deep float ComputeErrorAndEnergy(float *e, const float *z, const float *mu, size_t n) noexcept override; void ComputeError(float *e, const float *z, const float *mu, size_t n) noexcept override; + void IncrementCounter(int *ptr) noexcept override; + void AdamStep(float *param, const float *grad, float *m, float *v, - size_t n, int t, float lr, + size_t n, const int *t, const float *lr, float beta1 = 0.9f, float beta2 = 0.999f, float eps = 1e-8f) noexcept override; void AdamWStep(float *param, const float *grad, float *m, float *v, - size_t n, int t, float lr, float weightDecay, + size_t n, const int *t, const float *lr, float weightDecay, float beta1 = 0.9f, float beta2 = 0.999f, float eps = 1e-8f) noexcept override; DeviceType GetDeviceType() const noexcept override { return DeviceType::DEVICE_GPU; } @@ -54,5 +56,8 @@ namespace Deep private: cublasHandle_t handle; cudaStream_t stream; + cudaGraph_t graph = nullptr; + cudaGraphExec_t graphExec = nullptr; + bool hasGraph = false; }; } \ No newline at end of file diff --git a/include/deepity/backend/IComputeBackend.h b/include/deepity/backend/IComputeBackend.h index 7b905e9..6363cdc 100644 --- a/include/deepity/backend/IComputeBackend.h +++ b/include/deepity/backend/IComputeBackend.h @@ -147,11 +147,24 @@ namespace Deep // Optimizer + /// @brief Increments *counter by 1, on-device (a single, dedicated + /// kernel launch on GPU; a plain ++ on CPU). Exists so Adam/AdamW's + /// timestep can live entirely inside a captured CUDA graph -- see + /// AdamStep's own note for why t and lr moved from by-value host + /// arguments to device-resident pointers. + virtual void IncrementCounter(int *counter) noexcept = 0; + + /// @brief t and lr are now device-resident pointers, not host values + /// taken by copy -- a graph captures the ARGUMENTS baked in at capture + /// time, so a host int/float would freeze t and lr at whatever they + /// were during the one-time capture call, silently never advancing on + /// any later replay. Reading them from device memory each call lets + /// the kernel itself see up-to-date values every replay. virtual void AdamStep(float *param, const float *grad, float *m, float *v, - size_t n, int t, float lr, + size_t n, const int *t, const float *lr, float beta1 = 0.9f, float beta2 = 0.999f, float eps = 1e-8f) noexcept = 0; virtual void AdamWStep(float *param, const float *grad, float *m, float *v, - size_t n, int t, float lr, float weightDecay, + size_t n, const int *t, const float *lr, float weightDecay, float beta1 = 0.9f, float beta2 = 0.999f, float eps = 1e-8f) noexcept = 0; virtual DeviceType GetDeviceType() const noexcept = 0; diff --git a/include/deepity/layers/SimplePCLayer.h b/include/deepity/layers/SimplePCLayer.h index b388e08..aad0943 100644 --- a/include/deepity/layers/SimplePCLayer.h +++ b/include/deepity/layers/SimplePCLayer.h @@ -199,7 +199,7 @@ namespace Deep /// @brief Sets the learning rate used for weight updates. /// @param lr The new learning rate. - void SetLearningRate(float lr) noexcept { this->lr = lr; } + void SetLearningRate(float lr) noexcept; /// @brief Sets the inference rate (Euler integration step size). /// @param ir The new inference rate. void SetInferenceRate(float ir) noexcept { this->ir = ir; } @@ -330,6 +330,9 @@ namespace Deep float lmbda; bool isClamped = false; + int *t_device = nullptr; + float *lr_device = nullptr; + float muCacheThreshold = -1.0f; // -1 = disabled. 0 = exact clamped-only // (today's validated behavior). >0 = // approximate, extends to unclamped diff --git a/include/deepity/networks/SimplePCNetwork.h b/include/deepity/networks/SimplePCNetwork.h index 5e70858..f26aa2e 100644 --- a/include/deepity/networks/SimplePCNetwork.h +++ b/include/deepity/networks/SimplePCNetwork.h @@ -191,6 +191,9 @@ namespace Deep /// and Compile() needs to know which arena type to construct. DeviceType device; + bool graphCaptured = false; + int capturedInferenceSteps = -1; + /// @brief Used when device == DEVICE_CPU. Only one of /// cpuArena/gpuArena is ever actually constructed for a given /// network -- they're kept as two separate members (rather than diff --git a/src/SimplePCLayer.cpp b/src/SimplePCLayer.cpp index 7faa975..0d6f462 100644 --- a/src/SimplePCLayer.cpp +++ b/src/SimplePCLayer.cpp @@ -214,7 +214,7 @@ namespace Deep case OptimizerType::ADAM: case OptimizerType::ADAMW: { - t++; + backend->IncrementCounter(t_device); // was: t++ size_t num_weights = (size_t)nextSize * size; float grad_scale = -1.0f; @@ -230,15 +230,11 @@ namespace Deep backend->AxpyInto(grad_b, local_grad + batch * nextSize, nextSize, grad_scale); if (opt == OptimizerType::ADAMW) - { - backend->AdamWStep(W, grad_W, m_W, v_W, num_weights, t, lr, lmbda); - } + backend->AdamWStep(W, grad_W, m_W, v_W, num_weights, t_device, lr_device, lmbda); else - { - backend->AdamStep(W, grad_W, m_W, v_W, num_weights, t, lr); - } + backend->AdamStep(W, grad_W, m_W, v_W, num_weights, t_device, lr_device); - backend->AdamStep(b, grad_b, m_b, v_b, nextSize, t, lr); + backend->AdamStep(b, grad_b, m_b, v_b, nextSize, t_device, lr_device); break; } } @@ -287,12 +283,19 @@ namespace Deep { total += pad16(w_size) * 3; total += pad16(nextSize) * 3; + total += pad16(1) * 2; // t_device, lr_device } } return total; } + void SimplePCLayer::SetLearningRate(float lr) noexcept + { + this->lr = lr; + backend->CopyFromHost(lr_device, &this->lr, 1); + } + template void SimplePCLayer::BindMemory(ArenaT &arena) { @@ -341,6 +344,13 @@ namespace Deep backend->Zero(grad_W, w_size); backend->Zero(grad_b, nextSize); + + t_device = reinterpret_cast(arena.AllocateFloats(1)); + lr_device = arena.AllocateFloats(1); + + int zero = 0; + backend->CopyFromHost(reinterpret_cast(t_device), reinterpret_cast(&zero), 1); + backend->CopyFromHost(lr_device, &lr, 1); } } if constexpr (std::is_same_v) diff --git a/src/SimplePCNetwork.cpp b/src/SimplePCNetwork.cpp index 2c92849..b0474b5 100644 --- a/src/SimplePCNetwork.cpp +++ b/src/SimplePCNetwork.cpp @@ -2,6 +2,7 @@ #include #include #include +#include namespace Deep { @@ -143,15 +144,45 @@ namespace Deep ProjectForward(); GetTerminalLayer()->ClampState(y); - float finalEnergy = 0.0f; - for (int t = 0; t < inferenceSteps; ++t) + if (device == DeviceType::DEVICE_GPU) { - CalculateState(false); - UpdateState(); + if (!graphCaptured || capturedInferenceSteps != inferenceSteps) + { + // Warm-up pass, uncaptured: runs the exact same operation + // sequence once so any lazy cuBLAS internal workspace + // allocation (a documented cuBLAS + CUDA Graphs interaction) + // happens before capture begins, not during it. + for (int t = 0; t < inferenceSteps; ++t) + { + CalculateState(false); + UpdateState(); + } + UpdateWeights(); + std::cerr << "=== Warm-up pass complete, beginning capture ===\n"; + backend->BeginGraphCapture(); + for (int t = 0; t < inferenceSteps; ++t) + { + CalculateState(false); + UpdateState(); + } + UpdateWeights(); + backend->EndGraphCapture(); + graphCaptured = true; + capturedInferenceSteps = inferenceSteps; + } + backend->ReplayGraph(); + } + else + { + for (int t = 0; t < inferenceSteps; ++t) + { + CalculateState(false); + UpdateState(); + } + UpdateWeights(); } - finalEnergy = CalculateState(true); - UpdateWeights(); + float finalEnergy = CalculateState(true); GetTerminalLayer()->UnclampState(); return finalEnergy; diff --git a/src/backend/CPUBackend.cpp b/src/backend/CPUBackend.cpp index d6ae6f7..f0f3cfb 100644 --- a/src/backend/CPUBackend.cpp +++ b/src/backend/CPUBackend.cpp @@ -170,14 +170,16 @@ namespace Deep e[i] = z[i] - mu[i]; } + void CPUBackend::IncrementCounter(int *counter) noexcept { ++(*counter); } + void CPUBackend::AdamStep(float *param, const float *grad, float *m, float *v, - size_t n, int t, float lr, + size_t n, const int *t, const float *lr, float beta1, float beta2, float eps) noexcept { // Precompute bias correction - float beta1_t = 1.0f - Sleef_powf_u10(beta1, static_cast(t)); - float beta2_t = 1.0f - Sleef_powf_u10(beta2, static_cast(t)); - float step_size = lr * Sleef_sqrtf(beta2_t) / beta1_t; + float beta1_t = 1.0f - Sleef_powf_u10(beta1, static_cast(*t)); + float beta2_t = 1.0f - Sleef_powf_u10(beta2, static_cast(*t)); + float step_size = *lr * Sleef_sqrtf(beta2_t) / beta1_t; #pragma omp parallel for schedule(static) if (n > 256 && !omp_in_parallel()) for (size_t i = 0; i < n; ++i) @@ -191,12 +193,12 @@ namespace Deep } void CPUBackend::AdamWStep(float *param, const float *grad, float *m, float *v, - size_t n, int t, float lr, float weightDecay, + size_t n, const int *t, const float *lr, float weightDecay, float beta1, float beta2, float eps) noexcept { - float beta1_t = 1.0f - Sleef_powf_u10(beta1, static_cast(t)); - float beta2_t = 1.0f - Sleef_powf_u10(beta2, static_cast(t)); - float step_size = lr * Sleef_sqrtf(beta2_t) / beta1_t; + float beta1_t = 1.0f - Sleef_powf_u10(beta1, static_cast(*t)); + float beta2_t = 1.0f - Sleef_powf_u10(beta2, static_cast(*t)); + float step_size = *lr * Sleef_sqrtf(beta2_t) / beta1_t; #pragma omp parallel for schedule(static) if (n > 256 && !omp_in_parallel()) for (size_t i = 0; i < n; ++i) @@ -205,7 +207,7 @@ namespace Deep m[i] = beta1 * m[i] + (1.0f - beta1) * g; v[i] = beta2 * v[i] + (1.0f - beta2) * (g * g); - param[i] -= lr * weightDecay * param[i]; + param[i] -= *lr * weightDecay * param[i]; param[i] -= step_size * m[i] / (Sleef_sqrtf(v[i]) + eps); } } diff --git a/src/backend/CUDABackend.cu b/src/backend/CUDABackend.cu index c3c487d..e8e869d 100644 --- a/src/backend/CUDABackend.cu +++ b/src/backend/CUDABackend.cu @@ -178,9 +178,10 @@ namespace Deep cublasOperation_t cuTransA = transA ? CUBLAS_OP_T : CUBLAS_OP_N; cublasOperation_t cuTransB = transB ? CUBLAS_OP_T : CUBLAS_OP_N; - if (cublasSgemm(handle, cuTransB, cuTransA, - N, M, K, &alpha, B, ldb, A, lda, &beta, C, ldc) != CUBLAS_STATUS_SUCCESS) - std::cerr << "Failed to perform CUDA MatMul.\n"; + cublasStatus_t status = cublasSgemm(handle, cuTransB, cuTransA, + N, M, K, &alpha, B, ldb, A, lda, &beta, C, ldc); + if (status != CUBLAS_STATUS_SUCCESS) + std::cerr << "cublasSgemm status: " << status << "\n"; } void CUDABackend::Scale(float *buf, size_t n, float alpha) noexcept @@ -343,35 +344,6 @@ namespace Deep } } - __global__ void AdamStepKernel(float *param, const float *grad, float *m, float *v, - size_t n, float step_size, float beta1, float beta2, float eps) - { - size_t i = (size_t)blockIdx.x * blockDim.x + threadIdx.x; - if (i < n) - { - float g = grad[i]; - m[i] = beta1 * m[i] + (1.0f - beta1) * g; - v[i] = beta2 * v[i] + (1.0f - beta2) * (g * g); - param[i] -= step_size * m[i] / (sqrtf(v[i]) + eps); - } - } - - __global__ void AdamWStepKernel(float *param, const float *grad, float *m, float *v, - size_t n, float lr, float weightDecay, - float step_size, float beta1, float beta2, float eps) - { - size_t i = (size_t)blockIdx.x * blockDim.x + threadIdx.x; - if (i < n) - { - float g = grad[i]; - m[i] = beta1 * m[i] + (1.0f - beta1) * g; - v[i] = beta2 * v[i] + (1.0f - beta2) * (g * g); - - param[i] -= lr * weightDecay * param[i]; - param[i] -= step_size * m[i] / (sqrtf(v[i]) + eps); - } - } - #pragma endregion void CUDABackend::Activation(ActivationType type, float *buf, size_t n) noexcept @@ -470,32 +442,66 @@ namespace Deep ComputeErrorKernel<<>>(e, z, mu, n); } + __global__ void IncrementCounterKernel(int *counter) { *counter += 1; } + + __global__ void AdamStepKernel(float *param, const float *grad, float *m, float *v, + size_t n, const int *t_ptr, const float *lr_ptr, + float beta1, float beta2, float eps) + { + size_t i = (size_t)blockIdx.x * blockDim.x + threadIdx.x; + if (i < n) + { + float beta1_t = 1.0f - powf(beta1, (float)(*t_ptr)); + float beta2_t = 1.0f - powf(beta2, (float)(*t_ptr)); + float step_size = *lr_ptr * sqrtf(beta2_t) / beta1_t; + + float g = grad[i]; + m[i] = beta1 * m[i] + (1.0f - beta1) * g; + v[i] = beta2 * v[i] + (1.0f - beta2) * (g * g); + param[i] -= step_size * m[i] / (sqrtf(v[i]) + eps); + } + } + + __global__ void AdamWStepKernel(float *param, const float *grad, float *m, float *v, + size_t n, const int *t_ptr, const float *lr_ptr, float weightDecay, float beta1, float beta2, float eps) + { + size_t i = (size_t)blockIdx.x * blockDim.x + threadIdx.x; + if (i < n) + { + float beta1_t = 1.0f - powf(beta1, (float)(*t_ptr)); + float beta2_t = 1.0f - powf(beta2, (float)(*t_ptr)); + float step_size = *lr_ptr * sqrtf(beta2_t) / beta1_t; + + float g = grad[i]; + m[i] = beta1 * m[i] + (1.0f - beta1) * g; + v[i] = beta2 * v[i] + (1.0f - beta2) * (g * g); + param[i] -= *lr_ptr * weightDecay * param[i]; + param[i] -= step_size * m[i] / (sqrtf(v[i]) + eps); + } + } + + // AdamWStepKernel: same pattern, plus the weight-decay term + + void CUDABackend::IncrementCounter(int *counter) noexcept + { + IncrementCounterKernel<<<1, 1, 0, stream>>>(counter); + } + void CUDABackend::AdamStep(float *param, const float *grad, float *m, float *v, - size_t n, int t, float lr, + size_t n, const int *t, const float *lr, float beta1, float beta2, float eps) noexcept { - // Compute scalars on host CPU - float beta1_t = 1.0f - Sleef_powf_u10(beta1, static_cast(t)); - float beta2_t = 1.0f - Sleef_powf_u10(beta2, static_cast(t)); - float step_size = lr * Sleef_sqrtf(beta2_t) / beta1_t; - constexpr int BLOCK_SIZE = 256; const int blocks = static_cast((n + BLOCK_SIZE - 1) / BLOCK_SIZE); - - AdamStepKernel<<>>(param, grad, m, v, n, step_size, beta1, beta2, eps); + AdamStepKernel<<>>(param, grad, m, v, n, t, lr, beta1, beta2, eps); } void CUDABackend::AdamWStep(float *param, const float *grad, float *m, float *v, - size_t n, int t, float lr, float weightDecay, + size_t n, const int *t, const float *lr, float weightDecay, float beta1, float beta2, float eps) noexcept { - float beta1_t = 1.0f - Sleef_powf_u10(beta1, static_cast(t)); - float beta2_t = 1.0f - Sleef_powf_u10(beta2, static_cast(t)); - float step_size = lr * Sleef_sqrtf(beta2_t) / beta1_t; - constexpr int BLOCK_SIZE = 256; const int blocks = static_cast((n + BLOCK_SIZE - 1) / BLOCK_SIZE); - - AdamWStepKernel<<>>(param, grad, m, v, n, lr, weightDecay, step_size, beta1, beta2, eps); + AdamWStepKernel<<>>(param, grad, m, v, n, t, lr, weightDecay, beta1, beta2, eps); } } \ No newline at end of file From fcfb71bf63cdb86569a3ac6c61048ad3d0bdd745 Mon Sep 17 00:00:00 2001 From: Ra4ster Date: Mon, 7 Sep 2026 06:51:45 +0000 Subject: [PATCH 09/11] Now moving on to DKP-PCN CUDA --- bindings/pybinding.cpp | 11 +- include/deepity/backend/CPUBackend.h | 9 +- include/deepity/backend/CUDABackend.h | 12 +- include/deepity/backend/IComputeBackend.h | 124 +----- include/deepity/layers/DirectKPPCLayer.h | 79 +++- include/deepity/layers/SimplePCLayer.h | 2 + include/deepity/networks/DirectKPPCNetwork.h | 67 ++- include/deepity/networks/SimplePCNetwork.h | 3 +- logs/build.log | 237 ++++++++++- mnist.py | 4 +- pydeepity/DKPPCN.py | 4 +- pydeepity/__pycache__/DKPPCN.cpython-312.pyc | Bin 7695 -> 7733 bytes src/DirectKPPCLayer.cpp | 426 ++++++++----------- src/DirectKPPCNetwork.cpp | 137 ++++-- src/SimplePCNetwork.cpp | 44 +- src/backend/CPUBackend.cpp | 7 + src/backend/CUDABackend.cu | 114 +++-- temp.py | 3 +- 18 files changed, 822 insertions(+), 461 deletions(-) diff --git a/bindings/pybinding.cpp b/bindings/pybinding.cpp index d3ff0cc..9672082 100644 --- a/bindings/pybinding.cpp +++ b/bindings/pybinding.cpp @@ -442,11 +442,11 @@ void bind_networks(nb::module_ &m) .def("project_forward", &Deep::SimplePCNetwork::ProjectForward, "Seeds hidden layers from a genuine forward pass through current " "weights, instead of zero-init. Call AFTER clamp_input(), BEFORE " "the settling loop.") - .def("train_step_with_projection", [](Deep::SimplePCNetwork &self, FloatArray x, FloatArray y, int steps) + .def("train_step_with_projection", [](Deep::SimplePCNetwork &self, FloatArray x, FloatArray y, int steps, bool computeEnergy) { std::vector xvec(x.data(), x.data() + x.size()); std::vector yvec(y.data(), y.data() + y.size()); -return self.TrainStepWithProjection(xvec, yvec, steps); }, nb::arg("x"), nb::arg("y"), nb::arg("steps")) +return self.TrainStepWithProjection(xvec, yvec, steps, computeEnergy); }, nb::arg("x"), nb::arg("y"), nb::arg("steps"), nb::arg("computeEnergy") = true) .def("predict_with_projection", [](Deep::SimplePCNetwork &self, FloatArray x, int steps) { std::vector xvec(x.data(), x.data() + x.size()); @@ -505,7 +505,12 @@ return CopyToNewArray(out_beliefs.data(), {out_beliefs.size()}); }, nb::arg("x") return layers[index].get(); }, nb::rv_policy::reference_internal); nb::class_(m, "DirectKPPCNetwork", "Predictive Coding Network with Direct Kolen-Pollack feedback alignment.") - .def(nb::init(), nb::arg("batch_size")) + .def("__init__", [](Deep::DirectKPPCNetwork *self, int batch_size, const std::string &device) + { + Deep::DeviceType dt = (device == "cuda" || device == "gpu") + ? Deep::DeviceType::DEVICE_GPU + : Deep::DeviceType::DEVICE_CPU; + new (self) Deep::DirectKPPCNetwork(batch_size, dt); }, nb::arg("batch_size"), nb::arg("device") = "cpu") .def("add_layer", [](Deep::DirectKPPCNetwork &self, size_t size, size_t next_size, size_t terminal_size, float lr, float ir, float fl, float lmbda, const std::string &activation, const std::string &activation_deriv) { self.AddLayer(size, next_size, terminal_size, lr, ir, fl, lmbda, resolveActEnum(activation), resolveActEnum(activation_deriv)); }, nb::arg("size"), nb::arg("next_size"), nb::arg("terminal_size"), nb::arg("lr") = 1e-6f, nb::arg("ir") = 0.1f, nb::arg("fl") = 1e-4f, nb::arg("lmbda") = 1e-2f, nb::arg("activation") = "relu", nb::arg("activation_deriv") = "drelu") .def("compile", &Deep::DirectKPPCNetwork::Compile) diff --git a/include/deepity/backend/CPUBackend.h b/include/deepity/backend/CPUBackend.h index 7a57e1b..9fc0dce 100644 --- a/include/deepity/backend/CPUBackend.h +++ b/include/deepity/backend/CPUBackend.h @@ -12,7 +12,7 @@ namespace Deep // @remark these are no-ops for backend purposes void BeginGraphCapture() noexcept override {} - void EndGraphCapture() noexcept override {} + bool EndGraphCapture() noexcept override { return true; } // nothing to fail on CPU void ReplayGraph() noexcept override {} float *Allocate(size_t numFloats) override; @@ -24,10 +24,17 @@ namespace Deep void RandomizeNormal(float *buf, size_t n, float mean, float stddev, uint32_t seed) noexcept override; void RandomizeUniform(float *buf, size_t n, float min, float max, uint32_t seed) noexcept override; + /// @brief Must be called once, before Compile()'s first + /// BeginGraphCapture(), for any backend that needs to prepare + /// batch-size-dependent state (e.g. CUDABackend's cached all-ones + /// vector for SumRows' GEMV). No-op on CPUBackend. + void PrepareForBatchSize(size_t batchSize) noexcept override {} + void MatMul(bool transA, bool transB, int M, int N, int K, float alpha, const float *A, int lda, const float *B, int ldb, float beta, float *C, int ldc) noexcept override; + void SumRows(float *dst, const float *src, size_t batchSize, size_t width) noexcept override; void Scale(float *buf, size_t n, float alpha) noexcept override; void AxpyInto(float *y, const float *x, size_t n, float alpha) noexcept override; diff --git a/include/deepity/backend/CUDABackend.h b/include/deepity/backend/CUDABackend.h index 6d87b9e..e88716f 100644 --- a/include/deepity/backend/CUDABackend.h +++ b/include/deepity/backend/CUDABackend.h @@ -11,7 +11,7 @@ namespace Deep CUDABackend(); ~CUDABackend() override; void BeginGraphCapture() noexcept override; - void EndGraphCapture() noexcept override; + bool EndGraphCapture() noexcept override; void ReplayGraph() noexcept override; float *Allocate(size_t numFloats) override; @@ -23,6 +23,14 @@ namespace Deep void RandomizeNormal(float *buf, size_t n, float mean, float stddev, uint32_t seed) noexcept override; void RandomizeUniform(float *buf, size_t n, float min, float max, uint32_t seed) noexcept override; + void SumRows(float *dst, const float *src, size_t batchSize, size_t width) noexcept override; + + /// @brief Must be called once, before Compile()'s first + /// BeginGraphCapture(), for any backend that needs to prepare + /// batch-size-dependent state (e.g. CUDABackend's cached all-ones + /// vector for SumRows' GEMV). No-op on CPUBackend. + void PrepareForBatchSize(size_t batchSize) noexcept override; + void MatMul(bool transA, bool transB, int M, int N, int K, float alpha, const float *A, int lda, const float *B, int ldb, @@ -59,5 +67,7 @@ namespace Deep cudaGraph_t graph = nullptr; cudaGraphExec_t graphExec = nullptr; bool hasGraph = false; + void *workspace = nullptr; + float *onesVector = nullptr; }; } \ No newline at end of file diff --git a/include/deepity/backend/IComputeBackend.h b/include/deepity/backend/IComputeBackend.h index 6363cdc..2ebb66d 100644 --- a/include/deepity/backend/IComputeBackend.h +++ b/include/deepity/backend/IComputeBackend.h @@ -3,31 +3,6 @@ #include #include -/** - * @file IComputeBackend.h - * @brief Abstract interface for every core numerical operation a PC - * layer needs, so that CPUBackend and CUDABackend can be swapped behind - * one pointer -- no #ifdef DEEPITY_USE_CUDA anywhere except inside - * Backend.cpp's single factory function. - * - * Derived directly from what SimplePCLayer/DirectKPPCLayer actually call - * today (CalculateState, ComputeMuOnly, UpdateState, UpdateWeights, - * DirectFeedbackUpdate), not a generic "BLAS wrapper" -- a few operations - * below are kept as single, explicit, FUSED methods (FusedStateUpdate, - * ComputeErrorAndEnergy) specifically because decomposing them into - * separate elementwise-multiply/subtract/saxpy calls would lose the - * fusion benefit this codebase already relies on on CPU, and would cost - * even more on GPU (extra global-memory round-trips between kernel - * launches, where memory bandwidth is usually the real bottleneck). - * - * Device is fixed at Tensor/allocation time (see Tensor.h) -- nothing - * here supports moving a live buffer between CPU and GPU after creation. - * - * @warning This is a first draft, not yet implemented by either - * CPUBackend or CUDABackend. Several signatures are marked below as - * open questions -- confirm/adjust before treating this as final. - */ - namespace Deep { @@ -39,7 +14,14 @@ namespace Deep // Graphs virtual void BeginGraphCapture() noexcept = 0; - virtual void EndGraphCapture() noexcept = 0; + /// @brief Ends capture and instantiates the captured graph. + /// @return true if capture and instantiation both succeeded and + /// ReplayGraph() is now safe to call; false otherwise. Callers + /// MUST check this -- silently assuming success here was the + /// cause of a real bug: a failed capture left ReplayGraph() + /// permanently doing nothing on every subsequent call, since the + /// caller had no way to know capture never actually happened. + virtual bool EndGraphCapture() noexcept = 0; virtual void ReplayGraph() noexcept = 0; // Memory @@ -47,36 +29,18 @@ namespace Deep virtual float *Allocate(size_t numFloats) = 0; virtual void Free(float *ptr) noexcept = 0; virtual void Zero(float *ptr, size_t numFloats) noexcept = 0; - - /// @brief Device-to-device copy (both buffers already live on - /// this backend's device). virtual void Copy(float *dst, const float *src, size_t numFloats) noexcept = 0; - - /// @brief Host-to-device copy. No-op memcpy on CPUBackend; - /// cudaMemcpyHostToDevice on CUDABackend. This is the ONLY thing - /// needed for "load weights saved on CPU, run on GPU"; see the - /// constructor-time initialization pattern discussed separately. virtual void CopyFromHost(float *deviceDst, const float *hostSrc, size_t numFloats) noexcept = 0; - - /// @brief Device-to-host copy, e.g. for Save()/inspection. virtual void CopyToHost(float *hostDst, const float *deviceSrc, size_t numFloats) noexcept = 0; - - /// @brief Randomizes floats using standard normal distribution. - /// @param buf Array of floats - /// @param n Size of buf - /// @param mean Mu parameter of Normal Dist. - /// @param stddev Sigma parameter of Normal Dist. - /// @param seed Random seed virtual void RandomizeNormal(float *buf, size_t n, float mean, float stddev, uint32_t seed) noexcept = 0; - - /// @brief Randomizes floats using uniform distribution. - /// @param buf Array of floats - /// @param n Size of buf - /// @param mean Min value for random generation - /// @param stddev Max value for random generation - /// @param seed Random seed virtual void RandomizeUniform(float *buf, size_t n, float min, float max, uint32_t seed) noexcept = 0; + /// @brief Must be called once, before Compile()'s first + /// BeginGraphCapture(), for any backend that needs to prepare + /// batch-size-dependent state (e.g. CUDABackend's cached all-ones + /// vector for SumRows' GEMV). No-op on CPUBackend. + virtual void PrepareForBatchSize(size_t batchSize) noexcept = 0; + // GEMM virtual void MatMul(bool transA, bool transB, @@ -85,81 +49,39 @@ namespace Deep const float *B, int ldb, float beta, float *C, int ldc) noexcept = 0; + /// @brief dst[j] = sum over b in [0,batchSize) of src[b*width + j], for + /// all j in [0,width). Replaces a batchSize-iteration loop of + /// individual AxpyInto calls -- the reduction-direction counterpart to + /// AddBiasBroadcast, still unfixed until now. On GPU this is one + /// cublasSgemv call against a cached all-ones vector, reinterpreting + /// src's row-major [batchSize,width] layout as column-major + /// [width,batchSize] with no data movement (verified numerically). + virtual void SumRows(float *dst, const float *src, size_t batchSize, size_t width) noexcept = 0; + // Elementwise scalar ops - /// @brief buf *= alpha (cblas_sscal equivalent). Used for weight - /// decay (W *= 1-lambda) today. virtual void Scale(float *buf, size_t n, float alpha) noexcept = 0; - - /// @brief y += alpha * x (cblas_saxpy equivalent). Used for bias - /// adds and Adam/AdamW's own internal accumulation today. virtual void AxpyInto(float *y, const float *x, size_t n, float alpha) noexcept = 0; - - /// @brief buf[b, :] += bias[:] for every row b in [0, batchSize). Replaces - /// a batchSize-iteration loop of individual AxpyInto calls with one - /// launch -- on GPU, 250 separate kernel launches per call (each with - /// real, fixed host-side dispatch overhead regardless of how little work - /// it does) was the actual dominant cost in the settling loop, not the - /// energy-reduction syncs this was originally suspected to be. virtual void AddBiasBroadcast(float *buf, const float *bias, size_t batchSize, size_t width) noexcept = 0; // Activation - /// @brief In-place activation, matching Deep::relu/sigmoid/etc's - /// existing single-buffer signature. virtual void Activation(ActivationType type, float *buf, size_t n) noexcept = 0; - - /// @brief Two-buffer activation: reads src, writes phi(src) into - /// dst, src left untouched. Matches ComputeMuOnly()'s zF = phi(z) - /// pattern without needing a separate scopy first. virtual void ActivationInto(ActivationType type, float *dst, const float *src, size_t n) noexcept = 0; - - /// @brief In-place derivative, matching Deep::dRelu/dSigmoid/etc's - /// existing (buf, n, activated) signature. virtual void ActivationDerivative(ActivationType type, float *buf, size_t n, bool activated) noexcept = 0; - - /// @brief Two-buffer derivative: reads RAW src, writes f'(src) - /// into dst. Matches the dReluInto/dSigmoidInto/etc family added - /// to Activations.h this session. virtual void ActivationDerivativeInto(ActivationType type, float *dst, const float *src, size_t n) noexcept = 0; // Fused PC-specific ops - /// @brief z[i] += ir * (feedback[i] * deriv[i] - e[i]), for all i - /// in [0, n). Matches SimplePCLayer/DirectKPPCLayer's UpdateState() - /// fused settling-step update exactly. virtual void FusedStateUpdate(float *z, const float *feedback, const float *deriv, const float *e, size_t n, float ir) noexcept = 0; - - /// @brief e[i] = z[i] - mu[i], for all i in [0, n); returns - /// 0.5 * sum(e[i]^2). Matches CalculateState()'s error+energy - /// computation exactly (the fused AVX loop from earlier tonight). virtual float ComputeErrorAndEnergy(float *e, const float *z, const float *mu, size_t n) noexcept = 0; - - /// @brief e[i] = z[i] - mu[i], for all i in [0, n). Same computation as - /// ComputeErrorAndEnergy but WITHOUT the energy reduction -- no cuBLAS - /// dot-product call, no host/device sync. Use this during a settling - /// loop's discarded intermediate iterations, where only the final - /// energy value (from ComputeErrorAndEnergy) is ever actually used; - /// every earlier call was needlessly forcing a full GPU pipeline stall - /// just to compute a number nobody reads. virtual void ComputeError(float *e, const float *z, const float *mu, size_t n) noexcept = 0; // Optimizer - /// @brief Increments *counter by 1, on-device (a single, dedicated - /// kernel launch on GPU; a plain ++ on CPU). Exists so Adam/AdamW's - /// timestep can live entirely inside a captured CUDA graph -- see - /// AdamStep's own note for why t and lr moved from by-value host - /// arguments to device-resident pointers. virtual void IncrementCounter(int *counter) noexcept = 0; - /// @brief t and lr are now device-resident pointers, not host values - /// taken by copy -- a graph captures the ARGUMENTS baked in at capture - /// time, so a host int/float would freeze t and lr at whatever they - /// were during the one-time capture call, silently never advancing on - /// any later replay. Reading them from device memory each call lets - /// the kernel itself see up-to-date values every replay. virtual void AdamStep(float *param, const float *grad, float *m, float *v, size_t n, const int *t, const float *lr, float beta1 = 0.9f, float beta2 = 0.999f, float eps = 1e-8f) noexcept = 0; diff --git a/include/deepity/layers/DirectKPPCLayer.h b/include/deepity/layers/DirectKPPCLayer.h index 9ce30e8..812ab5c 100644 --- a/include/deepity/layers/DirectKPPCLayer.h +++ b/include/deepity/layers/DirectKPPCLayer.h @@ -2,12 +2,38 @@ #include #include +#include #include #include +#include #include #include #include +/** + * @file DirectKPPCLayer.h + * @brief Direct Kolen-Pollack predictive coding layer, now routed + * through IComputeBackend rather than calling cblas_Deep::_ directly + * -- same refactor discipline as SimplePCLayer's own backend port. + * + * @note Two independent optimizer timesteps/learning-rates exist here, + * not one: `t`/`lr` for the forward weights W, `tPsi`/`fl` for the + * direct-feedback weights Psi. Both need their own device-resident + * mirrors (t_device/lr_device, tPsi_device/fl_device) for the same + * reason SimplePCLayer's did -- IComputeBackend::AdamStep/AdamWStep take + * device pointers so their values can live inside a captured CUDA graph + * without freezing at capture time. + * + * @warning `backend` defaults to nullptr, in which case this layer + * constructs and owns its own CPUBackend internally -- every existing + * call site keeps working unchanged. Passing a real backend is only + * needed for GPU-aware call sites. + * @warning RandomizeWeights() is still host-only for the CPU fallback + * path's seed-drawing logic, but the actual weight fill now goes through + * backend->RandomizeNormal(), which is GPU-safe -- unlike + * SimplePCLayer's still-open RandomizeWeights gap, this one is resolved. + */ + namespace Deep { class DirectKPPCLayer : public Layer @@ -40,6 +66,15 @@ namespace Deep int t = 0; int tPsi = 0; + /// @brief The compute backend this layer routes all math + /// through. unique_ptr with a swappable deleter: no-op when an + /// external backend was supplied (network-owned, GPU case), + /// real delete when this layer had to construct its own + /// fallback CPUBackend. See SimplePCLayer.h for the identical + /// pattern and rationale. + using BackendDeleter = void (*)(IComputeBackend *); + std::unique_ptr backend; + // --- Memory Pointers --- // State float *z = nullptr; @@ -66,6 +101,13 @@ namespace Deep float *m_Psi = nullptr; float *v_Psi = nullptr; + // Device-resident optimizer scalars -- see file-level note. + // Only allocated when the corresponding optimizer is ADAM/ADAMW. + int *t_device = nullptr; + float *lr_device = nullptr; + int *tPsi_device = nullptr; + float *fl_device = nullptr; + // Scratch float *zF = nullptr; float *zFDeriv = nullptr; @@ -74,14 +116,22 @@ namespace Deep std::unique_ptr localArena; public: + /// @param backend Compute backend to route all math through. + /// Defaults to nullptr, in which case this layer + /// constructs and owns its own CPUBackend internally. DirectKPPCLayer(size_t size, size_t nextSize, size_t terminalSize, size_t batchSize, float learningRate, float inferenceRate, float feedback, float lmbda, - ActivationType aType, ActivationType dType); + ActivationType aType, ActivationType dType, + IComputeBackend *backend = nullptr); ~DirectKPPCLayer() override = default; // Setup - void BindMemory(MemoryArena &arena); + /// @brief Templated so either MemoryArena (CPU) or + /// DeviceMemoryArena (GPU) can be bound, resolved at compile + /// time -- see the .cpp's explicit instantiations. + template + void BindMemory(ArenaT &arena); size_t GetRequiredFloats() const noexcept; void RandomizeWeights(std::mt19937 &seedGenerator) noexcept; @@ -91,7 +141,8 @@ namespace Deep void SetTerminalLayer(DirectKPPCLayer *l) noexcept { terminalLayer = l; } // Core DKP-PC Mechanics - float CalculateState() noexcept override; + float CalculateState() noexcept override { return CalculateState(true); } + float CalculateState(bool needEnergy) noexcept; void ComputeMuOnly() noexcept; void UpdateState() noexcept override; // Will now pull from terminalLayer->GetErrors() void UpdateWeights() noexcept override; // Must compute \Delta W AND \Delta \Psi @@ -104,10 +155,24 @@ namespace Deep void SetOptimizer(OptimizerType o) noexcept { opt = o; } void SetPsiOptimizer(OptimizerType o) noexcept { optPsi = o; } - void SetLearningRate(float learningRate) noexcept { lr = learningRate; } + /// @brief Sets lr AND keeps its device-resident mirror + /// (lr_device) in sync -- required for a captured CUDA graph to + /// see an updated learning rate on later replays; a plain + /// member write alone would be invisible to already-captured + /// kernel arguments. + void SetLearningRate(float learningRate) noexcept; void SetInferenceRate(float inferenceRate) noexcept { ir = inferenceRate; } - void SetFeedbackRate(float feedbackRate) noexcept { fl = feedbackRate; } - void SetLambda(float lmbda) noexcept { lmbda = lmbda; } + /// @brief Sets fl (feedback learning rate) AND keeps its + /// device-resident mirror (fl_device) in sync -- see + /// SetLearningRate's own note; fl has the identical requirement + /// since it drives Psi's own Adam/AdamW step. + void SetFeedbackRate(float feedbackRate) noexcept; + /// @brief Fixed: the original had `lmbda = lmbda`, a + /// parameter-shadows-member bug that silently never updated the + /// actual member. Pre-existing, unrelated to the backend + /// refactor -- fixed here since it was noticed while reading + /// through the class closely. + void SetLambda(float lmbda) noexcept { this->lmbda = lmbda; } float *GetBeliefs() noexcept override { return z; } const float *GetErrors() const noexcept override { return e; } @@ -121,4 +186,4 @@ namespace Deep size_t GetOutputSize() const noexcept override { return nextSize; } size_t GetTerminalSize() const noexcept { return terminalSize; } }; -} +} \ No newline at end of file diff --git a/include/deepity/layers/SimplePCLayer.h b/include/deepity/layers/SimplePCLayer.h index aad0943..1c57327 100644 --- a/include/deepity/layers/SimplePCLayer.h +++ b/include/deepity/layers/SimplePCLayer.h @@ -159,6 +159,8 @@ namespace Deep /// layer's beliefs to update normally again. void UnclampState() noexcept; + bool IsClamped() const noexcept { return isClamped; } + /// @brief Returns this layer's belief buffer. /// @return Pointer to this layer's `size`-length beliefs. float *GetBeliefs() noexcept override { return z; } diff --git a/include/deepity/networks/DirectKPPCNetwork.h b/include/deepity/networks/DirectKPPCNetwork.h index 8fef9cd..5af466a 100644 --- a/include/deepity/networks/DirectKPPCNetwork.h +++ b/include/deepity/networks/DirectKPPCNetwork.h @@ -4,6 +4,8 @@ #include #include #include +#include +#include /** * @file DirectKPPCNetwork.h @@ -31,7 +33,13 @@ * it does not depend on any ordering among layers -- each layer's * update only reads Psi (which does not change during this phase) * and epsilon_L, both already available. Unlike GaussSeidelPCLayer, - * there is no sweep-ordering constraint here. + * there is no sweep-ordering constraint here. CONFIRMED by tracing + * every read/write in DirectFeedbackUpdate(): every layer writes + * only to its own proj/W, reads only from a shared, read-only + * terminalLayer error buffer and its own layerAbove's Psi -- zero + * cross-layer RAW/WAW hazard. This is the specific phase intended + * as the first real parallelism target (multiple CUDA streams, one + * per layer), once this class is confirmed correct on GPU. * * 2. Inference phase -- ordinary PC settling (CalculateState() + * UpdateState() across every layer), for inferenceSteps steps. @@ -47,10 +55,17 @@ * which (per DirectKPPCLayer) now updates both W (from the * settled state) and Psi (from the settled state and epsilon_L). * - * @warning Not yet gradient-checked at the network level. The - * individual layer's math should be independently verified before - * trusting any accuracy conclusion drawn from real training with this - * class. + * @note As of this revision, routed through IComputeBackend -- same + * refactor as SimplePCNetwork's own GPU port. `device` defaults to + * DEVICE_CPU, preserving existing behavior for anyone not explicitly + * requesting DEVICE_GPU. + * + * @warning Not yet gradient-checked at the network level, and not yet + * tested on GPU at all. The individual layer's math should be + * independently verified before trusting any accuracy conclusion drawn + * from real training with this class. This is Stage 1 of a three-stage + * plan (CPU-correct port -> GPU-correct, sequential -> exploit phase 1's + * confirmed cross-layer parallelism) -- this revision is Stage 1 only. */ namespace Deep @@ -60,7 +75,11 @@ namespace Deep public: /// @brief Constructs an empty network with a predetermined batch size. /// @param batchSize Batch size - explicit DirectKPPCNetwork(int batchSize) noexcept; + /// @param device Which device this network's layers should run + /// on. Defaults to DEVICE_CPU, preserving existing + /// behavior exactly for anyone not explicitly requesting + /// DEVICE_GPU. + explicit DirectKPPCNetwork(int batchSize, DeviceType device = DeviceType::DEVICE_CPU) noexcept; /// @brief Default destructor. ~DirectKPPCNetwork() = default; @@ -114,14 +133,16 @@ namespace Deep /// @brief Phase 1: runs DirectFeedbackUpdate() on every /// non-terminal layer. Order among layers does not matter (see /// class-level docs) -- unlike GaussSeidelPCNetwork, no sweep - /// ordering is required here. + /// ordering is required here. Still sequential in this + /// revision; see class-level warning. void DirectFeedbackUpdate() noexcept; /// @brief Phase 2: runs one ordinary PC settling step /// (CalculateState() + UpdateState()) across every layer. + /// @param computeEnergy asks for energy to be returned /// @return Total energy, summed from every layer's /// CalculateState(). - float Step() noexcept; + float Step(bool computeEnergy = true) noexcept; /// @brief Phase 3: updates every non-terminal layer's W and Psi. /// Called once after the settling loop completes. @@ -160,6 +181,8 @@ namespace Deep const std::vector> &GetLayers() const noexcept { return layers; } /// @brief Returns the batch size for the network's layers. int GetBatchSize() const noexcept { return batchSize; } + /// @brief Returns which device this network's layers run on. + DeviceType GetDevice() const noexcept { return device; } /// @brief Full train step: reset, clamp input+target, run all /// four DKP-PC phases in order, unclamp. @@ -178,17 +201,35 @@ namespace Deep /// perturbation and no target clamped, read the terminal's beliefs. std::vector Predict(const std::vector &x, int inferenceSteps); - /// @brief Loads all layers into one contiguous block of memory, - /// and wires layerAbove/layerBelow/terminalLayer across every - /// layer. + /// @brief Loads all layers into one contiguous block of memory + /// (MemoryArena for DEVICE_CPU, DeviceMemoryArena for + /// DEVICE_GPU), and wires layerAbove/layerBelow/terminalLayer + /// across every layer. void Compile(); private: /// @brief Every layer in the network, in the order they were added. std::vector> layers; - /// @brief The contiguous memory block backing every layer's buffers. - std::unique_ptr arena; + + /// @brief The network's own compute backend, created once at + /// construction and shared by every layer added afterward. + std::unique_ptr backend; + /// @brief Which device `backend` actually is. + DeviceType device; + + /// @brief Used when device == DEVICE_CPU. + std::unique_ptr cpuArena; + /// @brief Used when device == DEVICE_GPU. Only compiled when + /// DEEPITY_USE_CUDA is defined. +#if defined(DEEPITY_USE_CUDA) + std::unique_ptr gpuArena; +#endif + /// @brief The batch size shared by every layer in the network. int batchSize; + + // @brief CUDA Graph: + bool graphCaptured = false; + int capturedInferenceSteps = -1; }; } \ No newline at end of file diff --git a/include/deepity/networks/SimplePCNetwork.h b/include/deepity/networks/SimplePCNetwork.h index f26aa2e..63737e9 100644 --- a/include/deepity/networks/SimplePCNetwork.h +++ b/include/deepity/networks/SimplePCNetwork.h @@ -168,7 +168,8 @@ namespace Deep /// calls, update_weights, unclamp_state -- over 40 individual /// crossings per batch at STEPS=20). This does the whole sequence in /// ONE crossing instead. - float TrainStepWithProjection(const std::vector &x, const std::vector &y, int inferenceSteps); + float TrainStepWithProjection(const std::vector &x, const std::vector &y, + int inferenceSteps, bool computeEnergy = true); std::vector PredictWithProjection(const std::vector &x, int inferenceSteps); /// @brief Sets mu-cache threshold on every layer -- see diff --git a/logs/build.log b/logs/build.log index 0a034f9..c7bd3fd 100644 --- a/logs/build.log +++ b/logs/build.log @@ -1,11 +1,238 @@ --- Deepity Build Log (Release, arch=native) --- +=== CMake Configuration === +-- The C compiler identification is GNU 11.4.0 +-- The CXX compiler identification is GNU 11.4.0 +-- Detecting C compiler ABI info +-- Detecting C compiler ABI info - done +-- Check for working C compiler: /usr/bin/cc - skipped +-- Detecting C compile features +-- Detecting C compile features - done +-- Detecting CXX compiler ABI info +-- Detecting CXX compiler ABI info - done +-- Check for working CXX compiler: /usr/bin/g++ - skipped +-- Detecting CXX compile features +-- Detecting CXX compile features - done +-- CXX compiler: /usr/bin/g++ +-- Found OpenMP_C: -fopenmp (found version "4.5") +-- Found OpenMP_CXX: -fopenmp (found version "4.5") +-- Found OpenMP: TRUE (found version "4.5") +-- Looking for pthread.h +-- Looking for pthread.h - found +-- Performing Test CMAKE_HAVE_LIBC_PTHREAD +-- Performing Test CMAKE_HAVE_LIBC_PTHREAD - Success +-- Found Threads: TRUE +-- The CUDA compiler identification is NVIDIA 12.4.131 +-- Detecting CUDA compiler ABI info +-- Detecting CUDA compiler ABI info - done +-- Check for working CUDA compiler: /usr/local/cuda/bin/nvcc - skipped +-- Detecting CUDA compile features +-- Detecting CUDA compile features - done +-- CUDA support enabled. +-- Found Python: /usr/bin/python3.12 (found suitable version "3.12.13", minimum required is "3.8") found components: Interpreter Development.Module +-- Performing Test NB_HAS_MTLS_GNU2 +-- Performing Test NB_HAS_MTLS_GNU2 - Success +-- Using Intel MKL (found via MKLConfig.cmake). +-- Configuring SLEEF 3.9.0 +-- Could NOT find OpenSSL, try to set the path to OpenSSL root folder in the system variable OPENSSL_ROOT_DIR (missing: OPENSSL_CRYPTO_LIBRARY OPENSSL_INCLUDE_DIR) +-- Found PkgConfig: /usr/bin/pkg-config (found version "0.29.2") +-- Found OpenMP_C: -fopenmp (found version "4.5") +-- Found OpenMP_CXX: -fopenmp (found version "4.5") +-- Configuring build for SLEEF-v3.9.0 + Target system: Linux-6.8.0-138-generic + Target processor: x86_64 + Host system: Linux-6.8.0-138-generic + Host processor: x86_64 + Detected C compiler: GNU @ /usr/bin/cc + CMake: 3.22.1 + Make program: /usr/bin/ninja + CMake build type: Release +-- Using option `-Wall -Wno-unused-function -Wno-attributes -Wno-unused-result -Wno-psabi -ffp-contract=off -fno-math-errno -fno-trapping-math` to compile libsleef +-- Building shared libs : OFF +-- Building static test bins: OFF +-- MPFR : LIB_MPFR-NOTFOUND +-- GMP : LIBGMP-NOTFOUND +-- RT : /usr/lib/x86_64-linux-gnu/librt.a +-- FFTW3 : LIBFFTW3-NOTFOUND +-- FFTW3F : LIBFFTW3F-NOTFOUND +-- OPENSSL : +-- SDE : SDE_COMMAND-NOTFOUND +-- COMPILER_SUPPORTS_OPENMP : 1 +-- A version of SLEEF compatible with libm and libmvec in GNU libc will be produced (sleefgnuabi.so) +-- Configuring done +-- Generating done +-- Build files have been written to: /root/Deepity/build/Release + + === Compilation === -[1/5] Building CXX object CMakeFiles/Deepity.dir/src/backend/Backend.cpp.o -[2/5] Building CUDA object CMakeFiles/Deepity.dir/src/backend/CUDABackend.cu.o -[3/5] Linking CXX static library bin/libDeepity.a -[4/5] Linking CXX executable bin/AddBiasBroadcastVerify -[5/5] Linking CXX shared module bin/pydeepity.cpython-312-x86_64-linux-gnu.so +[1/153] Building C object _deps/sleef-build/src/common/CMakeFiles/common.dir/common.c.o +[2/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/mkrename_gnuabi.dir/mkrename_gnuabi.c.o +[3/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/mkmasked_gnuabi.dir/mkmasked_gnuabi.c.o +[4/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/mkalias.dir/mkalias.c.o +[5/153] Building C object _deps/sleef-build/src/common/CMakeFiles/qtesterutil_obj.dir/qtesterutil.c.o +[6/153] Linking C executable _deps/sleef-build/bin/mkmasked_gnuabi +[7/153] Linking C executable _deps/sleef-build/bin/mkrename_gnuabi +[8/153] Generating include/masked_avx512f_dp_gnuabi.h +[9/153] Building C object _deps/sleef-build/src/common/CMakeFiles/addSuffix.dir/addSuffix.c.o +[10/153] Generating include/masked_avx512f_sp_gnuabi.h +[11/153] Building C object _deps/sleef-build/src/common/CMakeFiles/testerutil_obj.dir/testerutil.c.o +[12/153] Generating include/renameavx2_gnuabi.h +Generating renameavx2_gnuabi.h: mkrename_gnuabi avx2 d 4 8 __m256d __m256 __m128i __m256i __AVX2__ +[13/153] Generating include/renameavx512f_gnuabi.h +Generating renameavx512f_gnuabi.h: mkrename_gnuabi avx512f e 8 16 __m512d __m512 __m256i __m512i __AVX512F__ +[14/153] Generating include/renameavx_gnuabi.h +Generating renameavx_gnuabi.h: mkrename_gnuabi avx c 4 8 __m256d __m256 __m128i struct\ {\ __m128i\ x,\ y;\ } __AVX__ +[15/153] Generating include/renamesse2_gnuabi.h +Generating renamesse2_gnuabi.h: mkrename_gnuabi sse2 b 2 4 _mm128d _mm128 _mm128i _mm128i __SSE2__ +[16/153] Linking C executable _deps/sleef-build/bin/mkalias +[17/153] Linking C executable _deps/sleef-build/bin/addSuffix +[18/153] Generating alias_AVX512F_dp.h.tmp +[19/153] Generating alias_AVX512F_sp.h.tmp +[20/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/mkdisp.dir/mkdisp.c.o +[21/153] Generating include/alias_avx512f.h +[22/153] Building CXX object _deps/sleef-build/src/common/CMakeFiles/psha_obj.dir/psha2_capi.cpp.o +[23/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefgnuabi.dir/rempitab.c.o +[24/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/mkrename.dir/mkrename.c.o +[25/153] Linking C executable _deps/sleef-build/bin/mkdisp +[26/153] Generating dispscalar.c.body +[27/153] Generating dispsse.c.tmp +[28/153] Generating dispavx.c.tmp +[29/153] Generating dispscalar.c +[30/153] Generating dispavx.c +[31/153] Generating dispsse.c +[32/153] Linking C executable _deps/sleef-build/bin/mkrename +[33/153] Generating include/renamepurec_scalar.h +Generating renamepurec_scalar.h: mkrename cinz_ 1 1 purec +[34/153] Generating include/renameavx512fnofma.h +Generating renameavx512fnofma.h: mkrename cinz_ 8 16 avx512fnofma +[35/153] Generating include/renameavx512f.h +Generating renameavx512f.h: mkrename finz_ 8 16 avx512f +[36/153] Generating include/renameavx2.h +Generating renameavx2.h: mkrename finz_ 4 8 avx2 +[37/153] Generating include/renameavx2128.h +Generating renameavx2128.h: mkrename finz_ 2 4 avx2128 +[38/153] Generating include/renamefma4.h +Generating renamefma4.h: mkrename finz_ 4 8 fma4 +[39/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetavx512fnofma.dir/sleefsimdsp.c.o +[40/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetavx512fnofma.dir/sleefsimddp.c.o +[41/153] Building CXX object CMakeFiles/nanobind-static.dir/usr/local/lib/python3.12/dist-packages/nanobind/src/nb_static_property.cpp.o +[42/153] Building CXX object CMakeFiles/nanobind-static.dir/usr/local/lib/python3.12/dist-packages/nanobind/src/nb_datetime.cpp.o +[43/153] Building CXX object CMakeFiles/nanobind-static.dir/usr/local/lib/python3.12/dist-packages/nanobind/src/implicit.cpp.o +[44/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefgnuabiavx2sp.dir/sleefsimdsp.c.o +[45/153] Building CXX object CMakeFiles/nanobind-static.dir/usr/local/lib/python3.12/dist-packages/nanobind/src/trampoline.cpp.o +[46/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefavx512f.dir/sleefsimdsp.c.o +[47/153] Building CXX object CMakeFiles/nanobind-static.dir/usr/local/lib/python3.12/dist-packages/nanobind/src/error.cpp.o +[48/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefgnuabiavx512fsp.dir/sleefsimdsp.c.o +[49/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefgnuabiavx2dp.dir/sleefsimddp.c.o +[50/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefgnuabiavx512fdp.dir/sleefsimddp.c.o +[51/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetavx512f.dir/sleefsimdsp.c.o +[52/153] Building CXX object CMakeFiles/nanobind-static.dir/usr/local/lib/python3.12/dist-packages/nanobind/src/common.cpp.o +[53/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefavx512fnofma.dir/sleefsimdsp.c.o +[54/153] Building CXX object CMakeFiles/nanobind-static.dir/usr/local/lib/python3.12/dist-packages/nanobind/src/nb_ndarray.cpp.o +[55/153] Generating include/renameavx.h +Generating renameavx.h: mkrename cinz_ 4 8 avx +[56/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefavx512fnofma.dir/sleefsimddp.c.o +[57/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefgnuabisse2sp.dir/sleefsimdsp.c.o +[58/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetavx2.dir/sleefsimdsp.c.o +[59/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefgnuabiavxdp.dir/sleefsimddp.c.o +[60/153] Generating include/renamesse4.h +Generating renamesse4.h: mkrename cinz_ 2 4 sse4 +[61/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetavx512f.dir/sleefsimddp.c.o +[62/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefgnuabisse2dp.dir/sleefsimddp.c.o +[63/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetavx2128.dir/sleefsimdsp.c.o +[64/153] Building CXX object CMakeFiles/nanobind-static.dir/usr/local/lib/python3.12/dist-packages/nanobind/src/nb_internals.cpp.o +[65/153] Generating include/renamesse2.h +Generating renamesse2.h: mkrename cinz_ 2 4 sse2 +[66/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetavx2.dir/sleefsimddp.c.o +[67/153] Building CXX object CMakeFiles/nanobind-static.dir/usr/local/lib/python3.12/dist-packages/nanobind/src/nb_enum.cpp.o +[68/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefavx512f.dir/sleefsimddp.c.o +[69/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefgnuabiavxsp.dir/sleefsimdsp.c.o +[70/153] Linking C static library _deps/sleef-build/lib/libsleefgnuabi.a +[71/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetavx2128.dir/sleefsimddp.c.o +[72/153] Building CXX object CMakeFiles/nanobind-static.dir/usr/local/lib/python3.12/dist-packages/nanobind/src/nb_func.cpp.o +[73/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefavx2.dir/sleefsimddp.c.o +[74/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefavx2.dir/sleefsimdsp.c.o +[75/153] Generating include/renamepurecfma_scalar.h +Generating renamepurecfma_scalar.h: mkrename finz_ 1 1 purecfma +[76/153] Building CXX object CMakeFiles/nanobind-static.dir/usr/local/lib/python3.12/dist-packages/nanobind/src/nb_type.cpp.o +[77/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetfma4.dir/sleefsimdsp.c.o +[78/153] Linking CXX static library bin/libnanobind-static.a +[79/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetfma4.dir/sleefsimddp.c.o +[80/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetsse4.dir/sleefsimdsp.c.o +[81/153] Generating include/renamecuda.h +Generating renamecuda.h: mkrename finz_ 1 1 cuda +[82/153] Generating sleeflibm_AVX.h.tmp +[83/153] Generating sleeflibm_AVX2.h.tmp +[84/153] Generating sleeflibm_AVX2128.h.tmp +[85/153] Generating sleeflibm_AVX512F.h.tmp +[86/153] Generating sleeflibm_AVX512FNOFMA.h.tmp +[87/153] Generating sleeflibm_AVX512F_.h.tmp +[88/153] Generating sleeflibm_AVX_.h.tmp +[89/153] Generating sleeflibm_DSP_SCALAR.h.tmp +[90/153] Generating sleeflibm_FMA4.h.tmp +[91/153] Generating sleeflibm_PURECFMA_SCALAR.h.tmp +[92/153] Generating sleeflibm_PUREC_SCALAR.h.tmp +[93/153] Generating sleeflibm_SSE2.h.tmp +[94/153] Generating sleeflibm_SSE4.h.tmp +[95/153] Generating sleeflibm_SSE_.h.tmp +[96/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefavx2128.dir/sleefsimdsp.c.o +[97/153] Generating include/renamedspscalar.h +[98/153] Generating ../../include/sleef.h +[99/153] Generating include/renamedsp128.h +[100/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefavx2128.dir/sleefsimddp.c.o +[101/153] Generating include/renamedsp256.h +[102/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetpurec_scalar.dir/sleefsimdsp.c.o +[103/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetavx.dir/sleefsimdsp.c.o +[104/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetavx.dir/sleefsimddp.c.o +[105/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetsse2.dir/sleefsimdsp.c.o +[106/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetpurecfma_scalar.dir/sleefsimdsp.c.o +[107/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetsse4.dir/sleefsimddp.c.o +[108/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleeffma4.dir/sleefsimddp.c.o +[109/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetpurec_scalar.dir/sleefsimddp.c.o +[110/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetpurecfma_scalar.dir/sleefsimddp.c.o +[111/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleeffma4.dir/sleefsimdsp.c.o +[112/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetsse2.dir/sleefsimddp.c.o +[113/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefsse4.dir/sleefsimddp.c.o +[114/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefavx.dir/sleefsimddp.c.o +[115/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefsse4.dir/sleefsimdsp.c.o +[116/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/dispscalar_obj.dir/dispscalar.c.o +[117/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefpurecfma_scalar.dir/sleefsimdsp.c.o +[118/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefpurec_scalar.dir/sleefsimdsp.c.o +[119/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefpurecfma_scalar.dir/sleefsimddp.c.o +[120/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefsse2.dir/sleefsimdsp.c.o +[121/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefsse2.dir/sleefsimddp.c.o +[122/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefavx.dir/sleefsimdsp.c.o +[123/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefpurec_scalar.dir/sleefsimddp.c.o +[124/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleef.dir/rempitab.c.o +[125/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleef.dir/sleefld.c.o +[126/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleef.dir/sleefqp.c.o +[127/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/dispsse_obj.dir/dispsse.c.o +[128/153] Building CXX object CMakeFiles/Deepity.dir/src/RBLayer.cpp.o +[129/153] Building CXX object CMakeFiles/Deepity.dir/src/DiscriminativePCLayer.cpp.o +[130/153] Building CXX object CMakeFiles/Deepity.dir/src/GaussSeidelPCLayer.cpp.o +[131/153] Building CXX object CMakeFiles/Deepity.dir/src/ConvPCLayer.cpp.o +[132/153] Building CXX object CMakeFiles/Deepity.dir/src/DiscriminativePCNetwork.cpp.o +[133/153] Building CXX object CMakeFiles/Deepity.dir/src/SimpleConvPCLayer.cpp.o +[134/153] Building CXX object CMakeFiles/Deepity.dir/src/DirectKPPCLayer.cpp.o +[135/153] Building CXX object CMakeFiles/Deepity.dir/src/ConvPCNetwork.cpp.o +[136/153] Building CXX object CMakeFiles/Deepity.dir/src/GaussSeidelPCNetwork.cpp.o +[137/153] Building CXX object CMakeFiles/Deepity.dir/src/StreamAlignedBatcher.cpp.o +[138/153] Building CXX object CMakeFiles/Deepity.dir/src/SimplePCNetwork.cpp.o +[139/153] Building CXX object CMakeFiles/Deepity.dir/src/SimpleConvPCNetwork.cpp.o +[140/153] Building CXX object CMakeFiles/Deepity.dir/src/SimplePCLayer.cpp.o +[141/153] Building CXX object CMakeFiles/Deepity.dir/src/backend/Tensor.cpp.o +[142/153] Building CXX object CMakeFiles/Deepity.dir/src/DirectKPPCNetwork.cpp.o +[143/153] Building CXX object CMakeFiles/Deepity.dir/src/backend/CPUBackend.cpp.o +[144/153] Building CXX object CMakeFiles/AddBiasBroadcastVerify.dir/tests/tAddBiasBroadcastVerify.cpp.o +[145/153] Building CXX object CMakeFiles/Deepity.dir/src/backend/Backend.cpp.o +[146/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/dispavx_obj.dir/dispavx.c.o +[147/153] Building CXX object CMakeFiles/Deepity.dir/src/ModelIO.cpp.o +[148/153] Linking C static library _deps/sleef-build/lib/libsleef.a +[149/153] Building CUDA object CMakeFiles/Deepity.dir/src/backend/CUDABackend.cu.o +[150/153] Linking CXX static library bin/libDeepity.a +[151/153] Building CXX object CMakeFiles/pydeepity.dir/bindings/pybinding.cpp.o +[152/153] Linking CXX executable bin/AddBiasBroadcastVerify +[153/153] Linking CXX shared module bin/pydeepity.cpython-312-x86_64-linux-gnu.so lto-wrapper: warning: using serial compilation of 6 LTRANS jobs diff --git a/mnist.py b/mnist.py index 879160d..6e6ebdf 100644 --- a/mnist.py +++ b/mnist.py @@ -49,7 +49,7 @@ def train_step_dfa(net, X, Y, inference_steps): net.direct_feedback_update() for _ in range(inference_steps): - net.step() + net.step(False) energy = 0.0 for layer in net.layers: @@ -75,7 +75,7 @@ def main() -> None: DECAY_RATE = 0.94 print(f"\nBuilding network (784->512->512->10), seed={SEED}...") - net = DKPPCN(batch_size=BATCH_SIZE) + net = DKPPCN(batch_size=BATCH_SIZE, device="gpu") net.add_layer(784, 512, TERMINAL_SIZE, lr=LR, ir=IR, fl=FL, lmbda=LMBDA, act="linear") # net.add_layer(512, 512, TERMINAL_SIZE, lr=LR, ir=IR, fl=FL, lmbda=LMBDA, act="sigmoid") net.add_layer(512, TERMINAL_SIZE, TERMINAL_SIZE, lr=LR, ir=IR, fl=FL, lmbda=LMBDA, act="sigmoid") diff --git a/pydeepity/DKPPCN.py b/pydeepity/DKPPCN.py index b0cc519..e4b8845 100644 --- a/pydeepity/DKPPCN.py +++ b/pydeepity/DKPPCN.py @@ -37,8 +37,8 @@ class DKPPCN(dy.DirectKPPCNetwork): until that verification is done. """ - def __init__(self, batch_size: int) -> None: - super().__init__(batch_size) + def __init__(self, batch_size: int, device: str = "cpu") -> None: + super().__init__(batch_size, device) def add_layer( self, diff --git a/pydeepity/__pycache__/DKPPCN.cpython-312.pyc b/pydeepity/__pycache__/DKPPCN.cpython-312.pyc index 9064da17f67df2f9db7fce686bdac30895cb866b..e69e31ab0861878289bb008788e4ec9468356be5 100644 GIT binary patch delta 1438 zcmZWoO>7%Q6y9C$#%ssx|2U0Z6Q?9G8-n9v+&`iaL<2%aq2ebXA6U7bG1{`Rt=Tmq zRS`W{B3wW)5=9(PuK^(;SyfLUP6!DQ;y@285ZpLIg^+UOz41D3)v@NAx9`pOy`Now zuzqtr{)1seC0N<#KD(UGeHj1gotG3zdRMwGQK=ylsi6?L5h4pjp(+Vcji^LxPwh)> znM@6g##BTa|HugvhC4d!2<#{glWEv7*aqx)z)cHo0=PO!20m5zQiE6{LSDJonFeQ+ zJSZ+?z@855EZ8xT1PdAo%z)1j2`9cB_~H?XEb`ywI{#CyX6B$cm4@58(9A<28Hm+1 z;pczI)m+eD0jSxd?v};^=5Y|f%SGtw0oe0&2BspllcY4z8o7PxEx9rKqGT=)*FyVK z%Y?pZx2QkGsMlwm?N$iK9V``2eExTtJGsI$SCubd__$%kntPO)QI_JGQJz1J zS5VExT1w+}jb$*fz@Ijrm$Uq1<7!PP7%ug!ZqKuK?JJZWgjfo%oj?#GIEYdyU@!8m zcn#RG)Or4O{PTx}LJAdnu02sRL)285ActGs-Jb2x17(z0 z%5Nv@qPU)%;BTkqilP-Uy!%_J@L}q7(8^l+ z)zgz!+LV$@&DI+hYkJfS1|kN3ze<&VlZLj2hv}b{Eg=#0SkvxUu19-*xa~AOk9JIL z)Z9rZnLUb7LlANrSon^D3D)+V?wjgedJ`$}nH?#e z;eY3#!C`s!mU2o=Vv_#sQ4QzNdK_ULz|{e;8eA;Hv-yqbEVxYB54C!IzLQ^@iNW)> zDKi{B6oLoaQsN=xDOCHTihS28dJB}a=0V{U@g2>@Ox}y{|{mzE0J#wS=j|9h8A#r@G z{)bP)_Y2?2Fx!`lmFxwe`LRxax3^dE_Ih^bb@nX3UR;^Mc0~$bU*GMLzC$Qj Hs~Gtog%vyK delta 1350 zcmZWoOOM-B6ux%i#L2{t6DQ+52u(ZFhdb>&n1oQ6PK(s4VlzmT1&TziF!!cS8B6ULmPs7PaJ(DeMqLRj zTbZ_!LI*!w#|elh;izy@B!Sa#rr>1YXe24PdBN4Uw6=<~kuTIUKra>ft{)W1fisPd z3sN5JsmM0L&WIvM4HdwriGoMIBKY(a!YA-F>&T1jC#h+|5G2u6g=QHVF~e8_m-u{; z*2-0YRydq&=8(K>S0RVzy#_PY;Ji+XFc<0?_}G@xp58{ENp0&ovS%mF*mfdGi9e=; zHTJ7^v4M7^HCql8Z#*J2P#vc?==qLg$AY*=+%DyhAV@jRy4&$Q$MItDpIuF#ma?p$ zE*A{=`(hLQqiWwIAFHK%mA};4$@l4oY*-MXB@Sf{6}Fc-S<=A_QXPyPx3fX0ou*uX z7FbC;UCDvz@p|=xYm?@AxXx~9?@K28QTuXPC}o#Dvu^?RQax)Tdkj`Lh0!ketu zBlZmYSYHPAUHz$~v(2n!&O%rWa+q_9J;+{woUe?_^BP2LnTkTm^{jkvOH|D9(OlA@ zgWlk#LpwgPqw&N-Umfo}dz5QMD%bKK!HD>|A9vZi0)AD+|RbFxG5|@(n`= z{6<&V!^*o$Gz)g1blqXc4`QBAX`KgWIGo~ehQl0(^Bfk~T=m$U1~*#(LHv{9(4`l- zsYM^cI&oe2NThH{5`->?uWE|3IPg}zJ&3&6LeJ!yqVBIs%E|l7`v`77G@kR@zJZG6 z-Ga4SDD7I+U9 +#include namespace Deep { + namespace + { + // Same mapping as SimplePCLayer.cpp -- see its own comment for + // why this is needed (activationType stores the FORWARD type, + // but ActivationDerivativeInto expects the DERIVATIVE type). + // Note this also closes a real gap the old code had: the + // original UpdateState()'s switch fell back to a raw function + // pointer for eSIGMOID (no dedicated derivative variant existed + // in that dispatch), whereas backend->ActivationDerivativeInto + // DOES have a real d_eSIGMOID case. + ActivationType ToDerivativeType(ActivationType fwd) + { + switch (fwd) + { + case ActivationType::RELU: + return ActivationType::dRELU; + case ActivationType::SIGMOID: + return ActivationType::dSIGMOID; + case ActivationType::eSIGMOID: + return ActivationType::d_eSIGMOID; + case ActivationType::TANH: + return ActivationType::dTANH; + case ActivationType::LINEAR: + return ActivationType::dLINEAR; + default: + return ActivationType::NONE; + } + } + + void DeleteBackend(IComputeBackend *p) { delete p; } + void NoOpDeleter(IComputeBackend *) {} + } + DirectKPPCLayer::DirectKPPCLayer(size_t size, size_t nextSize, size_t terminalSize, size_t batchSize, float learningRate, float inferenceRate, float feedback, float lmbda, - ActivationType aType, ActivationType dType) + ActivationType aType, ActivationType dType, + IComputeBackend *backend) : size(size), nextSize(nextSize), terminalSize(terminalSize), @@ -14,18 +51,36 @@ namespace Deep ir(inferenceRate), fl(feedback), lmbda(lmbda), - activationType(aType) + activationType(aType), + backend(backend ? backend : new CPUBackend(), + backend ? NoOpDeleter : DeleteBackend) { this->activation = To_Fn(aType); this->activationDerivative = To_dFn(dType); + this->activationDerivativeInto = To_dFn2(dType); localArena = std::make_unique(GetRequiredFloats()); BindMemory(*localArena); } + void DirectKPPCLayer::SetLearningRate(float learningRate) noexcept + { + lr = learningRate; + if (lr_device) + backend->CopyFromHost(lr_device, &lr, 1); + } + + void DirectKPPCLayer::SetFeedbackRate(float feedbackRate) noexcept + { + fl = feedbackRate; + if (fl_device) + backend->CopyFromHost(fl_device, &fl, 1); + } + // --- Setup --- - void DirectKPPCLayer::BindMemory(MemoryArena &arena) + template + void DirectKPPCLayer::BindMemory(ArenaT &arena) { size_t own_state_size = batchSize * size; size_t out_state_size = batchSize * nextSize; @@ -34,8 +89,8 @@ namespace Deep z = arena.AllocateFloats(own_state_size); e = arena.AllocateFloats(own_state_size); - std::memset(z, 0, own_state_size * sizeof(float)); - std::memset(e, 0, own_state_size * sizeof(float)); + backend->Zero(z, own_state_size); + backend->Zero(e, own_state_size); if (nextSize > 0) { @@ -51,14 +106,14 @@ namespace Deep zFDeriv = arena.AllocateFloats(own_state_size); feedbackScratch = arena.AllocateFloats(own_state_size); - std::memset(b, 0, nextSize * sizeof(float)); - std::memset(mu, 0, out_state_size * sizeof(float)); - std::memset(cachedMu, 0, out_state_size * sizeof(float)); - std::memset(proj, 0, out_state_size * sizeof(float)); - std::memset(Psi, 0, direct_size * sizeof(float)); - std::memset(zF, 0, own_state_size * sizeof(float)); - std::memset(zFDeriv, 0, own_state_size * sizeof(float)); - std::memset(feedbackScratch, 0, own_state_size * sizeof(float)); + backend->Zero(b, nextSize); + backend->Zero(mu, out_state_size); + backend->Zero(cachedMu, out_state_size); + backend->Zero(proj, out_state_size); + backend->Zero(Psi, direct_size); + backend->Zero(zF, own_state_size); + backend->Zero(zFDeriv, own_state_size); + backend->Zero(feedbackScratch, own_state_size); if (opt == OptimizerType::ADAM || opt == OptimizerType::ADAMW) { @@ -70,25 +125,47 @@ namespace Deep m_b = arena.AllocateFloats(nextSize); v_b = arena.AllocateFloats(nextSize); + backend->Zero(m_W, w_size); + backend->Zero(v_W, w_size); + backend->Zero(m_b, nextSize); + backend->Zero(v_b, nextSize); + backend->Zero(grad_W, w_size); + backend->Zero(grad_b, nextSize); + + // Device-resident t/lr for W's optimizer -- see header's + // file-level note for why these can't just be host + // values once graph capture is in the picture. + t_device = reinterpret_cast(arena.AllocateFloats(1)); + lr_device = arena.AllocateFloats(1); + int zero = 0; + backend->CopyFromHost(reinterpret_cast(t_device), reinterpret_cast(&zero), 1); + backend->CopyFromHost(lr_device, &lr, 1); + } + + if (optPsi == OptimizerType::ADAM || optPsi == OptimizerType::ADAMW) + { grad_Psi = arena.AllocateFloats(direct_size); m_Psi = arena.AllocateFloats(direct_size); v_Psi = arena.AllocateFloats(direct_size); - std::memset(m_W, 0, w_size * sizeof(float)); - std::memset(v_W, 0, w_size * sizeof(float)); - std::memset(m_Psi, 0, direct_size * sizeof(float)); + backend->Zero(m_Psi, direct_size); + backend->Zero(v_Psi, direct_size); + backend->Zero(grad_Psi, direct_size); - std::memset(m_b, 0, nextSize * sizeof(float)); - std::memset(v_b, 0, nextSize * sizeof(float)); - std::memset(v_Psi, 0, direct_size * sizeof(float)); - - std::memset(grad_W, 0, w_size * sizeof(float)); - std::memset(grad_b, 0, nextSize * sizeof(float)); - std::memset(grad_Psi, 0, direct_size * sizeof(float)); + // Same as above, but for Psi's SEPARATE optimizer state. + tPsi_device = reinterpret_cast(arena.AllocateFloats(1)); + fl_device = arena.AllocateFloats(1); + int zero = 0; + backend->CopyFromHost(reinterpret_cast(tPsi_device), reinterpret_cast(&zero), 1); + backend->CopyFromHost(fl_device, &fl, 1); } } - - if (localArena && localArena.get() != &arena) + if constexpr (std::is_same_v) + { + if (localArena && localArena.get() != &arena) + localArena.reset(); + } + else { localArena.reset(); } @@ -103,7 +180,6 @@ namespace Deep size_t own_state_size = batchSize * size; size_t direct_size = terminalSize * size; - // dz_dt removed: state size drops from 3 arrays to 2 total += pad16(own_state_size) * 2; if (nextSize > 0) @@ -114,17 +190,21 @@ namespace Deep total += pad16(w_size); total += pad16(nextSize); total += pad16(out_state_size) * 3; - // E, prevZ, and bottom_up removed total += pad16(own_state_size) * 3; // zF, zFDeriv, feedbackScratch + total += pad16(direct_size); // Psi if (opt == OptimizerType::ADAM || opt == OptimizerType::ADAMW) { total += pad16(w_size) * 3; total += pad16(nextSize) * 3; - total += pad16(direct_size) * 3; + total += pad16(1) * 2; // t_device, lr_device } - total += pad16(direct_size); + if (optPsi == OptimizerType::ADAM || optPsi == OptimizerType::ADAMW) + { + total += pad16(direct_size) * 3; + total += pad16(1) * 2; // tPsi_device, fl_device + } } return total; @@ -137,108 +217,39 @@ namespace Deep std::uniform_int_distribution seedDist; size_t Wsz = size * nextSize; + size_t Psisz = terminalSize * size; float limit = std::sqrt(2.0f / (size + nextSize)); float limPsi = std::sqrt(2.0f / (size + terminalSize)); - std::vector seeds(omp_get_max_threads()); - for (auto &s : seeds) - s = seedDist(seedGenerator); - -#pragma omp parallel if (!omp_in_parallel()) - { - std::mt19937 rng(seeds[omp_get_thread_num()]); - std::normal_distribution dist1(0.0f, limit); - std::normal_distribution dist2(0.0f, limPsi); + uint32_t seedW = seedDist(seedGenerator); + uint32_t seedPsi = seedDist(seedGenerator); -#pragma omp for - for (ptrdiff_t i = 0; i < (ptrdiff_t)Wsz; ++i) - W[i] = dist1(rng); - -#pragma omp for - for (ptrdiff_t i = 0; i < (ptrdiff_t)(terminalSize * size); ++i) - Psi[i] = dist2(rng); - } + backend->RandomizeNormal(W, Wsz, 0.0f, limit, seedW); + backend->RandomizeNormal(Psi, Psisz, 0.0f, limPsi, seedPsi); } // --- Core DKP-PC Mechanics --- - float DirectKPPCLayer::CalculateState() noexcept + float DirectKPPCLayer::CalculateState(bool needEnergy) noexcept { const size_t N = batchSize * size; if (layerBelow == nullptr) { - std::memset(e, 0, N * sizeof(float)); + backend->Zero(e, N); if (nextSize > 0) - { ComputeMuOnly(); - } return 0.0f; } - cblas_scopy(N, z, 1, e, 1); - cblas_saxpy(N, -1.0f, layerBelow->mu, 1, e, 1); - float totalEnergy = 0.0f; + if (needEnergy) + totalEnergy = backend->ComputeErrorAndEnergy(e, z, layerBelow->mu, N); + else + backend->ComputeError(e, z, layerBelow->mu, N); -#pragma omp parallel for schedule(static) reduction(+ : totalEnergy) if (batchSize > 4 && !omp_in_parallel()) - for (int batch = 0; batch < batchSize; ++batch) - { - const size_t offset = (size_t)batch * size; - size_t i = 0; - -#if defined(__AVX512F__) - __m512 half = _mm512_set1_ps(0.5f); - __m512 energy = _mm512_setzero_ps(); - size_t r = size % 16; - size_t simd_end = size - r; - for (; i < simd_end; i += 16) - { - __m512 e512 = _mm512_loadu_ps(&e[offset + i]); - energy = _mm512_fmadd_ps(half, _mm512_mul_ps(e512, e512), energy); - } - totalEnergy += _mm512_reduce_add_ps(energy); -#elif defined(__AVX2__) || defined(__AVX__) - __m256 half = _mm256_set1_ps(0.5f); - __m256 energy = _mm256_setzero_ps(); - size_t r = size % 8; - size_t simd_end = size - r; - for (; i < simd_end; i += 8) - { - __m256 e256 = _mm256_loadu_ps(&e[offset + i]); -#ifdef __FMA__ - energy = _mm256_fmadd_ps(half, _mm256_mul_ps(e256, e256), energy); -#else - energy = _mm256_add_ps(energy, _mm256_mul_ps(half, _mm256_mul_ps(e256, e256))); -#endif - } - totalEnergy += hsum256_ps(energy); -#elif defined(__SSE__) || defined(_M_AMD64) || defined(_M_X64) - __m128 half = _mm_set1_ps(0.5f); - __m128 energy = _mm_setzero_ps(); - size_t r = size % 4; - size_t simd_end = size - r; - for (; i < simd_end; i += 4) - { - __m128 e128 = _mm_loadu_ps(&e[offset + i]); -#ifdef __FMA__ - energy = _mm_fmadd_ps(half, _mm_mul_ps(e128, e128), energy); -#else - energy = _mm_add_ps(energy, _mm_mul_ps(half, _mm_mul_ps(e128, e128))); -#endif - } - totalEnergy += hsum128_ps(energy); -#endif - for (; i < size; ++i) - { - float err = e[offset + i]; - totalEnergy += 0.5f * err * err; - } - } if (nextSize > 0) - { ComputeMuOnly(); - } return totalEnergy; } @@ -253,47 +264,26 @@ namespace Deep if (isClamped && muCacheValid) { - cblas_scopy((int)Nout, cachedMu, 1, mu, 1); + backend->Copy(mu, cachedMu, Nout); return; } - cblas_scopy((int)N, z, 1, zF, 1); - switch (activationType) - { - case ActivationType::RELU: - Deep::relu(zF, N); - break; - case ActivationType::SIGMOID: - Deep::sigmoid(zF, N); - break; - case ActivationType::eSIGMOID: - Deep::e_sigmoid(zF, N); - break; - case ActivationType::TANH: - Deep::tanh(zF, N); - break; - case ActivationType::LINEAR: - Deep::linear(zF, N); - break; - default: - activation(zF, N); - break; // custom/unrecognized function pointer - } + backend->ActivationInto(activationType, zF, z, N); - cblas_sgemm( - CblasRowMajor, CblasNoTrans, CblasTrans, - batchSize, nextSize, size, - 1.0f, zF, size, W, size, 0.0f, mu, nextSize); + backend->MatMul( + /*transA=*/false, /*transB=*/true, + (int)batchSize, (int)nextSize, (int)size, + 1.0f, zF, (int)size, W, (int)size, 0.0f, mu, (int)nextSize); #pragma omp parallel for schedule(static) if (batchSize > 4 && !omp_in_parallel()) for (int batch = 0; batch < batchSize; ++batch) { - cblas_saxpy(nextSize, 1.0f, b, 1, mu + batch * nextSize, 1); + backend->AxpyInto(mu + batch * nextSize, b, nextSize, 1.0f); } if (isClamped) { - cblas_scopy((int)Nout, mu, 1, cachedMu, 1); + backend->Copy(cachedMu, mu, Nout); muCacheValid = true; } } @@ -309,61 +299,19 @@ namespace Deep { const float *e_above = layerAbove->GetErrors(); - switch (activationType) - { - case ActivationType::RELU: - Deep::dReluInto(zFDeriv, z, N); - break; - case ActivationType::SIGMOID: - Deep::dSigmoidInto(zFDeriv, z, N); - break; - case ActivationType::TANH: - Deep::dTanhInto(zFDeriv, z, N); - break; - case ActivationType::LINEAR: - Deep::dLinearInto(zFDeriv, z, N); - break; - default: - activationDerivativeInto(zFDeriv, z, N); - break; // custom/unrecognized, or eSIGMOID (no dedicated derivative variant exists) - } + backend->ActivationDerivativeInto(ToDerivativeType(activationType), zFDeriv, z, N); - cblas_sgemm( - CblasRowMajor, CblasNoTrans, CblasNoTrans, - batchSize, size, nextSize, - 1.0f, e_above, nextSize, W, size, - 0.0f, feedbackScratch, size); + backend->MatMul( + /*transA=*/false, /*transB=*/false, + (int)batchSize, (int)size, (int)nextSize, + 1.0f, e_above, (int)nextSize, W, (int)size, + 0.0f, feedbackScratch, (int)size); -#pragma omp parallel for schedule(static) if (batchSize > 4 && !omp_in_parallel()) - for (int batch = 0; batch < batchSize; ++batch) - { - float *RESTRICT zPtr = z; - const float *RESTRICT feedbackPtr = feedbackScratch; - const float *RESTRICT zFDerivPtr = zFDeriv; - const float *RESTRICT ePtr = e; - size_t offset = (size_t)batch * size; - for (size_t i = 0; i < size; ++i) - { - size_t idx = offset + i; - // Fused dz_dt directly into the z update to save memory writes - zPtr[idx] += ir * ((feedbackPtr[idx] * zFDerivPtr[idx]) - ePtr[idx]); - } - } + backend->FusedStateUpdate(z, feedbackScratch, zFDeriv, e, N, ir); } else // Output Layer { -#pragma omp parallel for schedule(static) if (batchSize > 4 && !omp_in_parallel()) - for (int batch = 0; batch < batchSize; ++batch) - { - float *RESTRICT zPtr = z; - const float *RESTRICT ePtr = e; - size_t offset = (size_t)batch * size; - for (size_t i = 0; i < size; ++i) - { - size_t idx = offset + i; - zPtr[idx] += ir * -ePtr[idx]; - } - } + backend->AxpyInto(z, e, N, -ir); } } @@ -373,51 +321,50 @@ namespace Deep return; const float *local_grad = layerAbove->GetErrors(); - float grad_scale = -1.0f / batchSize; switch (opt) { case OptimizerType::SGD: { if (lmbda > 0.0f) - cblas_sscal((size_t)nextSize * size, 1.0f - lmbda, W, 1); + backend->Scale(W, (size_t)nextSize * size, 1.0f - lmbda); - cblas_sgemm( - CblasRowMajor, CblasTrans, CblasNoTrans, - nextSize, size, batchSize, - lr / batchSize, local_grad, nextSize, zF, size, - 1.0f, W, size); + backend->MatMul( + /*transA=*/true, /*transB=*/false, + (int)nextSize, (int)size, (int)batchSize, + lr / batchSize, local_grad, (int)nextSize, zF, (int)size, + 1.0f, W, (int)size); float lr_batch = lr / batchSize; for (int batch = 0; batch < batchSize; batch++) - cblas_saxpy(nextSize, lr_batch, local_grad + batch * nextSize, 1, b, 1); + backend->AxpyInto(b, local_grad + batch * nextSize, nextSize, lr_batch); break; } case OptimizerType::ADAM: case OptimizerType::ADAMW: { - t++; + backend->IncrementCounter(t_device); size_t num_weights = (size_t)nextSize * size; float adam_scale = -1.0f; - cblas_sgemm( - CblasRowMajor, CblasTrans, CblasNoTrans, - nextSize, size, batchSize, - adam_scale, local_grad, nextSize, zF, size, - 0.0f, grad_W, size); + backend->MatMul( + /*transA=*/true, /*transB=*/false, + (int)nextSize, (int)size, (int)batchSize, + adam_scale, local_grad, (int)nextSize, zF, (int)size, + 0.0f, grad_W, (int)size); - std::memset(grad_b, 0, nextSize * sizeof(float)); + backend->Zero(grad_b, nextSize); for (int batch = 0; batch < batchSize; batch++) - cblas_saxpy(nextSize, adam_scale, local_grad + batch * nextSize, 1, grad_b, 1); + backend->AxpyInto(grad_b, local_grad + batch * nextSize, nextSize, adam_scale); if (opt == OptimizerType::ADAMW) - Deep::AdamWUpdate(W, grad_W, m_W, v_W, num_weights, t, lr, lmbda); + backend->AdamWStep(W, grad_W, m_W, v_W, num_weights, t_device, lr_device, lmbda); else - Deep::AdamUpdate(W, grad_W, m_W, v_W, num_weights, t, lr); + backend->AdamStep(W, grad_W, m_W, v_W, num_weights, t_device, lr_device); - Deep::AdamUpdate(b, grad_b, m_b, v_b, nextSize, t, lr); + backend->AdamStep(b, grad_b, m_b, v_b, nextSize, t_device, lr_device); break; } } @@ -426,31 +373,31 @@ namespace Deep { case OptimizerType::SGD: { - cblas_sgemm( - CblasRowMajor, CblasTrans, CblasNoTrans, - size, terminalSize, batchSize, - fl / batchSize, z, size, terminalLayer->GetErrors(), terminalSize, - 1.0f, Psi, terminalSize); + backend->MatMul( + /*transA=*/true, /*transB=*/false, + (int)size, (int)terminalSize, (int)batchSize, + fl / batchSize, z, (int)size, terminalLayer->GetErrors(), (int)terminalSize, + 1.0f, Psi, (int)terminalSize); break; } case OptimizerType::ADAMW: case OptimizerType::ADAM: { - tPsi++; + backend->IncrementCounter(tPsi_device); size_t num_weights_psi = size * terminalSize; float adam_scale = -1.0f; - cblas_sgemm( - CblasRowMajor, CblasTrans, CblasNoTrans, - size, terminalSize, batchSize, - adam_scale, z, size, terminalLayer->GetErrors(), terminalSize, - 0.0f, grad_Psi, terminalSize); + backend->MatMul( + /*transA=*/true, /*transB=*/false, + (int)size, (int)terminalSize, (int)batchSize, + adam_scale, z, (int)size, terminalLayer->GetErrors(), (int)terminalSize, + 0.0f, grad_Psi, (int)terminalSize); if (optPsi == OptimizerType::ADAMW) - Deep::AdamWUpdate(Psi, grad_Psi, m_Psi, v_Psi, num_weights_psi, tPsi, fl, lmbda); + backend->AdamWStep(Psi, grad_Psi, m_Psi, v_Psi, num_weights_psi, tPsi_device, fl_device, lmbda); else - Deep::AdamUpdate(Psi, grad_Psi, m_Psi, v_Psi, num_weights_psi, tPsi, fl); + backend->AdamStep(Psi, grad_Psi, m_Psi, v_Psi, num_weights_psi, tPsi_device, fl_device); break; } @@ -464,28 +411,28 @@ namespace Deep return; // proj = terminalLayer->GetErrors() @ layerAbove->GetDirectFeedbackWeights()^T - cblas_sgemm( - CblasRowMajor, CblasNoTrans, CblasTrans, - batchSize, nextSize, terminalSize, - 1.0f, terminalLayer->GetErrors(), terminalSize, - layerAbove->GetDirectFeedbackWeights(), terminalSize, - 0.0f, proj, nextSize); - + backend->MatMul( + /*transA=*/false, /*transB=*/true, + (int)batchSize, (int)nextSize, (int)terminalSize, + 1.0f, terminalLayer->GetErrors(), (int)terminalSize, + layerAbove->GetDirectFeedbackWeights(), (int)terminalSize, + 0.0f, proj, (int)nextSize); + // W += fl * proj^T @ zF - cblas_sgemm( - CblasRowMajor, CblasTrans, CblasTrans, - nextSize, size, batchSize, - fl / batchSize, proj, nextSize, - zF, size, - 1.0f, W, size); + backend->MatMul( + /*transA=*/true, /*transB=*/true, + (int)nextSize, (int)size, (int)batchSize, + fl / batchSize, proj, (int)nextSize, + zF, (int)size, + 1.0f, W, (int)size); } // --- Getters / Setters --- void DirectKPPCLayer::ClampState(const std::vector &inputData) noexcept { - size_t copySize = (std::min)(inputData.size(), (size_t)(batchSize * size)) * sizeof(float); - memcpy(z, inputData.data(), copySize); + size_t copyFloats = (std::min)(inputData.size(), (size_t)(batchSize * size)); + backend->CopyFromHost(z, inputData.data(), copyFloats); isClamped = true; muCacheValid = false; } @@ -498,6 +445,11 @@ namespace Deep void DirectKPPCLayer::ResetState() noexcept { size_t N = (size_t)batchSize * size; - std::memset(z, 0, N * sizeof(float)); + backend->Zero(z, N); } -} + + template void DirectKPPCLayer::BindMemory(MemoryArena &arena); +#if defined(DEEPITY_USE_CUDA) + template void DirectKPPCLayer::BindMemory(DeviceMemoryArena &arena); +#endif +} \ No newline at end of file diff --git a/src/DirectKPPCNetwork.cpp b/src/DirectKPPCNetwork.cpp index e9c7db7..4d9acdf 100644 --- a/src/DirectKPPCNetwork.cpp +++ b/src/DirectKPPCNetwork.cpp @@ -1,20 +1,23 @@ #include -#ifdef DEEPITY_USE_MKL -#include -#else -#include -#endif +#include +#include +#include +#include namespace Deep { - DirectKPPCNetwork::DirectKPPCNetwork(int batchSize) noexcept - : batchSize(batchSize) {} + DirectKPPCNetwork::DirectKPPCNetwork(int batchSize, DeviceType device) noexcept + : device(device), batchSize(batchSize) + { + backend = CreateBackend(device); + } void DirectKPPCNetwork::AddLayer(size_t size, size_t nextSize, size_t terminalSize, float lr, float ir, float fl, float lmbda, ActivationType aType, ActivationType dType) { - std::unique_ptr l = std::make_unique(size, nextSize, terminalSize, batchSize, lr, ir, fl, lmbda, aType, dType); + std::unique_ptr l = std::make_unique( + size, nextSize, terminalSize, batchSize, lr, ir, fl, lmbda, aType, dType, backend.get()); if (!layers.empty()) { @@ -52,29 +55,33 @@ namespace Deep float *nextZ = layers[i + 1]->GetBeliefs(); size_t n = layers[i]->GetBatchSize() * layers[i]->GetOutputSize(); - std::memcpy(nextZ, mu, n * sizeof(float)); + // Was: std::memcpy(nextZ, mu, n * sizeof(float)) -- wrong if + // mu/nextZ are device pointers (DEVICE_GPU). backend->Copy() + // is device-to-device, matching SimplePCNetwork's own fix + // for the identical bug. + backend->Copy(nextZ, mu, n); } } - float DirectKPPCNetwork::CalculateTerminalError() noexcept - { - return GetTerminalLayer()->CalculateState(); - } - void DirectKPPCNetwork::DirectFeedbackUpdate() noexcept { for (size_t i = 0; i < layers.size() - 1; ++i) layers[i]->DirectFeedbackUpdate(); } - float DirectKPPCNetwork::Step() noexcept + float DirectKPPCNetwork::CalculateTerminalError() noexcept + { + return GetTerminalLayer()->CalculateState(false); // only the error buffer matters here + } + + float DirectKPPCNetwork::Step(bool needEnergy) noexcept { float e = 0.0f; for (auto &l : layers) - e += l->CalculateState(); + e += l->CalculateState(needEnergy); for (auto &l : layers) l->UpdateState(); - return e; + return needEnergy ? e : 0.0f; } void DirectKPPCNetwork::UpdateWeights() noexcept @@ -87,30 +94,64 @@ namespace Deep const std::vector &y, int inferenceSteps) { + printf("TrainStep: device=%s, graphCaptured=%d\n", + (device == DeviceType::DEVICE_GPU ? "GPU" : "CPU"), (int)graphCaptured); + fflush(stdout); ResetState(); Clamp(x); ProjectForward(); GetTerminalLayer()->ClampState(y); - CalculateTerminalError(); - DirectFeedbackUpdate(); - - for (int t = 0; t < inferenceSteps; t++) - Step(); + if (device == DeviceType::DEVICE_GPU) + { + if (!graphCaptured || capturedInferenceSteps != inferenceSteps) + { + backend->BeginGraphCapture(); + CalculateTerminalError(); + DirectFeedbackUpdate(); + for (int t = 0; t < inferenceSteps; t++) + Step(false); + UpdateWeights(); + bool captureOk = backend->EndGraphCapture(); + + if (captureOk) + { + graphCaptured = true; + capturedInferenceSteps = inferenceSteps; + } + else + { + std::cerr << "Graph capture failed -- falling back to non-graph execution for this call.\n"; + } + } + + if (graphCaptured) + { + backend->ReplayGraph(); + } + else + { + CalculateTerminalError(); + DirectFeedbackUpdate(); + for (int t = 0; t < inferenceSteps; t++) + Step(false); + UpdateWeights(); + } + } + else + { + CalculateTerminalError(); + DirectFeedbackUpdate(); + for (int t = 0; t < inferenceSteps; t++) + Step(false); + UpdateWeights(); + } - // Sync e/mu/zF to the TRUE final z before UpdateWeights() reads them. - // Step()'s own CalculateState() each iteration reflects z BEFORE that - // iteration's UpdateState() moves it, so after the loop exits, e/mu/zF - // are one iteration stale relative to the final z -- exactly the bug - // the test's added TotalEnergy() call worked around. This also gives a - // more accurate finalEnergy for free, computed at the true final state. float finalEnergy = 0.0f; for (auto &l : layers) - finalEnergy += l->CalculateState(); + finalEnergy += l->CalculateState(true); - UpdateWeights(); GetTerminalLayer()->UnclampState(); - return finalEnergy; } @@ -130,7 +171,13 @@ namespace Deep const float *beliefs = terminal->GetBeliefs(); size_t count = terminal->GetBatchSize() * terminal->GetInputSize(); - return std::vector(beliefs, beliefs + count); + // Was: std::vector(beliefs, beliefs + count) -- the + // iterator-range constructor dereferences every element + // directly, wrong if beliefs is a device pointer. Allocate the + // host-side result first, then copy it out through the backend. + std::vector result(count); + backend->CopyToHost(result.data(), beliefs, count); + return result; } void DirectKPPCNetwork::Compile() @@ -144,12 +191,26 @@ namespace Deep for (auto &layer : layers) total_floats_needed += layer->GetRequiredFloats(); - arena = std::make_unique(total_floats_needed, false); // TODO: Consider adding huge pages as a param - - for (auto &layer : layers) + if (device == DeviceType::DEVICE_CPU) { - layer->BindMemory(*arena); - layer->SetTerminalLayer(layers.back().get()); + cpuArena = std::make_unique(total_floats_needed, false); // TODO: Consider adding huge pages as a param + for (auto &layer : layers) + { + layer->BindMemory(*cpuArena); + layer->SetTerminalLayer(layers.back().get()); + } } +#if defined(DEEPITY_USE_CUDA) + else + { + backend->PrepareForBatchSize(batchSize); + gpuArena = std::make_unique(backend.get(), total_floats_needed); + for (auto &layer : layers) + { + layer->BindMemory(*gpuArena); + layer->SetTerminalLayer(layers.back().get()); + } + } +#endif } -} +} \ No newline at end of file diff --git a/src/SimplePCNetwork.cpp b/src/SimplePCNetwork.cpp index b0474b5..baebb83 100644 --- a/src/SimplePCNetwork.cpp +++ b/src/SimplePCNetwork.cpp @@ -129,6 +129,9 @@ namespace Deep { layers[i]->ComputeMuOnly(); + if (layers[i + 1]->IsClamped()) + continue; // never overwrite a clamped layer's real target with a forward guess + const float *mu = layers[i]->GetMu(); float *nextZ = layers[i + 1]->GetBeliefs(); size_t n = layers[i]->GetBatchSize() * layers[i]->GetOutputSize(); @@ -137,43 +140,54 @@ namespace Deep } } - float SimplePCNetwork::TrainStepWithProjection(const std::vector &x, const std::vector &y, int inferenceSteps) + float SimplePCNetwork::TrainStepWithProjection(const std::vector &x, const std::vector &y, int inferenceSteps, bool computeEnergy) { ResetState(); Clamp(x); - ProjectForward(); GetTerminalLayer()->ClampState(y); if (device == DeviceType::DEVICE_GPU) { if (!graphCaptured || capturedInferenceSteps != inferenceSteps) { - // Warm-up pass, uncaptured: runs the exact same operation - // sequence once so any lazy cuBLAS internal workspace - // allocation (a documented cuBLAS + CUDA Graphs interaction) - // happens before capture begins, not during it. + backend->BeginGraphCapture(); + ProjectForward(); for (int t = 0; t < inferenceSteps; ++t) { CalculateState(false); UpdateState(); } UpdateWeights(); - std::cerr << "=== Warm-up pass complete, beginning capture ===\n"; - backend->BeginGraphCapture(); + bool captureOk = backend->EndGraphCapture(); + + if (captureOk) + { + graphCaptured = true; + capturedInferenceSteps = inferenceSteps; + } + else + { + std::cerr << "Graph capture failed -- falling back to non-graph execution for this call.\n"; + } + } + + if (graphCaptured) + { + backend->ReplayGraph(); + } + else + { for (int t = 0; t < inferenceSteps; ++t) { CalculateState(false); UpdateState(); } UpdateWeights(); - backend->EndGraphCapture(); - graphCaptured = true; - capturedInferenceSteps = inferenceSteps; } - backend->ReplayGraph(); } else { + ProjectForward(); for (int t = 0; t < inferenceSteps; ++t) { CalculateState(false); @@ -182,7 +196,7 @@ namespace Deep UpdateWeights(); } - float finalEnergy = CalculateState(true); + float finalEnergy = CalculateState(computeEnergy); GetTerminalLayer()->UnclampState(); return finalEnergy; @@ -218,7 +232,7 @@ namespace Deep void SimplePCNetwork::Compile() { #pragma omp parallel - { // Broadcast FTZ/DAZ hardware flags to ALL OpenMP worker threads + { _MM_SET_FLUSH_ZERO_MODE(_MM_FLUSH_ZERO_ON); _MM_SET_DENORMALS_ZERO_MODE(_MM_DENORMALS_ZERO_ON); } @@ -226,6 +240,8 @@ namespace Deep for (auto &layer : layers) total_floats_needed += layer->GetRequiredFloats(); + backend->PrepareForBatchSize(batchSize); + if (device == DeviceType::DEVICE_CPU) { cpuArena = std::make_unique(total_floats_needed, true); diff --git a/src/backend/CPUBackend.cpp b/src/backend/CPUBackend.cpp index f0f3cfb..d21a6aa 100644 --- a/src/backend/CPUBackend.cpp +++ b/src/backend/CPUBackend.cpp @@ -100,6 +100,13 @@ namespace Deep M, N, K, alpha, A, lda, B, ldb, beta, C, ldc); } + void CPUBackend::SumRows(float *dst, const float *src, size_t batchSize, size_t width) noexcept + { + memset(dst, 0, width * sizeof(float)); + for (size_t b = 0; b < batchSize; ++b) + cblas_saxpy(width, 1.0f, src + b * width, 1, dst, 1); + } + void CPUBackend::Scale(float *buf, size_t n, float alpha) noexcept { cblas_sscal(n, alpha, buf, 1); diff --git a/src/backend/CUDABackend.cu b/src/backend/CUDABackend.cu index e8e869d..86e0f29 100644 --- a/src/backend/CUDABackend.cu +++ b/src/backend/CUDABackend.cu @@ -11,35 +11,47 @@ namespace Deep cudaStreamCreate(&this->stream); cublasCreate(&this->handle); cublasSetStream(this->handle, this->stream); + + // constexpr size_t WORKSPACE_SIZE = 4 * 1024 * 1024; + // cudaMalloc(&workspace, WORKSPACE_SIZE); + // cublasSetWorkspace(handle, workspace, WORKSPACE_SIZE); } CUDABackend::~CUDABackend() { + if (hasGraph) + { + cudaGraphExecDestroy(graphExec); + cudaGraphDestroy(graph); + } + if (workspace) + cudaFree(workspace); cublasDestroy(this->handle); cudaStreamDestroy(this->stream); } void CUDABackend::BeginGraphCapture() noexcept { - cudaStreamBeginCapture(stream, cudaStreamCaptureModeThreadLocal); + cudaError_t err = cudaStreamBeginCapture(stream, cudaStreamCaptureModeThreadLocal); + if (err != cudaSuccess) + std::cerr << "cudaStreamBeginCapture failed: " << cudaGetErrorString(err) << "\n"; } - void CUDABackend::EndGraphCapture() noexcept + bool CUDABackend::EndGraphCapture() noexcept { cudaGraph_t newGraph; cudaError_t err = cudaStreamEndCapture(stream, &newGraph); if (err != cudaSuccess) { std::cerr << "cudaStreamEndCapture failed: " << cudaGetErrorString(err) << "\n"; - return; + return false; } if (hasGraph) { - // Re-capturing (e.g. batch size or settling-step count changed) - // -- destroy the previous instantiated graph first. cudaGraphExecDestroy(graphExec); cudaGraphDestroy(graph); + hasGraph = false; } graph = newGraph; @@ -47,11 +59,13 @@ namespace Deep if (err != cudaSuccess) { std::cerr << "cudaGraphInstantiate failed: " << cudaGetErrorString(err) << "\n"; - hasGraph = false; - return; + cudaGraphDestroy(graph); + graph = nullptr; + return false; } hasGraph = true; + return true; } void CUDABackend::ReplayGraph() noexcept @@ -61,7 +75,9 @@ namespace Deep std::cerr << "ReplayGraph() called before any graph was captured.\n"; return; } - cudaGraphLaunch(graphExec, stream); + cudaError_t err = cudaGraphLaunch(graphExec, stream); + if (err != cudaSuccess) + std::cerr << "cudaGraphLaunch failed: " << cudaGetErrorString(err) << "\n"; } float *CUDABackend::Allocate(size_t numFloats) @@ -83,12 +99,12 @@ namespace Deep void CUDABackend::Zero(float *ptr, size_t numFloats) noexcept { - cudaMemset(ptr, 0, numFloats * sizeof(float)); + cudaMemsetAsync(ptr, 0, numFloats * sizeof(float), stream); } void CUDABackend::Copy(float *dst, const float *src, size_t numFloats) noexcept { - cudaMemcpy(dst, src, numFloats * sizeof(float), cudaMemcpyDefault); + cudaMemcpyAsync(dst, src, numFloats * sizeof(float), cudaMemcpyDefault, stream); } void CUDABackend::CopyFromHost(float *deviceDst, const float *hostSrc, size_t numFloats) noexcept @@ -133,7 +149,8 @@ namespace Deep constexpr int BLOCK_SIZE = 256; const int blocks = static_cast((n + BLOCK_SIZE - 1) / BLOCK_SIZE); - normal_generation<<>>(state, buf, n, mean, stddev, seed); + normal_generation<<>>(state, buf, n, mean, stddev, seed); + cudaStreamSynchronize(stream); cudaFree(state); } @@ -151,17 +168,16 @@ namespace Deep << cudaGetErrorString(mallocErr) << "\n"; return; } - std::cerr << "curandState allocated OK: n=" << n << ", ptr=" << (void *)state << "\n"; constexpr int BLOCK_SIZE = 256; const int blocks = static_cast((n + BLOCK_SIZE - 1) / BLOCK_SIZE); - uniform_generation<<>>(state, buf, n, min, max - min, seed); + uniform_generation<<>>(state, buf, n, min, max - min, seed); cudaError_t launchErr = cudaGetLastError(); if (launchErr != cudaSuccess) std::cerr << "uniform_generation kernel launch failed: " << cudaGetErrorString(launchErr) << "\n"; - cudaDeviceSynchronize(); // force the kernel to finish and surface any async error + cudaStreamSynchronize(stream); cudaError_t syncErr = cudaGetLastError(); if (syncErr != cudaSuccess) std::cerr << "uniform_generation kernel execution failed: " << cudaGetErrorString(syncErr) << "\n"; @@ -169,19 +185,49 @@ namespace Deep cudaFree(state); } + __global__ void FillOnesKernel(float *buf, size_t n) + { + size_t i = (size_t)blockIdx.x * blockDim.x + threadIdx.x; + if (i < n) + buf[i] = 1.0f; + } + + void CUDABackend::PrepareForBatchSize(size_t batchSize) noexcept + { + if (onesVector) + cudaFree(onesVector); + cudaMalloc(&onesVector, batchSize * sizeof(float)); + constexpr int BLOCK_SIZE = 256; + const int blocks = static_cast((batchSize + BLOCK_SIZE - 1) / BLOCK_SIZE); + FillOnesKernel<<>>(onesVector, batchSize); + cudaStreamSynchronize(stream); + } + void CUDABackend::MatMul(bool transA, bool transB, int M, int N, int K, float alpha, const float *A, int lda, const float *B, int ldb, float beta, float *C, int ldc) noexcept { - // CUDA is *column-major*. cublasOperation_t cuTransA = transA ? CUBLAS_OP_T : CUBLAS_OP_N; cublasOperation_t cuTransB = transB ? CUBLAS_OP_T : CUBLAS_OP_N; - cublasStatus_t status = cublasSgemm(handle, cuTransB, cuTransA, N, M, K, &alpha, B, ldb, A, lda, &beta, C, ldc); if (status != CUBLAS_STATUS_SUCCESS) - std::cerr << "cublasSgemm status: " << status << "\n"; + { + cudaError_t cudaErr = cudaGetLastError(); + std::cerr << "cublasSgemm status: " << status + << ", underlying cudaError: " << cudaErr + << " (" << cudaGetErrorString(cudaErr) << ")\n"; + } + } + + void CUDABackend::SumRows(float *dst, const float *src, size_t batchSize, size_t width) noexcept + { + float alpha = 1.0f, beta = 0.0f; + cublasStatus_t status = cublasSgemv(handle, CUBLAS_OP_N, width, batchSize, + &alpha, src, width, onesVector, 1, &beta, dst, 1); + if (status != CUBLAS_STATUS_SUCCESS) + std::cerr << "cublasSgemv (SumRows) status: " << status << "\n"; } void CUDABackend::Scale(float *buf, size_t n, float alpha) noexcept @@ -208,7 +254,7 @@ namespace Deep constexpr int BLOCK_SIZE = 256; size_t total = batchSize * width; const int blocks = static_cast((total + BLOCK_SIZE - 1) / BLOCK_SIZE); - AddBiasBroadcastKernel<<>>(buf, bias, batchSize, width); + AddBiasBroadcastKernel<<>>(buf, bias, batchSize, width); } #pragma region ACTIVATIONS_AND_KERNELS @@ -348,7 +394,6 @@ namespace Deep void CUDABackend::Activation(ActivationType type, float *buf, size_t n) noexcept { - // For in-place, pass buf as both dst and src ActivationInto(type, buf, buf, n); } @@ -360,19 +405,19 @@ namespace Deep switch (type) { case ActivationType::RELU: - ReluKernelInto<<>>(dst, src, n); + ReluKernelInto<<>>(dst, src, n); break; case ActivationType::SIGMOID: - sigmoidKernelInto<<>>(dst, src, n); + sigmoidKernelInto<<>>(dst, src, n); break; case ActivationType::eSIGMOID: - eSigmoidKernelInto<<>>(dst, src, n); + eSigmoidKernelInto<<>>(dst, src, n); break; case ActivationType::TANH: - tanhKernelInto<<>>(dst, src, n); + tanhKernelInto<<>>(dst, src, n); break; case ActivationType::LINEAR: - linearKernelInto<<>>(dst, src, n); + linearKernelInto<<>>(dst, src, n); break; case ActivationType::NONE: default: @@ -393,19 +438,19 @@ namespace Deep switch (type) { case ActivationType::dRELU: - dReluKernelInto<<>>(dst, src, n); + dReluKernelInto<<>>(dst, src, n); break; case ActivationType::dSIGMOID: - dSigmoidKernelInto<<>>(dst, src, n); + dSigmoidKernelInto<<>>(dst, src, n); break; case ActivationType::d_eSIGMOID: - d_eSigmoidKernelInto<<>>(dst, src, n); + d_eSigmoidKernelInto<<>>(dst, src, n); break; case ActivationType::dTANH: - dTanhKernelInto<<>>(dst, src, n); + dTanhKernelInto<<>>(dst, src, n); break; case ActivationType::dLINEAR: - dLinearKernelInto<<>>(dst, src, n); + dLinearKernelInto<<>>(dst, src, n); break; case ActivationType::NONE: default: @@ -419,7 +464,7 @@ namespace Deep constexpr int BLOCK_SIZE = 256; const int blocks = static_cast((n + BLOCK_SIZE - 1) / BLOCK_SIZE); - FusedStateUpdateKernel<<>>(z, feedback, deriv, e, n, ir); + FusedStateUpdateKernel<<>>(z, feedback, deriv, e, n, ir); } float CUDABackend::ComputeErrorAndEnergy(float *e, const float *z, const float *mu, size_t n) noexcept @@ -427,7 +472,7 @@ namespace Deep constexpr int BLOCK_SIZE = 256; const int blocks = static_cast((n + BLOCK_SIZE - 1) / BLOCK_SIZE); - ComputeErrorKernel<<>>(e, z, mu, n); + ComputeErrorKernel<<>>(e, z, mu, n); float sum_of_squares = 0.0f; cublasSdot(handle, n, e, 1, e, 1, &sum_of_squares); @@ -439,7 +484,7 @@ namespace Deep { constexpr int BLOCK_SIZE = 256; const int blocks = static_cast((n + BLOCK_SIZE - 1) / BLOCK_SIZE); - ComputeErrorKernel<<>>(e, z, mu, n); + ComputeErrorKernel<<>>(e, z, mu, n); } __global__ void IncrementCounterKernel(int *counter) { *counter += 1; } @@ -463,7 +508,8 @@ namespace Deep } __global__ void AdamWStepKernel(float *param, const float *grad, float *m, float *v, - size_t n, const int *t_ptr, const float *lr_ptr, float weightDecay, float beta1, float beta2, float eps) + size_t n, const int *t_ptr, const float *lr_ptr, float weightDecay, + float beta1, float beta2, float eps) { size_t i = (size_t)blockIdx.x * blockDim.x + threadIdx.x; if (i < n) @@ -480,8 +526,6 @@ namespace Deep } } - // AdamWStepKernel: same pattern, plus the weight-decay term - void CUDABackend::IncrementCounter(int *counter) noexcept { IncrementCounterKernel<<<1, 1, 0, stream>>>(counter); diff --git a/temp.py b/temp.py index 031ccdc..9181d83 100644 --- a/temp.py +++ b/temp.py @@ -95,7 +95,8 @@ def main() -> None: Y_batch = Y_shuf[b * BATCH_SIZE:(b + 1) * BATCH_SIZE] # 3. Call C++ Native Loop (avoid Nanobind overhead) - energy = net.train_step_with_projection(X_batch, Y_batch, STEPS) + compute_energy = (b == n_batches - 1) + energy = net.train_step_with_projection(X_batch, Y_batch, STEPS, compute_energy) epoch_energy += energy From 2d560253265a4ca86ea9cacc67444df7850d8612 Mon Sep 17 00:00:00 2001 From: Ra4ster Date: Mon, 7 Sep 2026 19:52:32 +0000 Subject: [PATCH 10/11] Now moved DKP-PCN to CUDA. Next up, training on ImageNet! --- CMakeLists.txt | 8 +- imagenet.py | 208 ++++++++++++++++ include/deepity/layers/DirectKPPCLayer.h | 100 +++----- logs/build.log | 236 +------------------ mnist.py | 22 +- pydeepity/__pycache__/DKPPCN.cpython-312.pyc | Bin 7733 -> 7733 bytes src/DirectKPPCLayer.cpp | 57 ++--- src/DirectKPPCNetwork.cpp | 36 +-- tests/tMatMulLargeAsymmetricVerify.cpp | 107 +++++++++ 9 files changed, 401 insertions(+), 373 deletions(-) create mode 100644 imagenet.py create mode 100644 tests/tMatMulLargeAsymmetricVerify.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 629d5c0..1f95933 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -312,12 +312,12 @@ target_compile_definitions(Deepity PUBLIC SLEEF_STATIC_LIBS) option(DEEPITY_BUILD_TESTS "Build test/verification executables" ON) -add_executable(AddBiasBroadcastVerify tests/tAddBiasBroadcastVerify.cpp) -target_link_libraries(AddBiasBroadcastVerify PRIVATE Deepity) -set_target_properties(AddBiasBroadcastVerify PROPERTIES +add_executable(MatMulLargeAsymmetricVerify tests/tMatMulLargeAsymmetricVerify.cpp) +target_link_libraries(MatMulLargeAsymmetricVerify PRIVATE Deepity) +set_target_properties(MatMulLargeAsymmetricVerify PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin ) -add_test(NAME AddBiasBroadcastVerify COMMAND AddBiasBroadcastVerify) +add_test(NAME MatMulLargeAsymmetricVerify COMMAND MatMulLargeAsymmetricVerify) # --- Compiler flags ------------------------------------------------- diff --git a/imagenet.py b/imagenet.py new file mode 100644 index 0000000..c1c3f37 --- /dev/null +++ b/imagenet.py @@ -0,0 +1,208 @@ +""" +Tiny ImageNet-200 training for DKPPCN. + +v2: deeper/wider network (~63M params, up from ~15M), matching the +literature's suggestion that published, working PCN results on Tiny +ImageNet use substantially larger networks (PCX reports up to +AlexNet-scale, ~160M params) than a naive MNIST-scale architecture. +LR/FL deliberately held at the confirmed-stable 5e-5 from the previous +run, so this test isolates architecture size as the one changed +variable -- if energy stays stable and accuracy improves, that confirms +size was the real gap; if energy destabilizes again at the same LR, +that's equally informative (suggests LR needs to scale down further as +network size grows, a documented phenomenon in this literature). +""" +import numpy as np +import os +import sys +from time import perf_counter +from pydeepity import DKPPCN +from PIL import Image + +IMG_SIZE = 64 +IMG_DIM = IMG_SIZE * IMG_SIZE * 3 +DATA_DIR = "tiny-imagenet-200" + + +def load_wnids(data_dir): + with open(os.path.join(data_dir, "wnids.txt")) as f: + return [line.strip() for line in f if line.strip()] + + +def load_image_as_vector(path): + img = Image.open(path).convert("RGB") + return np.asarray(img, dtype=np.uint8).reshape(-1) + + +def load_train_set(data_dir, wnids): + print("Loading Tiny ImageNet training set (100,000 images)...") + wnid_to_idx = {w: i for i, w in enumerate(wnids)} + + X = np.zeros((len(wnids) * 500, IMG_DIM), dtype=np.uint8) + y_idx = np.zeros(len(wnids) * 500, dtype=np.int64) + + pos = 0 + for wnid in wnids: + img_dir = os.path.join(data_dir, "train", wnid, "images") + filenames = sorted(os.listdir(img_dir)) + for fname in filenames: + X[pos] = load_image_as_vector(os.path.join(img_dir, fname)) + y_idx[pos] = wnid_to_idx[wnid] + pos += 1 + print(f" loaded class {wnid} ({pos}/{len(wnids) * 500})", end="\r") + print() + + return X[:pos], y_idx[:pos], len(wnids) + + +def load_val_set(data_dir, wnids): + print("Loading Tiny ImageNet validation set (10,000 images)...") + wnid_to_idx = {w: i for i, w in enumerate(wnids)} + + annotations = {} + with open(os.path.join(data_dir, "val", "val_annotations.txt")) as f: + for line in f: + parts = line.strip().split("\t") + annotations[parts[0]] = parts[1] + + img_dir = os.path.join(data_dir, "val", "images") + filenames = sorted(annotations.keys()) + + X = np.zeros((len(filenames), IMG_DIM), dtype=np.uint8) + y_idx = np.zeros(len(filenames), dtype=np.int64) + + for i, fname in enumerate(filenames): + X[i] = load_image_as_vector(os.path.join(img_dir, fname)) + y_idx[i] = wnid_to_idx[annotations[fname]] + if i % 1000 == 0: + print(f" loaded {i}/{len(filenames)}", end="\r") + print() + + return X, y_idx + + +def to_float_batch(X_uint8_batch): + return X_uint8_batch.astype(np.float32) / 255.0 + + +def to_one_hot(y_idx_batch, n_classes, eps=0.001): + Y = np.full((len(y_idx_batch), n_classes), eps, dtype=np.float32) + Y[np.arange(len(y_idx_batch)), y_idx_batch] = 1.0 - eps + return Y + + +def main() -> None: + SEED = int(sys.argv[1]) if len(sys.argv) > 1 else 7 + EPOCHS = int(sys.argv[2]) if len(sys.argv) > 2 else 50 + INFERENCE_STEPS = int(sys.argv[3]) if len(sys.argv) > 3 else 2 + + if not os.path.isdir(DATA_DIR): + raise FileNotFoundError( + f"'{DATA_DIR}' not found in the current directory -- " + f"run this script from wherever you unzipped tiny-imagenet-200.zip." + ) + + wnids = load_wnids(DATA_DIR) + X_train_u8, y_train_idx, N_CLASSES = load_train_set(DATA_DIR, wnids) + X_val_u8, y_val_idx = load_val_set(DATA_DIR, wnids) + + BATCH_SIZE = 250 + TERMINAL_SIZE = N_CLASSES + + HIDDEN_1 = 4096 + HIDDEN_2 = 2048 + + LR = 5e-5 + IR = 0.15 + FL = 5e-5 + LMBDA = 1e-4 + DECAY_RATE = 0.94 + + print(f"\nBuilding network ({IMG_DIM}->{HIDDEN_1}->{HIDDEN_2}->{TERMINAL_SIZE}), seed={SEED}...") + net = DKPPCN(batch_size=BATCH_SIZE, device="gpu") + net.add_layer(IMG_DIM, HIDDEN_1, TERMINAL_SIZE, lr=LR, ir=IR, fl=FL, lmbda=LMBDA, act="linear") + net.add_layer(HIDDEN_1, HIDDEN_2, TERMINAL_SIZE, lr=LR, ir=IR, fl=FL, lmbda=LMBDA, act="sigmoid") + net.add_layer(HIDDEN_2, TERMINAL_SIZE, TERMINAL_SIZE, lr=LR, ir=IR, fl=FL, lmbda=LMBDA, act="sigmoid") + net.add_layer(TERMINAL_SIZE, 0, TERMINAL_SIZE, lr=LR, ir=IR, fl=FL, lmbda=LMBDA, act="linear") + net.set_optimizer("ADAM") + net.set_psi_optimizer("ADAM") + net.compile() + net.randomize_weights() + + total_params = IMG_DIM * HIDDEN_1 + HIDDEN_1 * HIDDEN_2 + HIDDEN_2 * TERMINAL_SIZE + total_params += (IMG_DIM + HIDDEN_1 + HIDDEN_2) * TERMINAL_SIZE + print(f"Approx. total parameters (W + Psi): {total_params:,}") + + print(f"\n*** TINY IMAGENET DKP-PC RUN (v2: bigger network) ***") + print(f"Training DKPPCN: {EPOCHS} epochs, inference_steps={INFERENCE_STEPS}, ") + print(f"lr={LR}, ir={IR}, fl={FL}, lmbda={LMBDA}, decay_rate={DECAY_RATE}") + print(f"NOTE: LR/FL held fixed from the previous, confirmed-stable run --") + print(f" this test isolates architecture size as the only changed variable.\n") + + rng = np.random.default_rng(SEED) + n_train = len(X_train_u8) + n_batches = n_train // BATCH_SIZE + start_time = perf_counter() + epoch_accs = [] + + for epoch in range(EPOCHS): + current_lr = LR * (DECAY_RATE ** epoch) + net.set_learning_rate(current_lr) + current_fl = FL * (DECAY_RATE ** epoch) + net.set_feedback_rate(current_fl) + + indices = rng.permutation(n_train) + epoch_energy = 0.0 + + for b in range(n_batches): + batch_idx = indices[b * BATCH_SIZE:(b + 1) * BATCH_SIZE] + X_batch = to_float_batch(X_train_u8[batch_idx]) + Y_batch = to_one_hot(y_train_idx[batch_idx], N_CLASSES) + + energy = net.train_step(X_batch, Y_batch, INFERENCE_STEPS) + epoch_energy += energy + + N_ACC_BATCHES = 10 + correct = 0 + total = 0 + for b in range(min(N_ACC_BATCHES, len(X_val_u8) // BATCH_SIZE)): + X_batch = to_float_batch(X_val_u8[b * BATCH_SIZE:(b + 1) * BATCH_SIZE]) + y_batch = y_val_idx[b * BATCH_SIZE:(b + 1) * BATCH_SIZE] + + preds = net.predict(X_batch, INFERENCE_STEPS).reshape(BATCH_SIZE, N_CLASSES) + pred_classes = np.argmax(preds, axis=1) + correct += np.sum(pred_classes == y_batch) + total += BATCH_SIZE + + epoch_acc = 100.0 * correct / total + epoch_accs.append(epoch_acc) + avg_energy = epoch_energy / n_batches + elapsed = perf_counter() - start_time + print(f"Epoch {epoch+1}/{EPOCHS} | Time: {elapsed:.1f}s | Acc: {epoch_acc:.2f}% | Avg energy: {avg_energy:.4f}") + + train_time = perf_counter() - start_time + print(f"\nTraining complete in {train_time:.1f}s.") + + print("\nRunning final validation evaluation (full 10,000 images)...") + correct = 0 + total = 0 + for i in range(0, len(X_val_u8), BATCH_SIZE): + X_batch_u8 = X_val_u8[i:i + BATCH_SIZE] + y_batch = y_val_idx[i:i + BATCH_SIZE] + if len(X_batch_u8) != BATCH_SIZE: + continue + + X_batch = to_float_batch(X_batch_u8) + preds = net.predict(X_batch, INFERENCE_STEPS).reshape(BATCH_SIZE, N_CLASSES) + pred_classes = np.argmax(preds, axis=1) + correct += np.sum(pred_classes == y_batch) + total += BATCH_SIZE + + val_acc = 100.0 * correct / total + print(f"\n=== Result ===") + print(f"DKPPCN Tiny ImageNet validation accuracy: {val_acc:.2f}%") + print(f"Train time: {train_time:.1f}s") + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/include/deepity/layers/DirectKPPCLayer.h b/include/deepity/layers/DirectKPPCLayer.h index 812ab5c..c2d78e5 100644 --- a/include/deepity/layers/DirectKPPCLayer.h +++ b/include/deepity/layers/DirectKPPCLayer.h @@ -12,26 +12,16 @@ /** * @file DirectKPPCLayer.h - * @brief Direct Kolen-Pollack predictive coding layer, now routed - * through IComputeBackend rather than calling cblas_Deep::_ directly - * -- same refactor discipline as SimplePCLayer's own backend port. + * @brief Direct Kolen-Pollack predictive coding layer, routed through + * IComputeBackend. * - * @note Two independent optimizer timesteps/learning-rates exist here, - * not one: `t`/`lr` for the forward weights W, `tPsi`/`fl` for the - * direct-feedback weights Psi. Both need their own device-resident - * mirrors (t_device/lr_device, tPsi_device/fl_device) for the same - * reason SimplePCLayer's did -- IComputeBackend::AdamStep/AdamWStep take - * device pointers so their values can live inside a captured CUDA graph - * without freezing at capture time. - * - * @warning `backend` defaults to nullptr, in which case this layer - * constructs and owns its own CPUBackend internally -- every existing - * call site keeps working unchanged. Passing a real backend is only - * needed for GPU-aware call sites. - * @warning RandomizeWeights() is still host-only for the CPU fallback - * path's seed-drawing logic, but the actual weight fill now goes through - * backend->RandomizeNormal(), which is GPU-safe -- unlike - * SimplePCLayer's still-open RandomizeWeights gap, this one is resolved. + * @note As of this revision, ComputeMuOnly()'s bias-add and + * UpdateWeights()'s bias-gradient accumulation both go through + * AddBiasBroadcast()/SumRows() instead of a per-batch-row AxpyInto loop + * -- same fix SimplePCLayer already had, ported here. biasGradScratch is + * a new, small (nextSize-length) buffer allocated unconditionally + * (regardless of optimizer) specifically for the SGD branch's + * accumulate-not-overwrite bias update, which SumRows alone can't do. */ namespace Deep @@ -41,7 +31,7 @@ namespace Deep protected: size_t size; size_t nextSize; - size_t terminalSize; // The size of the final output layer (e.g., 10 for MNIST) + size_t terminalSize; size_t batchSize; float lr; @@ -54,7 +44,7 @@ namespace Deep DirectKPPCLayer *layerAbove = nullptr; DirectKPPCLayer *layerBelow = nullptr; - DirectKPPCLayer *terminalLayer = nullptr; // Direct pathway to \epsilon_L + DirectKPPCLayer *terminalLayer = nullptr; ActivationType activationType; ActivationFn activation; @@ -66,29 +56,19 @@ namespace Deep int t = 0; int tPsi = 0; - /// @brief The compute backend this layer routes all math - /// through. unique_ptr with a swappable deleter: no-op when an - /// external backend was supplied (network-owned, GPU case), - /// real delete when this layer had to construct its own - /// fallback CPUBackend. See SimplePCLayer.h for the identical - /// pattern and rationale. using BackendDeleter = void (*)(IComputeBackend *); std::unique_ptr backend; - // --- Memory Pointers --- - // State float *z = nullptr; float *e = nullptr; - // Forward Weights float *W = nullptr; float *b = nullptr; float *mu = nullptr; float *cachedMu = nullptr; - float *Psi = nullptr; // Maps \epsilon_L directly to this layer's state + float *Psi = nullptr; float *proj = nullptr; - // Forward Optimizer Buffers float *grad_W = nullptr; float *grad_b = nullptr; float *m_W = nullptr; @@ -96,29 +76,31 @@ namespace Deep float *m_b = nullptr; float *v_b = nullptr; - // Direct Feedback Optimizer Buffers float *grad_Psi = nullptr; float *m_Psi = nullptr; float *v_Psi = nullptr; - // Device-resident optimizer scalars -- see file-level note. - // Only allocated when the corresponding optimizer is ADAM/ADAMW. int *t_device = nullptr; float *lr_device = nullptr; int *tPsi_device = nullptr; float *fl_device = nullptr; - // Scratch float *zF = nullptr; float *zFDeriv = nullptr; float *feedbackScratch = nullptr; + /// @brief nextSize-length scratch buffer for SumRows' output in + /// UpdateWeights()'s SGD branch -- SGD needs `b += lr_batch * + /// sum(local_grad)`, an accumulate, which SumRows alone can't + /// express (it only overwrites). Allocated unconditionally + /// (regardless of which optimizer is selected) since it's cheap + /// -- at most `nextSize` floats -- and simpler than branching + /// allocation on optimizer choice for this one small buffer. + float *biasGradScratch = nullptr; + std::unique_ptr localArena; public: - /// @param backend Compute backend to route all math through. - /// Defaults to nullptr, in which case this layer - /// constructs and owns its own CPUBackend internally. DirectKPPCLayer(size_t size, size_t nextSize, size_t terminalSize, size_t batchSize, float learningRate, float inferenceRate, float feedback, float lmbda, ActivationType aType, ActivationType dType, @@ -126,52 +108,46 @@ namespace Deep ~DirectKPPCLayer() override = default; - // Setup - /// @brief Templated so either MemoryArena (CPU) or - /// DeviceMemoryArena (GPU) can be bound, resolved at compile - /// time -- see the .cpp's explicit instantiations. template void BindMemory(ArenaT &arena); size_t GetRequiredFloats() const noexcept; void RandomizeWeights(std::mt19937 &seedGenerator) noexcept; - // Topology void SetLayerAbove(DirectKPPCLayer *l) noexcept { layerAbove = l; } void SetLayerBelow(DirectKPPCLayer *l) noexcept { layerBelow = l; } void SetTerminalLayer(DirectKPPCLayer *l) noexcept { terminalLayer = l; } - // Core DKP-PC Mechanics + /// @brief Matches Layer's virtual interface exactly (always + /// computes real energy). float CalculateState() noexcept override { return CalculateState(true); } + /// @brief NOT a virtual override -- see SimplePCLayer's + /// identical pattern. Lets the settling loop skip the + /// cublasSdot-based energy reduction (and its capture-time + /// sync) when the caller doesn't need the value. float CalculateState(bool needEnergy) noexcept; + void ComputeMuOnly() noexcept; - void UpdateState() noexcept override; // Will now pull from terminalLayer->GetErrors() - void UpdateWeights() noexcept override; // Must compute \Delta W AND \Delta \Psi + void UpdateState() noexcept override; + void UpdateWeights() noexcept override; void DirectFeedbackUpdate() noexcept; - // Getters / Setters void ClampState(const std::vector &inputData) noexcept; void UnclampState() noexcept; void ResetState() noexcept; + /// @brief Whether this layer is currently clamped -- needed so + /// DirectKPPCNetwork::ProjectForward() can skip overwriting an + /// already-clamped layer's z with a forward-projected guess. + /// Same real bug SimplePCNetwork::ProjectForward() had; fixed + /// here for the same reason, before it gets exercised for the + /// first time by moving ProjectForward() inside graph capture. + bool IsClamped() const noexcept { return isClamped; } + void SetOptimizer(OptimizerType o) noexcept { opt = o; } void SetPsiOptimizer(OptimizerType o) noexcept { optPsi = o; } - /// @brief Sets lr AND keeps its device-resident mirror - /// (lr_device) in sync -- required for a captured CUDA graph to - /// see an updated learning rate on later replays; a plain - /// member write alone would be invisible to already-captured - /// kernel arguments. void SetLearningRate(float learningRate) noexcept; void SetInferenceRate(float inferenceRate) noexcept { ir = inferenceRate; } - /// @brief Sets fl (feedback learning rate) AND keeps its - /// device-resident mirror (fl_device) in sync -- see - /// SetLearningRate's own note; fl has the identical requirement - /// since it drives Psi's own Adam/AdamW step. void SetFeedbackRate(float feedbackRate) noexcept; - /// @brief Fixed: the original had `lmbda = lmbda`, a - /// parameter-shadows-member bug that silently never updated the - /// actual member. Pre-existing, unrelated to the backend - /// refactor -- fixed here since it was noticed while reading - /// through the class closely. void SetLambda(float lmbda) noexcept { this->lmbda = lmbda; } float *GetBeliefs() noexcept override { return z; } diff --git a/logs/build.log b/logs/build.log index c7bd3fd..7477d72 100644 --- a/logs/build.log +++ b/logs/build.log @@ -1,238 +1,10 @@ --- Deepity Build Log (Release, arch=native) --- -=== CMake Configuration === --- The C compiler identification is GNU 11.4.0 --- The CXX compiler identification is GNU 11.4.0 --- Detecting C compiler ABI info --- Detecting C compiler ABI info - done --- Check for working C compiler: /usr/bin/cc - skipped --- Detecting C compile features --- Detecting C compile features - done --- Detecting CXX compiler ABI info --- Detecting CXX compiler ABI info - done --- Check for working CXX compiler: /usr/bin/g++ - skipped --- Detecting CXX compile features --- Detecting CXX compile features - done --- CXX compiler: /usr/bin/g++ --- Found OpenMP_C: -fopenmp (found version "4.5") --- Found OpenMP_CXX: -fopenmp (found version "4.5") --- Found OpenMP: TRUE (found version "4.5") --- Looking for pthread.h --- Looking for pthread.h - found --- Performing Test CMAKE_HAVE_LIBC_PTHREAD --- Performing Test CMAKE_HAVE_LIBC_PTHREAD - Success --- Found Threads: TRUE --- The CUDA compiler identification is NVIDIA 12.4.131 --- Detecting CUDA compiler ABI info --- Detecting CUDA compiler ABI info - done --- Check for working CUDA compiler: /usr/local/cuda/bin/nvcc - skipped --- Detecting CUDA compile features --- Detecting CUDA compile features - done --- CUDA support enabled. --- Found Python: /usr/bin/python3.12 (found suitable version "3.12.13", minimum required is "3.8") found components: Interpreter Development.Module --- Performing Test NB_HAS_MTLS_GNU2 --- Performing Test NB_HAS_MTLS_GNU2 - Success --- Using Intel MKL (found via MKLConfig.cmake). --- Configuring SLEEF 3.9.0 --- Could NOT find OpenSSL, try to set the path to OpenSSL root folder in the system variable OPENSSL_ROOT_DIR (missing: OPENSSL_CRYPTO_LIBRARY OPENSSL_INCLUDE_DIR) --- Found PkgConfig: /usr/bin/pkg-config (found version "0.29.2") --- Found OpenMP_C: -fopenmp (found version "4.5") --- Found OpenMP_CXX: -fopenmp (found version "4.5") --- Configuring build for SLEEF-v3.9.0 - Target system: Linux-6.8.0-138-generic - Target processor: x86_64 - Host system: Linux-6.8.0-138-generic - Host processor: x86_64 - Detected C compiler: GNU @ /usr/bin/cc - CMake: 3.22.1 - Make program: /usr/bin/ninja - CMake build type: Release --- Using option `-Wall -Wno-unused-function -Wno-attributes -Wno-unused-result -Wno-psabi -ffp-contract=off -fno-math-errno -fno-trapping-math` to compile libsleef --- Building shared libs : OFF --- Building static test bins: OFF --- MPFR : LIB_MPFR-NOTFOUND --- GMP : LIBGMP-NOTFOUND --- RT : /usr/lib/x86_64-linux-gnu/librt.a --- FFTW3 : LIBFFTW3-NOTFOUND --- FFTW3F : LIBFFTW3F-NOTFOUND --- OPENSSL : --- SDE : SDE_COMMAND-NOTFOUND --- COMPILER_SUPPORTS_OPENMP : 1 --- A version of SLEEF compatible with libm and libmvec in GNU libc will be produced (sleefgnuabi.so) --- Configuring done --- Generating done --- Build files have been written to: /root/Deepity/build/Release - - === Compilation === -[1/153] Building C object _deps/sleef-build/src/common/CMakeFiles/common.dir/common.c.o -[2/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/mkrename_gnuabi.dir/mkrename_gnuabi.c.o -[3/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/mkmasked_gnuabi.dir/mkmasked_gnuabi.c.o -[4/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/mkalias.dir/mkalias.c.o -[5/153] Building C object _deps/sleef-build/src/common/CMakeFiles/qtesterutil_obj.dir/qtesterutil.c.o -[6/153] Linking C executable _deps/sleef-build/bin/mkmasked_gnuabi -[7/153] Linking C executable _deps/sleef-build/bin/mkrename_gnuabi -[8/153] Generating include/masked_avx512f_dp_gnuabi.h -[9/153] Building C object _deps/sleef-build/src/common/CMakeFiles/addSuffix.dir/addSuffix.c.o -[10/153] Generating include/masked_avx512f_sp_gnuabi.h -[11/153] Building C object _deps/sleef-build/src/common/CMakeFiles/testerutil_obj.dir/testerutil.c.o -[12/153] Generating include/renameavx2_gnuabi.h -Generating renameavx2_gnuabi.h: mkrename_gnuabi avx2 d 4 8 __m256d __m256 __m128i __m256i __AVX2__ -[13/153] Generating include/renameavx512f_gnuabi.h -Generating renameavx512f_gnuabi.h: mkrename_gnuabi avx512f e 8 16 __m512d __m512 __m256i __m512i __AVX512F__ -[14/153] Generating include/renameavx_gnuabi.h -Generating renameavx_gnuabi.h: mkrename_gnuabi avx c 4 8 __m256d __m256 __m128i struct\ {\ __m128i\ x,\ y;\ } __AVX__ -[15/153] Generating include/renamesse2_gnuabi.h -Generating renamesse2_gnuabi.h: mkrename_gnuabi sse2 b 2 4 _mm128d _mm128 _mm128i _mm128i __SSE2__ -[16/153] Linking C executable _deps/sleef-build/bin/mkalias -[17/153] Linking C executable _deps/sleef-build/bin/addSuffix -[18/153] Generating alias_AVX512F_dp.h.tmp -[19/153] Generating alias_AVX512F_sp.h.tmp -[20/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/mkdisp.dir/mkdisp.c.o -[21/153] Generating include/alias_avx512f.h -[22/153] Building CXX object _deps/sleef-build/src/common/CMakeFiles/psha_obj.dir/psha2_capi.cpp.o -[23/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefgnuabi.dir/rempitab.c.o -[24/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/mkrename.dir/mkrename.c.o -[25/153] Linking C executable _deps/sleef-build/bin/mkdisp -[26/153] Generating dispscalar.c.body -[27/153] Generating dispsse.c.tmp -[28/153] Generating dispavx.c.tmp -[29/153] Generating dispscalar.c -[30/153] Generating dispavx.c -[31/153] Generating dispsse.c -[32/153] Linking C executable _deps/sleef-build/bin/mkrename -[33/153] Generating include/renamepurec_scalar.h -Generating renamepurec_scalar.h: mkrename cinz_ 1 1 purec -[34/153] Generating include/renameavx512fnofma.h -Generating renameavx512fnofma.h: mkrename cinz_ 8 16 avx512fnofma -[35/153] Generating include/renameavx512f.h -Generating renameavx512f.h: mkrename finz_ 8 16 avx512f -[36/153] Generating include/renameavx2.h -Generating renameavx2.h: mkrename finz_ 4 8 avx2 -[37/153] Generating include/renameavx2128.h -Generating renameavx2128.h: mkrename finz_ 2 4 avx2128 -[38/153] Generating include/renamefma4.h -Generating renamefma4.h: mkrename finz_ 4 8 fma4 -[39/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetavx512fnofma.dir/sleefsimdsp.c.o -[40/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetavx512fnofma.dir/sleefsimddp.c.o -[41/153] Building CXX object CMakeFiles/nanobind-static.dir/usr/local/lib/python3.12/dist-packages/nanobind/src/nb_static_property.cpp.o -[42/153] Building CXX object CMakeFiles/nanobind-static.dir/usr/local/lib/python3.12/dist-packages/nanobind/src/nb_datetime.cpp.o -[43/153] Building CXX object CMakeFiles/nanobind-static.dir/usr/local/lib/python3.12/dist-packages/nanobind/src/implicit.cpp.o -[44/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefgnuabiavx2sp.dir/sleefsimdsp.c.o -[45/153] Building CXX object CMakeFiles/nanobind-static.dir/usr/local/lib/python3.12/dist-packages/nanobind/src/trampoline.cpp.o -[46/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefavx512f.dir/sleefsimdsp.c.o -[47/153] Building CXX object CMakeFiles/nanobind-static.dir/usr/local/lib/python3.12/dist-packages/nanobind/src/error.cpp.o -[48/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefgnuabiavx512fsp.dir/sleefsimdsp.c.o -[49/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefgnuabiavx2dp.dir/sleefsimddp.c.o -[50/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefgnuabiavx512fdp.dir/sleefsimddp.c.o -[51/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetavx512f.dir/sleefsimdsp.c.o -[52/153] Building CXX object CMakeFiles/nanobind-static.dir/usr/local/lib/python3.12/dist-packages/nanobind/src/common.cpp.o -[53/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefavx512fnofma.dir/sleefsimdsp.c.o -[54/153] Building CXX object CMakeFiles/nanobind-static.dir/usr/local/lib/python3.12/dist-packages/nanobind/src/nb_ndarray.cpp.o -[55/153] Generating include/renameavx.h -Generating renameavx.h: mkrename cinz_ 4 8 avx -[56/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefavx512fnofma.dir/sleefsimddp.c.o -[57/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefgnuabisse2sp.dir/sleefsimdsp.c.o -[58/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetavx2.dir/sleefsimdsp.c.o -[59/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefgnuabiavxdp.dir/sleefsimddp.c.o -[60/153] Generating include/renamesse4.h -Generating renamesse4.h: mkrename cinz_ 2 4 sse4 -[61/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetavx512f.dir/sleefsimddp.c.o -[62/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefgnuabisse2dp.dir/sleefsimddp.c.o -[63/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetavx2128.dir/sleefsimdsp.c.o -[64/153] Building CXX object CMakeFiles/nanobind-static.dir/usr/local/lib/python3.12/dist-packages/nanobind/src/nb_internals.cpp.o -[65/153] Generating include/renamesse2.h -Generating renamesse2.h: mkrename cinz_ 2 4 sse2 -[66/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetavx2.dir/sleefsimddp.c.o -[67/153] Building CXX object CMakeFiles/nanobind-static.dir/usr/local/lib/python3.12/dist-packages/nanobind/src/nb_enum.cpp.o -[68/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefavx512f.dir/sleefsimddp.c.o -[69/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefgnuabiavxsp.dir/sleefsimdsp.c.o -[70/153] Linking C static library _deps/sleef-build/lib/libsleefgnuabi.a -[71/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetavx2128.dir/sleefsimddp.c.o -[72/153] Building CXX object CMakeFiles/nanobind-static.dir/usr/local/lib/python3.12/dist-packages/nanobind/src/nb_func.cpp.o -[73/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefavx2.dir/sleefsimddp.c.o -[74/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefavx2.dir/sleefsimdsp.c.o -[75/153] Generating include/renamepurecfma_scalar.h -Generating renamepurecfma_scalar.h: mkrename finz_ 1 1 purecfma -[76/153] Building CXX object CMakeFiles/nanobind-static.dir/usr/local/lib/python3.12/dist-packages/nanobind/src/nb_type.cpp.o -[77/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetfma4.dir/sleefsimdsp.c.o -[78/153] Linking CXX static library bin/libnanobind-static.a -[79/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetfma4.dir/sleefsimddp.c.o -[80/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetsse4.dir/sleefsimdsp.c.o -[81/153] Generating include/renamecuda.h -Generating renamecuda.h: mkrename finz_ 1 1 cuda -[82/153] Generating sleeflibm_AVX.h.tmp -[83/153] Generating sleeflibm_AVX2.h.tmp -[84/153] Generating sleeflibm_AVX2128.h.tmp -[85/153] Generating sleeflibm_AVX512F.h.tmp -[86/153] Generating sleeflibm_AVX512FNOFMA.h.tmp -[87/153] Generating sleeflibm_AVX512F_.h.tmp -[88/153] Generating sleeflibm_AVX_.h.tmp -[89/153] Generating sleeflibm_DSP_SCALAR.h.tmp -[90/153] Generating sleeflibm_FMA4.h.tmp -[91/153] Generating sleeflibm_PURECFMA_SCALAR.h.tmp -[92/153] Generating sleeflibm_PUREC_SCALAR.h.tmp -[93/153] Generating sleeflibm_SSE2.h.tmp -[94/153] Generating sleeflibm_SSE4.h.tmp -[95/153] Generating sleeflibm_SSE_.h.tmp -[96/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefavx2128.dir/sleefsimdsp.c.o -[97/153] Generating include/renamedspscalar.h -[98/153] Generating ../../include/sleef.h -[99/153] Generating include/renamedsp128.h -[100/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefavx2128.dir/sleefsimddp.c.o -[101/153] Generating include/renamedsp256.h -[102/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetpurec_scalar.dir/sleefsimdsp.c.o -[103/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetavx.dir/sleefsimdsp.c.o -[104/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetavx.dir/sleefsimddp.c.o -[105/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetsse2.dir/sleefsimdsp.c.o -[106/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetpurecfma_scalar.dir/sleefsimdsp.c.o -[107/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetsse4.dir/sleefsimddp.c.o -[108/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleeffma4.dir/sleefsimddp.c.o -[109/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetpurec_scalar.dir/sleefsimddp.c.o -[110/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetpurecfma_scalar.dir/sleefsimddp.c.o -[111/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleeffma4.dir/sleefsimdsp.c.o -[112/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefdetsse2.dir/sleefsimddp.c.o -[113/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefsse4.dir/sleefsimddp.c.o -[114/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefavx.dir/sleefsimddp.c.o -[115/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefsse4.dir/sleefsimdsp.c.o -[116/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/dispscalar_obj.dir/dispscalar.c.o -[117/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefpurecfma_scalar.dir/sleefsimdsp.c.o -[118/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefpurec_scalar.dir/sleefsimdsp.c.o -[119/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefpurecfma_scalar.dir/sleefsimddp.c.o -[120/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefsse2.dir/sleefsimdsp.c.o -[121/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefsse2.dir/sleefsimddp.c.o -[122/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefavx.dir/sleefsimdsp.c.o -[123/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleefpurec_scalar.dir/sleefsimddp.c.o -[124/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleef.dir/rempitab.c.o -[125/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleef.dir/sleefld.c.o -[126/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/sleef.dir/sleefqp.c.o -[127/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/dispsse_obj.dir/dispsse.c.o -[128/153] Building CXX object CMakeFiles/Deepity.dir/src/RBLayer.cpp.o -[129/153] Building CXX object CMakeFiles/Deepity.dir/src/DiscriminativePCLayer.cpp.o -[130/153] Building CXX object CMakeFiles/Deepity.dir/src/GaussSeidelPCLayer.cpp.o -[131/153] Building CXX object CMakeFiles/Deepity.dir/src/ConvPCLayer.cpp.o -[132/153] Building CXX object CMakeFiles/Deepity.dir/src/DiscriminativePCNetwork.cpp.o -[133/153] Building CXX object CMakeFiles/Deepity.dir/src/SimpleConvPCLayer.cpp.o -[134/153] Building CXX object CMakeFiles/Deepity.dir/src/DirectKPPCLayer.cpp.o -[135/153] Building CXX object CMakeFiles/Deepity.dir/src/ConvPCNetwork.cpp.o -[136/153] Building CXX object CMakeFiles/Deepity.dir/src/GaussSeidelPCNetwork.cpp.o -[137/153] Building CXX object CMakeFiles/Deepity.dir/src/StreamAlignedBatcher.cpp.o -[138/153] Building CXX object CMakeFiles/Deepity.dir/src/SimplePCNetwork.cpp.o -[139/153] Building CXX object CMakeFiles/Deepity.dir/src/SimpleConvPCNetwork.cpp.o -[140/153] Building CXX object CMakeFiles/Deepity.dir/src/SimplePCLayer.cpp.o -[141/153] Building CXX object CMakeFiles/Deepity.dir/src/backend/Tensor.cpp.o -[142/153] Building CXX object CMakeFiles/Deepity.dir/src/DirectKPPCNetwork.cpp.o -[143/153] Building CXX object CMakeFiles/Deepity.dir/src/backend/CPUBackend.cpp.o -[144/153] Building CXX object CMakeFiles/AddBiasBroadcastVerify.dir/tests/tAddBiasBroadcastVerify.cpp.o -[145/153] Building CXX object CMakeFiles/Deepity.dir/src/backend/Backend.cpp.o -[146/153] Building C object _deps/sleef-build/src/libm/CMakeFiles/dispavx_obj.dir/dispavx.c.o -[147/153] Building CXX object CMakeFiles/Deepity.dir/src/ModelIO.cpp.o -[148/153] Linking C static library _deps/sleef-build/lib/libsleef.a -[149/153] Building CUDA object CMakeFiles/Deepity.dir/src/backend/CUDABackend.cu.o -[150/153] Linking CXX static library bin/libDeepity.a -[151/153] Building CXX object CMakeFiles/pydeepity.dir/bindings/pybinding.cpp.o -[152/153] Linking CXX executable bin/AddBiasBroadcastVerify -[153/153] Linking CXX shared module bin/pydeepity.cpython-312-x86_64-linux-gnu.so +[1/4] Building CXX object CMakeFiles/Deepity.dir/src/DirectKPPCLayer.cpp.o +[2/4] Linking CXX static library bin/libDeepity.a +[3/4] Linking CXX executable bin/MatMulLargeAsymmetricVerify +[4/4] Linking CXX shared module bin/pydeepity.cpython-312-x86_64-linux-gnu.so lto-wrapper: warning: using serial compilation of 6 LTRANS jobs diff --git a/mnist.py b/mnist.py index 6e6ebdf..b7f47e4 100644 --- a/mnist.py +++ b/mnist.py @@ -39,26 +39,6 @@ def load_full_mnist(): Y_train[np.arange(y_train_labels.shape[0]), y_train_labels] = 1.0 - eps return X_train, Y_train, X_test, y_test_labels -def train_step_dfa(net, X, Y, inference_steps): - net.reset_state() - net.clamp_input(X) - net.project_forward() - net.get_terminal_layer().clamp_state(Y) - - net.calculate_terminal_error() - net.direct_feedback_update() - - for _ in range(inference_steps): - net.step(False) - - energy = 0.0 - for layer in net.layers: - energy += layer.calculate_state() - - net.update_weights() - net.get_terminal_layer().unclamp_state() - return energy - def main() -> None: SEED = int(sys.argv[1]) if len(sys.argv) > 1 else 7 EPOCHS = int(sys.argv[2]) if len(sys.argv) > 2 else 50 @@ -112,7 +92,7 @@ def main() -> None: X_batch = X_shuf[b * BATCH_SIZE:(b + 1) * BATCH_SIZE] Y_batch = Y_shuf[b * BATCH_SIZE:(b + 1) * BATCH_SIZE] - energy = train_step_dfa(net, X_batch, Y_batch, INFERENCE_STEPS) + energy = net.train_step(X_batch, Y_batch, INFERENCE_STEPS) epoch_energy += energy N_ACC_BATCHES = 10 diff --git a/pydeepity/__pycache__/DKPPCN.cpython-312.pyc b/pydeepity/__pycache__/DKPPCN.cpython-312.pyc index e69e31ab0861878289bb008788e4ec9468356be5..91e5e60a2572be3c5ebc539f39e9301550da2b99 100644 GIT binary patch delta 19 ZcmdmLv(<*{G%qg~0}z~fxsgj-4gfoi1(g5* delta 19 ZcmdmLv(<*{G%qg~0}zP#Y~<3G0{}EE1V8`) diff --git a/src/DirectKPPCLayer.cpp b/src/DirectKPPCLayer.cpp index 4316723..eef2e54 100644 --- a/src/DirectKPPCLayer.cpp +++ b/src/DirectKPPCLayer.cpp @@ -8,14 +8,6 @@ namespace Deep { namespace { - // Same mapping as SimplePCLayer.cpp -- see its own comment for - // why this is needed (activationType stores the FORWARD type, - // but ActivationDerivativeInto expects the DERIVATIVE type). - // Note this also closes a real gap the old code had: the - // original UpdateState()'s switch fell back to a raw function - // pointer for eSIGMOID (no dedicated derivative variant existed - // in that dispatch), whereas backend->ActivationDerivativeInto - // DOES have a real d_eSIGMOID case. ActivationType ToDerivativeType(ActivationType fwd) { switch (fwd) @@ -77,8 +69,6 @@ namespace Deep backend->CopyFromHost(fl_device, &fl, 1); } - // --- Setup --- - template void DirectKPPCLayer::BindMemory(ArenaT &arena) { @@ -106,6 +96,8 @@ namespace Deep zFDeriv = arena.AllocateFloats(own_state_size); feedbackScratch = arena.AllocateFloats(own_state_size); + biasGradScratch = arena.AllocateFloats(nextSize); + backend->Zero(b, nextSize); backend->Zero(mu, out_state_size); backend->Zero(cachedMu, out_state_size); @@ -114,6 +106,7 @@ namespace Deep backend->Zero(zF, own_state_size); backend->Zero(zFDeriv, own_state_size); backend->Zero(feedbackScratch, own_state_size); + backend->Zero(biasGradScratch, nextSize); if (opt == OptimizerType::ADAM || opt == OptimizerType::ADAMW) { @@ -132,9 +125,6 @@ namespace Deep backend->Zero(grad_W, w_size); backend->Zero(grad_b, nextSize); - // Device-resident t/lr for W's optimizer -- see header's - // file-level note for why these can't just be host - // values once graph capture is in the picture. t_device = reinterpret_cast(arena.AllocateFloats(1)); lr_device = arena.AllocateFloats(1); int zero = 0; @@ -152,14 +142,14 @@ namespace Deep backend->Zero(v_Psi, direct_size); backend->Zero(grad_Psi, direct_size); - // Same as above, but for Psi's SEPARATE optimizer state. tPsi_device = reinterpret_cast(arena.AllocateFloats(1)); fl_device = arena.AllocateFloats(1); - int zero = 0; - backend->CopyFromHost(reinterpret_cast(tPsi_device), reinterpret_cast(&zero), 1); + int zeroPsi = 0; + backend->CopyFromHost(reinterpret_cast(tPsi_device), reinterpret_cast(&zeroPsi), 1); backend->CopyFromHost(fl_device, &fl, 1); } } + if constexpr (std::is_same_v) { if (localArena && localArena.get() != &arena) @@ -190,20 +180,21 @@ namespace Deep total += pad16(w_size); total += pad16(nextSize); total += pad16(out_state_size) * 3; - total += pad16(own_state_size) * 3; // zF, zFDeriv, feedbackScratch - total += pad16(direct_size); // Psi + total += pad16(own_state_size) * 3; + total += pad16(direct_size); + total += pad16(nextSize); if (opt == OptimizerType::ADAM || opt == OptimizerType::ADAMW) { total += pad16(w_size) * 3; total += pad16(nextSize) * 3; - total += pad16(1) * 2; // t_device, lr_device + total += pad16(1) * 2; } if (optPsi == OptimizerType::ADAM || optPsi == OptimizerType::ADAMW) { total += pad16(direct_size) * 3; - total += pad16(1) * 2; // tPsi_device, fl_device + total += pad16(1) * 2; } } @@ -228,8 +219,6 @@ namespace Deep backend->RandomizeNormal(Psi, Psisz, 0.0f, limPsi, seedPsi); } - // --- Core DKP-PC Mechanics --- - float DirectKPPCLayer::CalculateState(bool needEnergy) noexcept { const size_t N = batchSize * size; @@ -275,11 +264,7 @@ namespace Deep (int)batchSize, (int)nextSize, (int)size, 1.0f, zF, (int)size, W, (int)size, 0.0f, mu, (int)nextSize); -#pragma omp parallel for schedule(static) if (batchSize > 4 && !omp_in_parallel()) - for (int batch = 0; batch < batchSize; ++batch) - { - backend->AxpyInto(mu + batch * nextSize, b, nextSize, 1.0f); - } + backend->AddBiasBroadcast(mu, b, batchSize, nextSize); if (isClamped) { @@ -309,7 +294,7 @@ namespace Deep backend->FusedStateUpdate(z, feedbackScratch, zFDeriv, e, N, ir); } - else // Output Layer + else { backend->AxpyInto(z, e, N, -ir); } @@ -336,8 +321,8 @@ namespace Deep 1.0f, W, (int)size); float lr_batch = lr / batchSize; - for (int batch = 0; batch < batchSize; batch++) - backend->AxpyInto(b, local_grad + batch * nextSize, nextSize, lr_batch); + backend->SumRows(biasGradScratch, local_grad, batchSize, nextSize); + backend->AxpyInto(b, biasGradScratch, nextSize, lr_batch); break; } @@ -355,9 +340,8 @@ namespace Deep adam_scale, local_grad, (int)nextSize, zF, (int)size, 0.0f, grad_W, (int)size); - backend->Zero(grad_b, nextSize); - for (int batch = 0; batch < batchSize; batch++) - backend->AxpyInto(grad_b, local_grad + batch * nextSize, nextSize, adam_scale); + backend->SumRows(grad_b, local_grad, batchSize, nextSize); + backend->Scale(grad_b, nextSize, adam_scale); if (opt == OptimizerType::ADAMW) backend->AdamWStep(W, grad_W, m_W, v_W, num_weights, t_device, lr_device, lmbda); @@ -406,7 +390,6 @@ namespace Deep void DirectKPPCLayer::DirectFeedbackUpdate() noexcept { - // If the layer above has no Psi weights (i.e. it is the terminal layer), skip DFA if (layerAbove == nullptr || layerAbove->GetDirectFeedbackWeights() == nullptr) return; @@ -418,17 +401,15 @@ namespace Deep layerAbove->GetDirectFeedbackWeights(), (int)terminalSize, 0.0f, proj, (int)nextSize); - // W += fl * proj^T @ zF + // W += fl/batchSize * proj^T @ zF backend->MatMul( - /*transA=*/true, /*transB=*/true, + /*transA=*/true, /*transB=*/false, (int)nextSize, (int)size, (int)batchSize, fl / batchSize, proj, (int)nextSize, zF, (int)size, 1.0f, W, (int)size); } - // --- Getters / Setters --- - void DirectKPPCLayer::ClampState(const std::vector &inputData) noexcept { size_t copyFloats = (std::min)(inputData.size(), (size_t)(batchSize * size)); diff --git a/src/DirectKPPCNetwork.cpp b/src/DirectKPPCNetwork.cpp index 4d9acdf..c4ea5ca 100644 --- a/src/DirectKPPCNetwork.cpp +++ b/src/DirectKPPCNetwork.cpp @@ -2,7 +2,6 @@ #include #include #include -#include namespace Deep { @@ -51,14 +50,20 @@ namespace Deep { layers[i]->ComputeMuOnly(); + // Skip a layer that's already clamped (the terminal layer, + // once ClampState(y) has run) -- overwriting its real target + // with a forward-projected guess is exactly the bug + // SimplePCNetwork::ProjectForward() had, fixed here before + // it gets exercised for the first time by moving this call + // inside graph capture, which requires ClampState(y) to run + // BEFORE this, not after, for the capture ordering to work. + if (layers[i + 1]->IsClamped()) + continue; + const float *mu = layers[i]->GetMu(); float *nextZ = layers[i + 1]->GetBeliefs(); size_t n = layers[i]->GetBatchSize() * layers[i]->GetOutputSize(); - // Was: std::memcpy(nextZ, mu, n * sizeof(float)) -- wrong if - // mu/nextZ are device pointers (DEVICE_GPU). backend->Copy() - // is device-to-device, matching SimplePCNetwork's own fix - // for the identical bug. backend->Copy(nextZ, mu, n); } } @@ -71,7 +76,7 @@ namespace Deep float DirectKPPCNetwork::CalculateTerminalError() noexcept { - return GetTerminalLayer()->CalculateState(false); // only the error buffer matters here + return GetTerminalLayer()->CalculateState(false); } float DirectKPPCNetwork::Step(bool needEnergy) noexcept @@ -94,12 +99,12 @@ namespace Deep const std::vector &y, int inferenceSteps) { - printf("TrainStep: device=%s, graphCaptured=%d\n", - (device == DeviceType::DEVICE_GPU ? "GPU" : "CPU"), (int)graphCaptured); - fflush(stdout); ResetState(); Clamp(x); - ProjectForward(); + // Moved BEFORE ProjectForward() -- required for ProjectForward's + // new IsClamped() guard to actually protect the terminal layer, + // and required so ProjectForward() can safely move inside the + // captured region below. GetTerminalLayer()->ClampState(y); if (device == DeviceType::DEVICE_GPU) @@ -107,6 +112,7 @@ namespace Deep if (!graphCaptured || capturedInferenceSteps != inferenceSteps) { backend->BeginGraphCapture(); + ProjectForward(); // now inside capture -- see note above CalculateTerminalError(); DirectFeedbackUpdate(); for (int t = 0; t < inferenceSteps; t++) @@ -131,6 +137,7 @@ namespace Deep } else { + ProjectForward(); CalculateTerminalError(); DirectFeedbackUpdate(); for (int t = 0; t < inferenceSteps; t++) @@ -140,6 +147,7 @@ namespace Deep } else { + ProjectForward(); CalculateTerminalError(); DirectFeedbackUpdate(); for (int t = 0; t < inferenceSteps; t++) @@ -171,10 +179,6 @@ namespace Deep const float *beliefs = terminal->GetBeliefs(); size_t count = terminal->GetBatchSize() * terminal->GetInputSize(); - // Was: std::vector(beliefs, beliefs + count) -- the - // iterator-range constructor dereferences every element - // directly, wrong if beliefs is a device pointer. Allocate the - // host-side result first, then copy it out through the backend. std::vector result(count); backend->CopyToHost(result.data(), beliefs, count); return result; @@ -183,7 +187,7 @@ namespace Deep void DirectKPPCNetwork::Compile() { #pragma omp parallel - { // Broadcast FTZ/DAZ hardware flags to ALL OpenMP worker threads + { _MM_SET_FLUSH_ZERO_MODE(_MM_FLUSH_ZERO_ON); _MM_SET_DENORMALS_ZERO_MODE(_MM_DENORMALS_ZERO_ON); } @@ -193,7 +197,7 @@ namespace Deep if (device == DeviceType::DEVICE_CPU) { - cpuArena = std::make_unique(total_floats_needed, false); // TODO: Consider adding huge pages as a param + cpuArena = std::make_unique(total_floats_needed, false); for (auto &layer : layers) { layer->BindMemory(*cpuArena); diff --git a/tests/tMatMulLargeAsymmetricVerify.cpp b/tests/tMatMulLargeAsymmetricVerify.cpp new file mode 100644 index 0000000..1f44b23 --- /dev/null +++ b/tests/tMatMulLargeAsymmetricVerify.cpp @@ -0,0 +1,107 @@ +/** + * @file tMatMulLargeAsymmetricVerify.cpp + * @brief Reproduces, in isolation, the exact GEMM shape that crashed + * DirectFeedbackUpdate() on Tiny ImageNet: transA=true, transB=true, + * M=1024 (nextSize), N=12288 (size), K=250 (batchSize) -- K much + * smaller than M/N, a shape tMatMulVerify.cpp's tiny (M=N=K=2-3) cases + * never exercised, and one likely to select a different cuBLAS internal + * kernel (the crash was inside ampere_sgemm_128x64_tt specifically). + * + * Differential test: CPU's GEMM path is already trusted (verified + * correct in tMatMulVerify.cpp and used throughout tonight's CPU runs), + * so any GPU disagreement at this specific scale isolates a real, + * scale-dependent bug rather than a general transpose-logic error. + */ +#include +#include +#include +#include +#include + +using namespace Deep; + +int main() +{ + // Exact real dimensions from DirectFeedbackUpdate()'s second GEMM + // on the Tiny ImageNet run (layer 0: size=12288, nextSize=1024, + // batchSize=250). + const int M = 1024; // nextSize + const int N = 12288; // size + const int K = 250; // batchSize + + std::mt19937 rng(42); + std::normal_distribution dist(0.0f, 1.0f); + + // proj stored [K, M] (batchSize, nextSize) -- transA=true means + // op(A) = A^T = [M, K]. + std::vector hProj(K * M); + for (auto &v : hProj) + v = dist(rng); + + // zF stored [K, N] (batchSize, size) -- transB=true means + // op(B) = B^T = [N, K]... wait, matching the real call: zF is + // passed as B with transB=true, stored [K, N] per the same + // batchSize-major convention as every other buffer in this codebase. + std::vector hZF(K * N); + for (auto &v : hZF) + v = dist(rng); + + // W stored [M, N] (nextSize, size), started at zero (beta=1.0 in + // the real call accumulates onto existing weights; zero here isolates + // just this GEMM's own contribution for comparison). + std::vector hWZero(M * N, 0.0f); + + float alpha = 0.5f; // arbitrary, matches fl/batchSize's role + float beta = 1.0f; + + auto RunOn = [&](DeviceType device, const char *name) -> std::vector + { + auto backend = CreateBackend(device); + + Tensor proj(backend.get(), device, hProj); + Tensor zF(backend.get(), device, hZF); + Tensor W(backend.get(), device, hWZero); + + backend->MatMul( + /*transA=*/true, /*transB=*/true, + M, N, K, + alpha, proj.Data(), M, + zF.Data(), N, + beta, W.Data(), N); + + std::vector result(M * N); + W.CopyToHost(result.data()); + printf("[%s] done\n", name); + return result; + }; + + printf("Running CPU reference...\n"); + std::vector cpuResult = RunOn(DeviceType::DEVICE_CPU, "CPU"); + + printf("Running GPU (this is where the real crash happened)...\n"); + std::vector gpuResult = RunOn(DeviceType::DEVICE_GPU, "GPU"); + + printf("Comparing...\n"); + int mismatches = 0; + float maxDiff = 0.0f; + for (size_t i = 0; i < cpuResult.size(); ++i) + { + float diff = std::fabs(cpuResult[i] - gpuResult[i]); + maxDiff = std::max(maxDiff, diff); + if (diff > 1e-2f) + mismatches++; + } + + printf("Max diff: %f, mismatches (>1e-2): %d / %zu\n", maxDiff, mismatches, cpuResult.size()); + + if (mismatches > 0) + { + printf("FAILED: GPU MatMul disagrees with CPU at this exact real-world scale.\n"); + return 1; + } + + printf("PASSED (though note: if this test PASSES but the real training run still\n"); + printf("crashes, the bug is NOT in MatMul itself -- look at buffer sizing/lifetime\n"); + printf("in DirectKPPCLayer's actual DirectFeedbackUpdate() call site instead.\n"); + return 0; +} \ No newline at end of file From 582cf57ad9487e058064c7dfdfa37374db35961f Mon Sep 17 00:00:00 2001 From: Ra4ster Date: Thu, 10 Sep 2026 21:38:13 -0400 Subject: [PATCH 11/11] Added GELU. --- bindings/pybinding.cpp | 8 + deepity_build/__pycache__/cli.cpython-314.pyc | Bin 0 -> 16493 bytes .../__pycache__/cmake_runner.cpython-314.pyc | Bin 0 -> 4744 bytes .../__pycache__/config.cpython-314.pyc | Bin 0 -> 4496 bytes .../__pycache__/git_info.cpython-314.pyc | Bin 0 -> 1929 bytes .../__pycache__/process.cpython-314.pyc | Bin 0 -> 3071 bytes .../__pycache__/__init__.cpython-314.pyc | Bin 0 -> 2763 bytes .../__pycache__/base.cpython-314.pyc | Bin 0 -> 5945 bytes .../__pycache__/rich_reporter.cpython-314.pyc | Bin 0 -> 19742 bytes include/deepity/utils/Activations.h | 326 +++++++++++++++ logs/build.log | 394 +++++++++++++++++- mnist.py | 2 +- .../ConvolutionalPCN.cpython-314.pyc | Bin 0 -> 7518 bytes pydeepity/__pycache__/DKPPCN.cpython-314.pyc | Bin 0 -> 9331 bytes .../GaussSeidelPCN.cpython-314.pyc | Bin 0 -> 8083 bytes .../__pycache__/SequentialPCN.cpython-314.pyc | Bin 0 -> 13979 bytes .../SimpleConvolutionalPCN.cpython-314.pyc | Bin 0 -> 9173 bytes .../__pycache__/SimplePCN.cpython-314.pyc | Bin 0 -> 13851 bytes .../__pycache__/__init__.cpython-314.pyc | Bin 0 -> 1112 bytes .../__pycache__/_backend.cpython-314.pyc | Bin 0 -> 443 bytes pydeepity/__pycache__/utils.cpython-314.pyc | Bin 0 -> 3932 bytes 21 files changed, 721 insertions(+), 9 deletions(-) create mode 100644 deepity_build/__pycache__/cli.cpython-314.pyc create mode 100644 deepity_build/__pycache__/cmake_runner.cpython-314.pyc create mode 100644 deepity_build/__pycache__/config.cpython-314.pyc create mode 100644 deepity_build/__pycache__/git_info.cpython-314.pyc create mode 100644 deepity_build/__pycache__/process.cpython-314.pyc create mode 100644 deepity_build/reporting/__pycache__/__init__.cpython-314.pyc create mode 100644 deepity_build/reporting/__pycache__/base.cpython-314.pyc create mode 100644 deepity_build/reporting/__pycache__/rich_reporter.cpython-314.pyc create mode 100644 pydeepity/__pycache__/ConvolutionalPCN.cpython-314.pyc create mode 100644 pydeepity/__pycache__/DKPPCN.cpython-314.pyc create mode 100644 pydeepity/__pycache__/GaussSeidelPCN.cpython-314.pyc create mode 100644 pydeepity/__pycache__/SequentialPCN.cpython-314.pyc create mode 100644 pydeepity/__pycache__/SimpleConvolutionalPCN.cpython-314.pyc create mode 100644 pydeepity/__pycache__/SimplePCN.cpython-314.pyc create mode 100644 pydeepity/__pycache__/__init__.cpython-314.pyc create mode 100644 pydeepity/__pycache__/_backend.cpython-314.pyc create mode 100644 pydeepity/__pycache__/utils.cpython-314.pyc diff --git a/bindings/pybinding.cpp b/bindings/pybinding.cpp index 9672082..d1b9fb2 100644 --- a/bindings/pybinding.cpp +++ b/bindings/pybinding.cpp @@ -56,6 +56,8 @@ namespace return Deep::e_sigmoid; if (act == "relu") return Deep::relu; + if (act == "gelu") + return Deep::gelu; if (act == "linear") return Deep::linear; return Deep::relu; @@ -71,6 +73,8 @@ namespace return Deep::d_eSigmoid; if (act == "drelu") return Deep::dRelu; + if (act == "dgelu") + return Deep::dGelu; if (act == "dLinear") return Deep::dLinear; return Deep::dRelu; @@ -86,6 +90,10 @@ namespace return Deep::ActivationType::RELU; if (act == "drelu") return Deep::ActivationType::dRELU; + if (act == "gelu") + return Deep::ActivationType::GELU; + if (act == "dgelu") + return Deep::ActivationType::dGELU; if (act == "sigmoid") return Deep::ActivationType::SIGMOID; if (act == "dsigmoid") diff --git a/deepity_build/__pycache__/cli.cpython-314.pyc b/deepity_build/__pycache__/cli.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..73293361c5029b6ba54c1eee8e73dadb8dd878d8 GIT binary patch literal 16493 zcmbt*eQX=ocIQwWev=|4Q4(cI*2tD*(UK^CD7NFoi5=Opl-RN|w&eto!_wqPqD_%} zGedu{X`{ThFLDypPFigAur2DMUDOX)U<2#|8(^`EzV7a87T6C_q96GtdZ-r-Ht2uK zvkULOzxH?T%#f5V$HseO>(21rGxwf*?$s#@8dS9x}~sWx`Et9EvGs19~_s!rUk6RvT$ z>Ne`|sGfH%B0u3B_o=?|dbNJsulmOWY5?UH(KgXA-l#T?H>pkIK{d$o>=Vu7EouvU zc1*O6ht$w`o7%>HJ14e{x2x@4oak!h)DF>&v{P&mJ*}MRZRJvdlttYx`tYlseyLqz ztLVqCfKeuh4fxf_%DVAh6MhAavK<$9ip_`K;}~Zw%A`{_gxO?DQiOgfG8Yk& zX(gGEgqV9wQseUwjVh(pT?tTUGrJTa0`UxIE?t)5sxq97#jgy9g-cRAmQy4lsS0W7 zsw4~XloXQ{L7kV})kb2fDkm@HR0;HGUJ`QYR5E=9B8&*5^a8z)!A47}d;`qJrN|_v zLS07DZ1MSpOhV%oOcl*}Hac-ZoH%y^qs-6d(s7N?V%BI*>3W=voI;ZC>+93@4$o&6 zq+vOuNPC8-Nh$anvOcP<5g^!fu3aB*WC=$Uii51d#YOe0;yNKLWq%}LfW zL)@}!+A?J`pR0F~Ge_6FXX~cxb{Pb7ep+TT)E2u$%e2MI?W;3s=Pff%vuuiOV&*UX z-i-xm(q^sG*4g)2eO{Als@{C6-bJ1*@XL(yS+(0Lwe!=w$#vosd$+-?bDB4)rkc#W z>fPKxrh?Tx^GUQ1ddz2z%i5-GQ?1qC=95`NrG4}dQlrsj*L0Ojl7U`Ym8rwzZXVJV zots*;Pur*3&3dZ$Ct5^bJ2=qNM0)HpX|bBEqDyqw@MSk!j_}iU0q(pbR~O-Obz>Nn ztfPjm!r^4y8v%}Uaoa(ab)2zeoikl#bE0Ra+kBp`jJg+_{4DH9V5KAOBSA`;B>e#vQNIsB|W@EXO%6KOpHQRzD z&q?{Zp-jGEXlO_PLx%;*qbl=3qkio6Y&%b;QB|H7KzR;l8@5m>ekI?riE#l3Y&IpS zkP`JEAE@*oj*T1H`B0@HoqLHL8#fLM;U>+Soy$atvuGkIYkVrHs2V?)%3RXCVlJ&F z7o-!ijHB7EA4U-Id@??-S(O`#=FFyIDvp5#nGQP50hwk}S0&A!jmbE0G)Gd2Qa?bg zq6Xny)|8ai+)C~ec#BJlqFHe?hh3UoR~Iyo!87B}M?Wx4Shb4A z#y?7V)B~B8rRIv`{7K?B_{ikcY);iISNGL9p_X)-&(~+PxMcH^nNeV_oihLL) zG=Y^*m^N5$43!#t3yr;tZ>$GH%k8DMaG@=HcVMmUz*_L&;^}Waq4oN0<@y)D9Qd+x z#e4FBwa)L};Obm%95v1DpP%~d)Sa%i;NHd4_Z`7iN5@|`{Y}STc6`xV>^`&}X!+dx znRofZTA+8y_79%GdZ=Uh{J->+LIZ`+z*=ai6dEanMph2KelPU;(nQ%GD*1(iUnu!^ ztonDX3p?*j{8jyJpI$GR*r>Dk-mpBdT8ROx%e~>|>f2V@hwk}?7RT;8d>{Va2fz2> z`#*fY?5w}#yXpJHzhUQGEoG;_?DVcXJxgOBdp2BlkNrD0*WSP8>i_Z3?KeN!xfJ{3 zz8?=2T>Tp!lr4@v3~)Ag+1>tUqj!939g&sx;SJ8RCr}QwEM32?Zdfg@&WBd3&Hm8M z+58_){&4d4{^i6ye&>Bp!>w0tzEbkEuX@^<$add7aJPSD-)k%0(M?jbxmUbBcc$-* z-#xzK-urWY-~ann7v~Nuq(=Rh-r!iDot@I=Nlnv_ozemx=#+l9?$94XeMMde_oiju zB3i^c(JJ!8P+r5ff1({v<74vN)n5l_#=nF4c~N*zn1sTh*`d4SsQPvPZ0{_K|?fI9ClkOBupG4FWMq?qWvI` z7I5M4+M3!SI%g^>?6Z^+^C+Za@Svd#J+)>JhQ*72z9;zb={(h@X3eBLZP_avXJSBX z5F5p&nwGbw4KmthvVERO3)>0xhQzi_w3aC-ZjWE$HnH85{!@G1A$E$}LD?k=HEnM# z`)<(g5O<0_&m5&*)Y&KQI%p`PPi?&)bm8Zq+YP#b=b#$|-OzK;ML;+F9CUj?xA!r+ zikexK_jE<6zJ>)Yk4#&q8mpz|Q$=0;DN|d5X8u%*`Bc4E?c4x2_Nk$Y(mC^AAD_t|Eqif>t)7&0eTMr3GC~hgrvmfWR{Q(&8?#u>W!?VSzgr~q6B+aku=+J=~8YE zW~?oZqe9UfhRUZo;`5m#G@879Xy|h0k^-IYa=vpiw{Qv8b!Jw;sRF%HN*+U6<()%AX#g(JwLA)d6GBv=rfS}p z9G^G!7|pJeY**am|7$zye6~zi%?Hw;&vwerw3 z5svF>I_pL%6OS2#qD1n(p`ipc1ReEAU`kYfR0Yd;Lo?Tt3mDQH0)a=2i?e4>j70Wr z!RtQodLD9vz;qy|R(QJrtjuL}8w1lMekC@C4ndB}d@?&IFo@^cyaZ9QxkILkEKJ>) zngG^fkx9`YA;xown8vfztxpnF&Pn+f$1-wU5=PG-KPD)-EU_w_nH2gfRuPq)gKcvK zwh#vH5)Cdap)f{GK99QPbCl2w)yK<)xtP&gGbZC~41=6Mx>ZACV-Z1=2ojXi30N#@ z25{Xx7KA=GYh@h?TWi{5Q^U=gqWP-i_ny6y%<5~0B*j1{ELNri88slc`UU=CWqyYAqE0%ijk~VpU}*Kn1_Q8mvpH=3u++5LF$bT$>k%WmqT{w$LUpz$>8 z*iIzBjo)F1e1x(I7@(fH**T>%MrN3zd2{I+W*mkouGw)|U6EvkU_8Te*X=8}-T``7 zmQhHXp5lY@vdFFbnoGW(RbNjzc=VyQuHL>l1}oR*De#Y&QFbw}vV}7O^j|9_T6s!mEMsT412;8{F_PVmpZIYl!PvmD|bjA)M%Ee*EWTdm>q~rlfp0H68_WKVvah{d-?G8i`MNfIoUP*_=SIyf zu$oP(fJqhH;4Qv07K6+MvS1Bat4Y>QmGy3}%;@=@b1Qsnk>7ovcblRJlzmNQe|tF) zEc?cv(zhl?16I#$>Q=2hQG@U3hJ%TQ2WK$*E7IOh_4{6WN@)Wteeo%!j5qXV>zdox zyOU2T^`mryxA_hjTzkORHg20xYTHw2+w(;b;7x&VSv-E<;VC)V@PEb8w%*!PY8_Z@ z9r$_c(9+3r@8I9e{N+sXg^6PC#HTY~$xCDF-iF&3KA-t?=Jw>B19$pY8V8ErLCSpX z_O(*$?n3MCm6m~(#=)X@h;k>E=Sti66t?eK>D;^0xUcBlzbUu0eS8J~N5)q=CsrC? zFM8iV@gJZ3&c}r!59>LX_m=Y`=k111JdDRs!4X<@2BUBEC zO2OVju=j3fA$VYk|HO6Q<6rS~uJD~zD8YaOI5Og~N-PgZXa7E=M6`+aVc@h_x-WxN zP9w=TV!g42R|P{fH;zR)#x{|>s{)M3uhL6|>dQxwKrK0jTMa_OFcFsgoC?S&Az-B5 zV*iekmG1o;bpP8ncKhmgS+3q67<-iqShU`sm(fE&BW6FzQ&UT4yntN3@^0k;xg?jz zF637HBGn?~(nk2$h1`m_rCMYfRlA9sXO(Z^quKO6_4Kg|wQgbXBDZ2ZRbQB?cvPyM zMv;S+$1aSsCG3??;D^SOB~snHhiW6yZ>>l#$l)Sw7^heo713>`Y(1y zB$ZVp3XOeVpMF-f=)CKAa+GWUHe2NnK*J{A5Ac9pqHcw(1X|Hii{+^0JN;Jnwr*Hi#(huohK)VpYz{PSIP_;X7udGp z(Vx9sThE41f3D~3?(h8US(jVwdNtrt+k*shYxjT=us|0;8OT+?F4*BUGk#?q)3#Z_ zjPU7gxi6fyoU>(}=NvCtqLy4;&Kj)=gTNcLJQ%f4k>2#!WpZJz=D7>3MaNW)&(nOx zqy!JplHduruo@!LsDA+XHyj~ ztoc4yk(J5U+`nRcRs5_MEYrMbHNncPd)o2+e$|-@Ft!!sRfX=do|&B{7t?MNdSVda z|CqMISI$rSs<9W(TkEuEux8(fJiPC6hpxfL05}Wt$1&AwwpqQS9pgL#$k%7)Zv}v1 zJiJv})JDtmv1{5@r8pe`&Cp{6_-W1s%pb?p@b{HJZ@(eUy=FZ#`^~2+^on-OM$JOp zMVmQ$nB4y-le>Mo{YsJf9c6LF8qb) zF!z`)t{v_e`d7`N_#ViOOrpmwQwad15DF+XMohCi$Lxazy?Mr8KACt z7~*sDgf|EdEc8>fl>qQq$kK@(u2euV4}^Hkrhqm9-H-s^>NEmSF~uEYv0ene7!0M7 zvr_y96gdH5c?8a>KyiVN#DGE({!&3#31ZO!n}~bzCx}l#+4cBOEb<>=bn;E3rx8mh zj2dBo3}OPR?#QI01l(#K9ol92bLMbrHU`;+cgmy=$>d;@uOrc%2`Q_vAOJhU3Mg2i zfvK8^)vL%~bLpxcO65U%vxnZa>(O8=4orj00Ozpy0eLr(&?#OUn}}!+ME)U8jZh?n zM@aIl4zNHeg08@6fl6~x3dxfa4Dyh#P+4QOjIbVx%BZDhZ70mYNTVg0G=JGfuNyuF zCLOQd6}<_qUjJ38)S=|!ELy_HRC0teGOlIPg)|rC=`v;!m59P%jB52du~|#FQD*Am z9}ok7$QVdv=At?Z$;MFr5#?GDFQ_@q>7frteYK|~D48}+&BoZ!Jh}%RVoOMxPbV~L z)r@W#kr}2_g+%RPTpI;?7pP{X5j7(a5SJOcL3drXZ;O)JT4hC-5>+Js8@%={{*?Ey zk`eW{jq|jA99bM)_jfE$-wm(%M;6D+JqK2MUb$^A1%%aruyTBQapGH7!$+aAzxmet zH{UP$dkg;Fl7H8#e^=QXEP4A1-oB!@AH^kqSHa&^^7j?|eIK|S0A4PyjQ_r2TFW)XU zo?IHeU(*I6O<>ip27A|nyOvIT6F%}~d@cMM+FyF(7v7eY*1@|AMeobsbat0IBdeW} zwaz_Xyl}7c(9-06|F-2=(ce`Lgn(QX0^z$u#Q*|Lnm%`a=3H(s26|aDy9(Z2ckAzY z53*mq1#fTBy9@lf8cNRX1?TpnQ&ZOeUY-tKa+qZHg# z2=01lv-Y|R&h918cTUdLbo2c8)i4HHutSH_uqZtUh|$M_kB;xa(B_w&ZO8~ z@OFdHJG}1o-x~jD{PsIVZ})v?pycc-IJ@q+i_U>>{UJu$QSf#wzx!oV(R-xqZG4om zaPG#Np^fX7@8Pq4+wy_6`W;KwZ+0Fkd;N^rzCy4M%=Um;Vaap*9i82S%Wo8&{om9` z`fl&N;P4Uvl*icx&=c93U02b$yG+c73eM2-_AlNpI$wQsu?~#h3f&5Q6#BJ7N6NoF z7C5!t_ILa-4{rZCaH0>luMRdL{cpih4!8fl-E(T#_1i?XJZAst2+W6Q`uOGA%U0M+H&+cT z*$QtFt&zH!ilsDT;1$qx4c+1UETXB34P@|(+g5oZZ)&S7u!J_8$1X$H$C(1l%FrQ5 z%Qd-xzDBZO*H-8Y^!vK4?68{mtF>0*MRlDc?1=pgofXOVLizynQVha{kVwWUyw7y^ z3>mH#a!^$q9Huh{FS;-1;8UU<626%yy*uOtO(o%*MxD?;CeNH>tuX~>Ar0pMBCFlC zjt^2LNVQ;Y;-Wc`LEOwFinm6`%+%5}o1V!MOq{=28hT4bK}~$GXEBo_p4VI2_9n;T*Yd>5<$=*;q0` zypmJ=2BZUUMc0mW-;o{+pFaU7rV;juHgqr2Q#@%k?lZ$7!kl3g4Vw28bYl)Ph6ak2 zHGVCTKW1usEPj@Hp`ZOUTCq#;+mQ-q_#|m0GP?f{)5!3>Me-*q3~p-Gm|hYj!W>YT zf2ayPK2ZfR=ni}2Jy;LA$B0Zw-7^HZSc>PEcU+Gebw-&ohA25$(wrPdqCq#N5ImYe zOs|3`m?j2re-`u{^wtK}nuf*TOJ6E zv!?@ndbpy#v6HnZzd{Lv)}L!*e-|w(6v;{(`;g^4Hg-x^{9POD;qHi@QDqUd>>?bC zvG@hjn5!05t6G`Nwr{NNhZh=6%aOi9*lM?@qmZt|NoGA;e^U(2z_==i%vR_+_S`M& zpQ#}2z`qS20ru|(owF*2!g_aNrt&QVEO=u8v-G_{CAi3Z4_h>XDr-Xj>>^+Ll|jaJ z*0oi9y^iLZoIHs&0;P0pg|CTD^sf>!RpkO-0k}}oQOz-*P~xurK0x%)zZ#nuAC67C z_FL$)v6M3v$WjXx9Kx)HtJ=u0&Tt-fnsH}r&x4XCP%G+w z_6YCL+pA5@abn?STa-U#?CQjNa}+QiDtMk)ga2&qs~&24&^Y<=jyb*2?o;1ezKO*DvT{>+?0-woiaD)R1UuHj`=x2ly(tp^eEMYf`zNvagA4h#0G%Qs0-_t8JZt_=16xw_%pvzGm zap7q|<^P2erd-TpabN`3Tf6V{thMec@WI97>%N_Lj<5M%(7!l&@5A>N-&?kqLj8qM ze(1^weP6T{ohN_ctVgI1DSg`tj%_7JcfrwJa_n4n?5yEGTxbas3}8mql=#j9 z-+3qT1ww8*i~M1xcD`EVkFN8b%d>ai{_66|f_#rxzTum1&ldU4b-s1kUgWzN-N7P% z$fT1>{JBRjflKu6*9sZGfA{LpNt^BOZJv|OE)BXf4M#K@wqTV`PKJ9lH`HmwfFUF} zD$@oeU!){PNe?A`l#s3?W1HjTe}RNnhdK}2k4-bCwQFA8S5FoSa8DbN(hjfBMgkf2 zIYerr9vDVzPIH>yT{G|&_3D9Y3^!#kCqf4p$c&Rlhj!T%KTNgAlzc!5A$R)hrD;Z; zr1&%R=O`7B#-mR~=gWE=<|}d;d6ZZ$@^Y3FmI8MiFZ=^{^p{-EuTbXyRsCMe{ztPG`cCJc$9V*z rJles$&;NYf+6ZXm(QeYCH-5*$@!q9;(b|R&h<`Npot5LaF$VuXl%6Qy literal 0 HcmV?d00001 diff --git a/deepity_build/__pycache__/cmake_runner.cpython-314.pyc b/deepity_build/__pycache__/cmake_runner.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b74d0a14a607a703e7bd527a5df43f23a40d605e GIT binary patch literal 4744 zcmb6dOKcNKw%WgLx1Apj`C(!Yp9B*ehrqzhKxT0qlNj>xG{FlZLr-IO5;IPBtGms| z;*gbATG$I=FR;>T1*Zw8$!W!Lj&p2pGP1Xa)gsM|Rx9lxER2xmw6Dr;J27Eq_o-d= zs@|)r_p7Q`9%yc25kT_lpRPA~5&8}{EU;G)>Qw=dWyB#JT}2#~peAg(Er4-b!mis% z?a&>hcIr;3?FrWet<%6magGEt;nv*p?==juecR-NUVI2u|EZsjQ5!?~fw5}KFu z0PZDt6KC6xLd}JbX7j3~1%=>fKAX$LWNALT5X?%t9Ml)Zppe${LN54OdP(?P{4AK3 zmzD%66U=2LQ5y_Vrbmz@Sr_!IENRowb|eLT5%+JIElq)r_=mv;KTw`}5>5b#f*B3R8e zFRM#}uHi-YnIA72%_rZYAL?hDyWm$sPKvpLgwb1|C+KdH^IA~9P~#ldPe&u66!zaWA+ z1YK5DA9NB690X(;eed0MfAaT}k2=@A?=8p6p4LBKToPiE7 z3H9m_nPX>{b9Tgojo31p8NEIxCh_nT^9s;IYS&(^UQP4z{28N&9 zxvZw+82MDitAc`Vs48(m6eKPs#V$+;L4o~xl4^O0i$|Yt;%eacJrf@BZ;H00XPrm9kf~23ZRsY>-rFQGRLZ8b~g-wCfAg*;Z?SbawK=+YX*%v z-X7mD)}HKoo2Mpw;AiA$0SoLfwqbsh?D?7b;oN+eZ9w`RM%J~S!)GpmWUqzQYp#`e z`8Aop?u&op_FT~7OYXCxqzy{~yb!mIVDWh-rti5Udj_ z3neelPRva6qlxLU#Ps-uk;3jeN0f_A@MkBZxZ4&US6jebSWru*?0#=&2K?E~H}omMP&X%u2_G!3a4)S}<*~n2A8GT=rUlg+g9Z1~Cru%qXHX zmE@s^pG=H$QSR~x+^EcWd^*lWXXBbfRtEkivq*8(ZK7pqRR8x zoM_tNx-{J+9pW=t)nqHEep?YuPZbw6T{9i&d`2)G*K&epwkWstMOosnLAsEY7BrL1 zOW9led`1r*4bUVKYu1YM}izogwrS2=LBntJ<{5nn{b&jjSq5rhP%wO=nKNDXJ>ISWTBAsG_8sbXMbuR*azWimYXC zg*+usS@Tcp?)S#LXf*H!Kat~!j4 zBc+b0(GgvvpPeWUMeln4>FFqU^xPf)M@#qf!2ZYQ)&r4J;6yQSq7)b@21dSeeiMkV zc+37ntEa#5pImX5eg2Z~K+$)gyZudm$OU+Nty_71K04zG;A@V4APckkSP5Z%@M z8qrNHZ(31nXSvn)kbS@!dxzIfu7!*}r#D(h%AV%sb8n~0I58v%We|{ey^e6I9Dz5% zQzuE0E8&etk{np=#Mm!ozvqET!^r~9e%+MoB`zKfY~pG)L|S#}fQ9{Um(^PyfKBb6-$RnVyvOk+D$C|Y_6Tn3bate@T^nz(AGi?nUvbsa> zpTBp0o#`qwyUI*+iRmsf-J30p*R_clr>kx;%PLsU|Jk4F7;yih=Y9=LHP5lwb5K0m zvx%$KZqlmD0lWli50x9x{C_56gL7$?IWR3eOAK9Gs!*Sy+0jbBSBkap+YipxEkJFP^z}=uQEZ*z03J zs*Y`MbSyZt>d7)3yxaY}+2+Q+bys~GqNYR5Ce8C1InDDX%k%Sj_;UjPRe)n(#fe+x z${Gd?u2^`Y!<|8{s4d$RKWoT^pyJ3=v6RH^#BY@dWlk^2nS4(CSp5X3IGnTy6q|O6 uqW*@?{DAg-huq(yrf(7ZchvO*3jFATza`$V@Ob@ErC<}k>4JD56aE*p*263S literal 0 HcmV?d00001 diff --git a/deepity_build/__pycache__/config.cpython-314.pyc b/deepity_build/__pycache__/config.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..90c8b2f594c13896bd7f9493cc25a11db464d237 GIT binary patch literal 4496 zcmcgv-ESMm5#J+^A0kDOvPet*iq^3ji?&3{t{Mk!5=3S!TdqHplkI-WhvVH+JnMMJ z?kz1#KobEj&=@FC7wv-{`j9>qMuEN;DEbHVNrh`fJ)nTm51@UMD;McwXZ9$PvYiHg zXzyWec4j|je>*!fdvYL?BJj=M`q}MYWeE8THg>=MkWqI24vaz8$rKS}jA%qCw}__9 zMJ`4~bd&%enNu&uL`?G0x%kC|NQ{!j*ioXXTI`Gx&Ql=Jjt>(pF&zmd)8UVZq?TMv zf;Or3X(=tO^`B8gm6nn6j5YxBY>zxAjd8)AEZct?&j+pk+k*rB>_* zD~4aNqF)<1Mv4bozk{Kvb1mDksOh;id#S|@VSDab%I+|}D(beoL`B^Y)MA41hV3$+ zo>&3F30k>gEHi5R)bN*?MTM8D;zT8S)uT=BnauZT-Eb`j=YT(TJ<3_r;{pYaw`7~t zvG2fyb;cR}^ZHL=C77<^SuKa9{JPfyv#N~xMvcMr)NbHlIaSNG#iU@qpq@)@;ZtwL zEv1S|kTzV`6LJqeH2W+=7^Y+R*ih#UQNIpMyvDsX=9=v@k-uozPQ?4{fFdgF&-p!9Y^STkka)K6yW=H{ktv0X=}2&UbDF`0AuEQ zpaww?0w5^px{r96x*-I&t1UoH*Ew7OvV&U=rptA&!OGn8*;E+}zRgVGmo3JcwpiW$ z>+)#Jp^ucBt2_ZiB`E;o4+HrDd6LT9o7>6_-Mbc&!ThIcp67Ra25}ecxWm>mlMMsF zd8ge>m9cep=H@p}w|)zqueV%qr<%v<^^0>0v|*Uwh3vRbEAux>bk^|0bC*a>P}mT- z4Fk{eTaNUss_hzlm70c2O>X#gx?+nucn!53hb10E|cHdcIF5|Yj_De*8o}{x^W@y#$8J;U0?{qJgXvKr|eYs`F z;7bgQzU^^1D(cn1Vt-k9O~33jF1vxFH`;ySu2{@8ELNfyJkNqg%>cwC(2m@%+<;G4 z>dd99UW;0udt6YL!D%#%yLO}1pm+g|s_npfr6Ao!!Fpk9r=QzeK-PY^uyB65bowP5 zJ6)UzmoOZFIb6g6LvQ$S@i8W#H{dx*6QyysD;UHS2n&AQZsI>Y!zS~Z4qyhmV2(cl ze&H|x61d77H6ZrX4Djz_X>Gq{o8sGWj1#oysxAhZ8Le_rpVzL<&Rssg@bVZzA*S%-VxKk)(vdyVG)NDC^xz(8KS&RO^za^O2Bb$o z`r01pfHo?!8ig2mG)SMrD^Zb&g(vYEgegK7t4#=saD;(++pGFPKSlr@l7@_yYSr_c zKs8&I5vWxN-a%TPyN(y%5Bhk^ZL`@Xq$tnT;mTW(RQw=j!fCrf0-V1J0c|#jHkZ7$ ztZ0ExglAnUl@@~>yDPY%n+?cdI@}@{C|BT^i+7FWo+nI%vkfW1R1RrE**Of%I=M-1 zMc(>8Ar{CElMj@-qA6OWtiaZz|G>UMoHNnlZWt^v;>z6P@rEIMob+{$(Gx>@J7#0B zAL#+D)T_~;uRjE2oqR@=*wj-b8;9giMp*} zEpH~vT~l8f*N0JFLYvA+R3ss&9QW9RC_ja|(?D*bqKnxn1nbnswET4qfll)4VD>gN z@qp*sir(Mn;D@2U{T(A5uNfdX0Y82O$yb2%!q*piv}N<4&9)Mj$3Yx^sM$j_oZUj*E@aQQB9i{Chn=r;7lnWw4AKw4)lf>if@y+D%u7Up} zbZ+6&U3A)R_5K&R)vKYiHXtkP3e-QWBp_hTybPq2iF&94q0HRh3sYCVh}Q1w1!|K{ z{h$1(+KZGLn|MlqY*_NwJtpWRzqTJ`cK$0;%5#1cS=;!!d_r8PRLT{{fY-%jtsRLb zWo2auMpvLng11I#$+V1e3Dw+Kp@^G5k;3N*R1EM8>5gnc@xhs4$zrPkRg1Yyv7Rc? zg}T9^D5^p|0!2rvS;qnc>n|)s=Pz8ND;{62Io?WXEeZK&5+5N&C5#5o2ZDa+pkoK= z=zD^Huf_m;+8(!^BDuT_2nQc$Pi`hpN*v{lzFN%j0qDuINODN zn}Q_Z&}3Y!@-wKA*YF&2vq&x>K{uC^hs$6{;)iz$x}^UbAW;3w!e2fC_7xtfg{{=U zy-QoNJt*5*d>9lCJW>yArL*_Gwv`#YcV#PASg(AX$L`0An`-`n`$#SRarn^2g`a$H z{n7*VkvjGymU$L|Ot%vy@qumRw&TE|>{Lf~sv{fk$i_Re7yE$|%6`uxz=a#bJzXtk zfvnfdEy5*U`z>}=jg5(Yvo1@o4F}(ZIWIl(AfCPyqJU1P? z8p16fOfsY5VFda2&(F@>xO`pjtu`gD;3!w1hfEglg}FBQcKG~zmoI<@W19bKAphQu zDvAPIP}G0*6Xn?7$k9KMk*E7fB>iCQ56bXopebwi~uJA{|~ERHi7^E literal 0 HcmV?d00001 diff --git a/deepity_build/__pycache__/git_info.cpython-314.pyc b/deepity_build/__pycache__/git_info.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..9d756901fbf74f13dfa62c4112c051b82c9026fc GIT binary patch literal 1929 zcmb^yO>YxN^zE*_{@TWIQZOxv;t;;9G%->lKng7?AwUEQ=?c zWdYZ*sjsg8rUAT?M0X`)MA@n$(gqVatbnP^D;HF*j!0ac*SLneHlMhV+>39RKn`z`{iqPoiDN`MTa(4an8N&Dksnr6?ORF9)H3_#} zo7*Milz5$#C~2XL&{Cb)E}@g91#$~1YI}^Z$+9AHw(ker=3d~l1*9|bxYMv1yMmNn zqrp1yosJXHhGEO4Y#m3~hHJnT3{_ppaP1(5do5wAhfxw?T7)H#my9r-C(RVPo))R| z9-sGXfzTMILa$QWcj`iSf@ag>B2f*3hDf>|hV8@Q|^`vX~6kv24z%R}DE~mPHNpcMCrd$1C-qNh&m8 z zNC&$*j;wa4;vnmcDOI+H5NX2_tf(hH2XK)$1Rp7N#Z*kSqF~arU!|_lDd8dY<13}- zQJn(Xv^kST574)z8m6WbJ0^<0gRFl1m~^5IJFrigel5Yh)o6>H`&*f1M3G-F(P$g? z!!k1Bwb*%^U0Uv=Fp01BGJ4yim*~}qaZ^2kR^?hu#d|O8zV_Ce+?M{|ZtL|q-Rn<` zN3mrt_iFu1Wd9Mj){VQRE~zVUauuMZE$6zoU4a?pNQYRBYCnRL;p4&SPPWFN)SDAo z_kXPAzAj@*)xpI#I0Pf`*9~S8e|pzms1Cyzcrc?4gQt|ULc`~LrL2ksC!cVkGmhhe zA}z5XDWI)q~p;PDm;7c{vfu)-#mc znrPXSk@ZxhL9xXz%zb#ab66x?uU3_#JM1~D*;C~>iE<+(XZ11bvVyarha__-kT-H3%wNnfe#R2iMk zLq=sCm@H}uSEqqi3AegEPRCIeHP8=;v|&>%CZ}H%_HPu9J}Dd(M&)^7&x^s~jln~Y z2M=wh5(C+tH01ifIP~R_%?xDn+Xf5_b(N>yMETp1Z$^Z1c-QDI5?Q~iZyHc2emDAc z?r!o$e($|o8^!TQ#qs;^K3IC1pLvv?`Tb{O?}kx+Y?PmkpL&#ge;X9^7&X`Kmmgex znxA@-p4!xq_XiumkUl+gdRB#p>e1N=?csY0LOG-@%MF}(=GH=-I&zfA=BHx_L~i%L zf-oxIX*`o;s-lq^PrPUvrBB5(`vY1*sT@;G_BuMCC_lp7FL2-$6n=uy*E*=Vdk3E> Z#T`{s`Zoc=mIms+O-b$;II$wDe*#w%n}h%W literal 0 HcmV?d00001 diff --git a/deepity_build/__pycache__/process.cpython-314.pyc b/deepity_build/__pycache__/process.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f786f413b2e2fde6dbc25f11c2fefe66bde66894 GIT binary patch literal 3071 zcmcIm-ESMk7N7O5*I%)nq)F4#Hkl-CsGDMkQfQDukSJ{lx3p#(%tOU)I{0f|_Egb@hx7+&tv{X<-OrKU@fDn1@~bCgy{JaJ~$PC`_~17f5-b7tr3 zeEiNiGoDJu5wwY?R<94q2>r+gohAm1u=O4=HKZVl=8+=I2rq?bXo&Hl8Ig*-m#752 zI1_$Jrt%QFCgxL}V|Mysq)4ZQU`(S*cmOH#0J6GiOo;#=<$PR;0UzglLP-GM#rZBJ zG>S6G>cuN%&9ZQXI7P#Cb0vqAHA+o;5qs3MOlle~c9%3_=y>4xodOo`0?zC zOp$5O)&fbugPnve3``9b(0pj}U4(S_cNo1YED4IBgcMPca!mU0FCkdwM_i-m*t%Qo zDp`(3@jLjEV;jCq4C)cv7hOueK_H48DCe@vM*OF9OHSFy5yv$qa(UuhH;UBF>4s4; z>5WdWE_kM;=i1iER&MxRs@m4jP*u_eN3qX+3`h>!+7XTJXfWj$Ic&=k6zUDMj9@;YWq`M%rckN@l z=HRrT6@(5%x&SV4T9}_E?LO^MtLLZwaQ<*11pDQJ05oz6c4wlOJa)+swNI;J_Q{ej zxYbY@13yu!hL*4|lrte;*3Gxdj_!-5O*0~akN|g;P1~b}>xY*dkGL*7Wej7IV3X;< zN2=}-jv>+yf4PMzFg3IhN&V;M-E`mmNUGlZ?mHX32OFdHBMq&Y#?2`HY2Zj>agE#_ zIB`qrt6gX>y4TxZU;N}`eXv2+9$z0_JF*VTCz{cTpZX7dd3G)G#l`iJ+x<`c`%>-P zMrz;tuh(CwFW1e+o6W>YkYf1-p zUOri6<`RMbIt0 zl?82UXpa5oE#HpN&Y1PXVD()qq}_7?ziXCRZBJ8#c}Rh07^a|ALWNMC-M^;=+2mEU zBxF%VeDs=bbRHa7kqTm-h2EZ4*e)$Ys33sUyT<<-L@0&^5uD3{V^2HX9xGx#y=MxB z1+jy2)WEQ);!$k6ShHLQDiqa%{SK5cC=n&iv^)Z(t*F^h-kfDt+2Rf{7foBU zux@ILw&PN>xP6k1OP*zAPB7rR;O2{zfBx#O2?XsWA_g{Ry%*cd*HLwr>Xa)jc95$xst5w z|BoKJ8%@`xkCK~V)H4Jz0{f;`raqi*ocj9A=V#WQ`QrIz;zUzAu@f|%gusB$69VSq z{dqzND*c{h5Twr`*!q|0a~c+Hr_ZQxauWe*m@U?=U*@A7>0m8yEIV`Qz5?bS4{3M> zo@pn7ick<970~dPzbpgsI+TY{Xcj6fgA*BHWE4n7CS%a~B0z(%mp^{FIW&2@@7X5m Xc__on?jeS{`!EUx;nyiZ57fK6rQ!$_GUMBkeU#F0-45Dhy~c8V5W+s zl($bRgy<$~23)xQGHu$<6?>VLb-iJ`{zQoezSpMgD&thx-j$GwfQmY!@d=o9eZlyW z?XW}CkGBPsyH1@3TyJ`yw;dZGtSVE_5tWyA*&stVbDE_?+|r2OdiuUjI&Q$aCfLn z^WYs-24;ouEDw*{evQIFxDFNVCZkTBIae4DyQf3Os7=S3K^VGKudTQIYS8j)W8FK; zdP#{4+xG)h5%}SG9G_^pUTr4um)vC-=1#j}-d_qp)Dr(W;C_NX{8ad*VYo>_SRsoL z=XX9Qqy}>%6C+hdyDz|yt^6dtT(xKo{7985*exz1|!26lN zec7deH*68&O=Hmqm3q)%6&{3ava-N~%Mg&TQe&*?igwalRW!a5hXq_(u`Jhjg=LkS z?Wo_fx^HEc#Rov^ApF8T(5#RbL{+ArVzU}pnQM`6~=o$i>7F z@brt_v>;@vD;@AqV2WVNzMZv z^~L(Zgh*`(*Ngg=T)$?8P3Dvojt`a)80Q64FzAL>f}Sl?`7T(J(VPKcg*-NfzV&{Z zSvMx`8WX?tuTP%1GkIcta`x`z?1RFt&hDAD!fZ#IeP9gUoc?lp-6-BMimT_=j3d7d zc8ud4^*G-FdbZ&l0}7Z{Hb>+Qug7^sI+$bVIL3z?*g!l1&m4mjgIT^rB2YGaap4VU zVi*n~@P=-wvWpFYwXLsM@S*mlnSz@83wJGeZvsimRvA1`c+# zga14*hMo~6GZYOBZlt0AH^lef(DURQVx5MIdcR`*Ff(S>%$Zt6 z1rYa1A4gfJ#89U=C?HXK*%chU6ZI`KpK)6RT!JnR1!ms=hGGmhg|Q_s$PVSmQ*3S> z+`#VkY1#drmfatkT#Ms__a`6T1LMr*S0MD(hgb8w23{ONDwMR@Opo^Fd@9M>QL>nR zlYF8qrlz{L6!A=YUY*M{)oo|go-^vJXG)ptzv@udA^%#AkbGEWkh@!feMx4>Dp)C7 zkhOB7Qx9AR`SmPo3My%A0^AD%EZaVm4C;6;)YuXd8dB}pwL0=?!jCaiU8}dppg07) zt$XfOhAagv+a>2@E3kE)HA3dW21!u;CTL2ir}Zv4_0J!^pa&3@2e#F#GW{5e4bmub zBrv?~2)KsU?66)9Y+fq^spJ=s-?!AgkLrH>x<}pl3m?5Kfn|CMm=={*;XekN+Vl4S zN@KGn%B8GKi6}4gBzXo$Bt@AfR6-FI*#0ispsw+yKe1Vb=Qo^jyfsfTT;194pbc10nG8zK$NqrTHsh#q+8Zf3+Ne% zGX-l=VXJ1Bv1IchJSyLVAdNIgMPObaaG8&4)ec#F2Nr7SFOIk24d7tViZ!CA=xE0p zL9OMnX)o)G$w?0-o1JkJ7YKQBP#d|rTL{d=4k Krg9@Eb^ZtR(wlVv literal 0 HcmV?d00001 diff --git a/deepity_build/reporting/__pycache__/base.cpython-314.pyc b/deepity_build/reporting/__pycache__/base.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5859bf8c7fc050cb18feaa0eb8eb77d6bbacfa4c GIT binary patch literal 5945 zcmcIo&2Jn<7O$S}`SAE7{*0Xuh!cm{V;g25jxi=OgdmZ4g%x@cTEQm0?QuIEc0NpX z4Vy%J@ByK{?KQ^{r}&mXfj_}>3ba5%;()k03}UbRUUhX(w;gPc!iw#^-bcOn zx~gaTdb13zaQVNV{*q+uM`D5oo|@Kr7j%o=VWxJI-Oz+~j{1q4i5t4mZy3UmynZuz zBPCMj*a$n%Ok~tJRvYMX6SGa|uts_Re*$&t7!} zw~M0Uh>hABW*N6^u~GC_@Tpo(Ykj4WO1GKNOePX{(n7y&m>Ps8Mp?OkPCX-%Ky{!{ zCxWJcCV{46Xc}l5XeNecfOY}R#?UUH-9UR{XclNM(7qVj4fF)i{utT=bO7jJ4DAIv z1avrt_5mFMdNPKd06GeEEQa<2Jq7f13>^S^2I$!sItcUypfASIA)x1gj>phppyz>J zh@m4uCxBjzp(o9k#He`*8$Rh}OlPykg~N;3F-AXX3H-2^TI(!2TkIotC$aDaV{5?9 zvJbQk&D6|9Uc(Cd&otLdaYxj7^+9$rSZ{Kh0x7j#;}u)DwD1M*YJQ_uaq_(8I)(f# zUi;K33OB#zIGZJLKlry+>!tEqo=d$-)w}ss+jU;qyzli`mI_P9vb>Q%&?`~63v%3E zsq_V2VrGxW&GmoaO`Q9nYKFI;V8!-5{Q^QmZ;rqnyUcB;ScrA%23U^@EVc zp~3S_WB$>HAifR;#)9??gZU5!eE|j^LEpZXxTAg+?q6JA=_{H`NhPcK!4^b0?Y z#}9Q2`NtiUf}R-Umd0OqV!olM4cfuJNl4%D3CwosslXS#GEObGUa8ppzIQToNDQa& z;pd4QKP>nudY*RrphzHVTS4g0HjUZF97tO%hGjd@T0V?9(a0YtmL0olai>_Tu9xoC zx#OJw-ZAnKf0;-q z_Wxwy7=(n`J<~L18W%y@Vx_4domtx-?xj_q=q>qfIdWoetHfTQ6{Cqg7;#Un zvRQV7Y(nu7mcZ|iy!{~KrHGgxjxqZH6 z#h@CD?lCzSxI{WAvtL(Yk8k!{M=^UkYW7^ym}^|#G3FjouT!(H?=`!}H(OFpbs{HhWeHJQ%W^3+FBdC;lPOq!{@PM1*s%pMK6*ch9G7+fNcay-;x>p& zkA{cjUg5aHwJIgo#q}j}FpS}|^qh*7_ml=v$NkBru|T@MMlB{H%KR@xLUAO9T4%ZcKAt!<_b+2oSn={rBmWTln{TY4*J%>mqNSpxu8Zul zWTk6g{X*s0(Y~=KAntB;Fd}_CgdRdn3d)kejzmpW5K|3&Za{dPb{yApze-XI{ zvghMK+N~Fhj_Z2;;pwY7Sg`uN)O zS=}zHkrS4+UgEBxJgn9%i&M_=lSIxDp*Js|AtDPMePHC|9{w(on?yb!@)4206Dbn8 zOXQzKYD8Qj4?w&Ti!OqqRJ3d%cxkncn1NI}ghY*voNxfwM!Bgd8$w-aK=9p&_r^mKc*$SLmR4ti;N zt(QxyoX!c-m3w^bWv==mvBbZ?6pB)pE?m31rfJ`^3qP@m@7dU|sYPw*Kkjb~e^P

dS^H1J#3Q0_nT}{q{PvKr1h|9nUXEZk|bJx#ENnSp?YG5k2p1{V6ZU>b+`+=O6V85D+{ME76Yl=v zcyYfJmk<{f$BB~u(s*gBP%Uf{6lblVxJF9%Sa_fDGR3`F@D_hXQLm^K`y-L5xIZ46 zio_7L1%ls81QDf2e2aZfd#zU77bsP`HPejJ2yhU31;WH;rzT_KJP90Z{ zo<8kUj%sD&Q|hEY?h7PT)*>o6m0)yAjR(~+-aq$?_(NRj5@`ZPgt({(@uE>FZc)Uz zRVl)tEww_}6SpZ=gl#NrSL_HoSlB@?7I!Kx_%6i_->nqGFIFU!DPd(Kr4->Z7A{dd z2$!>PsZxP(B@34+RR~wJut%vuxR!;>l{$p$S-3)JK)8{GE0rdMn_0L@X+gM^g{u`A z;Y}=DqqHHsnT2bWc7!`vxK7!E@KzSCS2_{)vT%d44dLx9+^Fn8cqa=tDP0J6vv9N0 z6K_$T!P4#0BqcO9Va)4v@99wFQaC72CgT2!cv7Z(GW14J4)|je7pMGcK*qYr@rj@u zjKo7~P`*fO+BJr`7wDgy6aOF)aitT1tHLk{uJ`u^A%L_d;XC4lsEA6DVo|J$O|dJO z{cbD9;`)kKQnQaS`cUl@v0MtpRR@J!mx7TXC}B#~EU~z%+0|e?p++?8z*Ho7hp5la z2_{CPXw!~%PfSe)yVa>!u%~-aoq9bu7LRoYg28Age#LnD*x-aJ z;y|oACsvaatIdhk8L?sOCSi{rK@0U}x_y%{?8wS<61&=C^lZz^ahW-;OwJyim7?7S zvt41(=*~#Ob8uf)L=P(NQ4#A@yiaG2Xr;+0rAUZ1XJVlC5|yZ(xFGmpY$_b|+BDZ; zzj|aUoS2MgPLS*+H5iL&C8wjI2uNSga1RCFitFL>q0nUTXxJZ(1;JIR-gNQlxL=Kf zl)JjRrYp_`LX#tJ`a_g(uA3qn+bNU&x5B|lf=2q~)gMkbpSw5}4#*e7iQvfFaewS` zS0ET0Q$tbW_51>t7rTN9h&1T@9<8MW{cxE2y3pf zsmW+K7!L*pye*nUo2(1FoZeccInqj4lF^GwGo`kW*GisD-X`*h6jh=?l?YEO!Mk1x z#$#+7$2E5(G3kr?W0(yd9`}dB2-~6){#a1+aK@5QgT5FJsYFbZ*!DMK#qnS)ZbV9W z)+AN{FRRi%)!eK=I1~xSv|3-LwK!J67YN0oVgD7alJgW_PGW_RRn1AO^s&jwE$Gmv zMROXHp*j75fR7eMbMrB=u+0|+SN6I!Yb+QZ&zl%Fsq4;SjM%~!IGH(>cADn&`9hIU z96YW%kH_(3#@~L1l2?T( z^pw+WTj|+q#D7}ba{CGjO84vJ`%?Wor|wHN$=YWZrG59M>UUmYv15zU@%vKqqSU%l zY_E5tbjia~p|SP-o$u{jZs@!&8xyULW!;F z5qYx**&{P1IF%OhX#jH9a*sd^W z9xZFjjDqzQ8Vdn{efk)|9O<-Y4yDbb$w&jkH4mXoTGl88%)8ZKs8+%&$GjG;LJf|C zg--auq$NL+IO5<~() zRi*XOTs|LaqDh~(NbRF4R4+ypvPI?#+E3+bvezD9L9fKtcoYR_C$&9rN;jnI(z3H5 zNXIOr zTRsa|aW=kOF9_D(M6{Kg%wmIdaOJtP6~h)oE@>4MPz3$o5eKkHY}vr!f^QLr1EtWa zY0ltVW4`#6Xi&2tq!q<4#%Pqp8Tl#>P^o-&$lG8q+=2Y7Lb|x@#{TR3|E72oE!4}` zUtX44Q&Q`kb6)*S+V#jHNNt(5Kn>=I1+yvmZTPuc*Ti~7Y)hKhPOpi_Al1%uQOmHU z(8f_Ly{{R_Td}@cwN5EpRw?_d5c{Im;i5vDPH9z~2A+%BhHZl;0?Z63Yh(WU^NCyL zIR!Et2&d4@5jE#I37Bn{g%H}Hi9j<&^xTql+B0!!evLgk!M6%Sw!uPUELF;et!Uj- zctwy?&hvLU3YSnSvev8;YgPr#fa#R#bw+F=TeGNOfORgPDK%)TcEj=4DfQ@qPQyGm zNWd-6;cXh$DJ|y}MN|@YLDDGLcH7$0d82Q{=$o=IP4}mtZYiNA}cKhO2@&m zi5(nwpsyJ8M}lDr43Y4w6$8`7)zHNRfU9N$8V^F;9QViKn*B;J9G-epv#7xU$~&UK zXn@$$9UMrHUGYaoUgDyrd^B5!F=u*Wf-sYOj`#89pWl}0 z)Q+6%<_VgWW*f>S>Ee`%(;LwK3Iq;1bnXOHQ15u&`^$e2q z1U4t?StKy5jV0$ZtXU)e$)IMvq=q1L5L6%I0-S@wDrm$&?Ko(-)M9-dD~G`)0)8gT zX!eVMBV!Y!`Iwvx#i2I~sqrgbI|mmu2;T7}HXJ|Jm%j?cD_W@-VJ8meIhJ?$CW3yv zuvV3eKzQmYl#acFw(Ob*=A9 z?@9CarMj-`{R>AQS_D_+L#I$)GvmJHo_qGDd(pG)nl0@tS$5W?oOOi!pCJ6eS#qP~ zdP!Q^oi6Q|3w(6>gUicX`chl^k{t)`o=r+e)1}_|_K&;nbS1sd-K|YZhtj3(b3-4U z`{3Mi$KF)O-emi;3zw48LBJpgA*F(=l(*NBF0H!x-J2)p#JQ~>?f78FT+95M3)2hX zdz+HdOAqZrRpU(Gt-iV5rONF%n#&J~52|ZdUAB@^z^Iaz^v-8KK6mHb^3DVJOYS_7 zlDx?yr@yXrqmMeXk0$D4Xy&zBug&;w`I1!~bKhOCe&YPG^OLe4mnFMjq|xr)_et-M zdp|k&$-#w#sj}|m@r$b#u_PdVZL_-Ek0ilUee=wG&E1-%vKN!i7uNy=T2lrP4zraq zl|flr-6#QsqQL_AT#>WOSVtwrwlBjREssO-=*co@Dv2;Q-6?i3YExZg zqzXb7(=A*N>YqVVmGl(#oTz82nF`h;CT}4KkWs~$>K1hfRn%efM&M~yV#?qUHpUt> z+nb?4d_r>_VH!v%3N#yI(l`*H8i-BTb2iP=WGGsoL(usZT7vR~X*@!4Fqw$JRP9&I;?mMsT`a_WH&asuEfaOD{6+T6S3d=B}v$8KJljW4(Mq*{BHoX_N> zb*Eap(aulHs&1Z1wjKEN*izZ)r1SJz;x^ImG(>^*(JzRenJl8*e8Ef>qzx@hIts_I zyP4QPL;y`8*Df1G&Qda(KWjl;<~B5ghA;t?Tf}e!tN}_%1~;rTic*Hkj_JZn0lfLV zXdsqL*x*gTB}!S;&ibHH7et}Mg~#d{wi8a=WiiLBZ)c@KseFR5RJ~)SE2UX&hIo@TBj#sWqHn^g$-NZoYOk7!aVoH zEwy8GyYea{y5R^qP|pxwv#M`cZwu-@n^i#%Zdh+C>h(Uh9`PQ7n{_H)Wt+0y;6Zu( z%w}*bWrwm;*`;(D>b1>)3E+j@N)Pr#76Qo|{IUQk^u#%lYD>jZol5VQ9 zBJz<}{g;DeWGH5W4M*hAZN7!K;S$0)mk_9&Je7z?6LC2dlP5xfKrkY|IRWv6nGu;% zn{w1dB!X^aepjWc7kwa`^CX)nhPTm2tkbH^ekHDZ2 z3TL;?veX|?qYgp~Xi5{^kO;}@qFY3`=@gmze4wv2T};B<00Ya&mw)mfWrjNRu^Z`{ zLl0?=(}}S$*e5uYv7jIIO>{r)=J=^=WY$SkBTQ14%#8Q)=`%-;z;=J^%*kmtqX{O4L8@Vj zPl#DW8k*zv1SV(v$~2dGWiG4f!m|FZJWc&Al!rR~cktkVibS{R>g*XcXAsHM=g6Tk zNVTx=>bQ(!up%`;Z4gfNHmk%})!!$t05a3n!Eq43`Ug~os7oszSEnX@lYaFwcF8l8 zOq*6E4x>Sw3q{mCKu79oHB24}ZR!Mh5%LJTs*~gqOICqa1eF+$x|cj+AzlZU``8E> zY-A9S219r?a7A;BB~&P+<1q))a;!KmpI?9feC@N5{zs@18v{%rV7P?~fTV`zO3c~j zrQ$93E9>ris*=^)7Y;0W4k36`x$R7Qs+K)+$|FB46047iD;7(6#frmGReJ5EM;@W7 z9{K5-qv__3uN~H^(uabTNmNjXLNz+uuvEM`UDL2!)0wL2T(0R&*7VMwymp**RDClr zGkI%rZivfucWrks-Tke5H@w#-+3+<2#P z+1r=$_APr4rM!ohq{BvTjgkA@viCX2w;xp9O)Np=)f;X7i_Psr1pchO?IN=hHqK%`Ap??~=>CHY<7QrU&1^TJx-ZwK`OZn#$7Kp+?R zvqMf8gg}t*jPn>rB@uj{%gQYv2^+b_(FjRUSN)K1&SRPB z5mAX8p?6G=xI${x#~B&bS5RO>t4s=ovaB+FWRB^ZZD)^6=P1M+%{*HhV)4{@AGHB&7D@xaZ4B;$ev+-I6h5$euy7dkx03;OL3ra)-ewf{80q%&5M=~2KH`c&_ zt(VWY*yzb@G6E?B&Fuhr04TYH^w6AG%T*P>+`hEHYPq|{@js=>^UFf&xJTo=Sz#F zG#yz&*wX$_1hTdrb$U$?Q*9XV?P73*+#x2-(&O z%M9*_8Hm@w{~#6zCf3QdxI-Apv}<5WM^#WFGx!1B{K#~%4k;+2o<~83`v$P{gA}3j z6<5hf%^zi!U-cML3F%QZr zZ@hi|?RS2#T(&(~wtcSnnl<0-n-hO;?_AaV^xclTwfBmXJ5MIhU07=Lf9AZH*~12H zTG0h#ATsM~{fl?qMVLMtqf5=9-=fhhuysELS^gdRh>*M|`Dj4wu~U;I`j7 zK0;k7fs!VRbFC0|x=NSVSTcDat8Ajx(=#MQ_FR=F5xkj!rw@6!4-fS(;Bh{!^8@_? zPb@JB*mgy0$XOjGbNKW;_8w%$1ZQY&X9{eI6kUH@wswNQWe)u%dR-ywAWfl#rV!rT z9`?v|j(q0y&Rr{q5xolgHttR6Sx{4mh#7Q1bNQHUjqOA0Z&Cjc)wy_Mh)yuGFe8er z(tQaga>0tE=MLaz0O4V-ucYZ$e*`a^4VSap;KgQbttDsb*W}Y2=wy^OZVC0%Ri9?s z1izVe_&L4Gza!7Q_=5;@zoto-(Rj9P%a|^RjhX3I|2;|-m||mL=&~BFLkIGwI6t3W zFMo;;|AwddpJ)sQshiw_ff1cen8b^c=B(59&nNW_8HEVqE@H;PY}=rJY=UfuT)3ob zA%q6hf>fM!Rd9?|)0%UM*qmXWo%M2Q`T31z){1_gm`mT$xwO-`2d~-2!&CmJVbl-N zpUJ4L#AV>k?}dvPP&YSBqEE?x9PH1r#-5Ac~JptpnV@d2l zqx_WS%fXDd`X$jEC4g?3o_6TQ=K*jn!ugrSyQ3rZ|4MI3bfaJ2(JG8R6@*Qe-#^t_ z*IP!t0GldO5L^`=lH*p%PlgQtnclgD4fh*Cv>gL7B!3Qwke>5Eg!FvO>e4~Vz`nVHNYEV)7%`*4RQRr< z-oE_v8Lt1!)w0RQ*nQgpXaVn_l(8@bI&&5#Yj^24bl2NmJe$2GXB{{amYt0$XJfLd z@9y|#&Z8NIZM=ZaI=(F->X#V3c|;qVXBHD{?i!rTx*<>LbbmgxsqL0HD~XyRkk5MWs~t_S1xa8n<# z)_NTm@cWLlL+|_E^DP{@SDqX^m!$iT=O2R`XvUwK^;+r5_tS38U(xW5tyyLpz}6&x z7F&~^bD$u4uHBj>CNUCv0K&Q|kj%o}AW?RnBjbRn->wM-&NHVg%fQ_#UHI0g?n;IB$|2#K{t)X689+j?o;eb;CMEMp@%YX2gQ6 zo955QzODT{DP@_%J9&(7<~eLLm>u5V+;C?vgsTZ8n=+*{+fo0Pa&a3C1{=+S4>`Cj zl)Lu~9deqL#aDD_$jKUt1_SD!B9(1u*b?~n4P29)eTPWDwM3M!)#evk3hwZk^VPK{##1o`^&ZV0 zE&1av@_Vp%S4fs1!Ol1}<`NuoRb?F>*?U?a$FBo&ssEh@tV1WptUsjJ_$zohM{_A( zT@x?mDm8L5=^gb?-qy!T_1_*=QvU}Hd83)Hp;2XKzL(~@cWyV$eDC}o7F)}6&6yuZ z7aJbikCm*k^Brg;&nZPYP>QsI*40)N%BZRcRB^nMrLvlq4%01BJtIL}i^}mt18!OQ z)gg8evt=dprTzq7&f4l z_RpOA)@~~O7~4=y%Dw?*YElPRN8wt2&_|al*@=r!F!1egY}y^hUEFi@9r4IhsHmEV z7zn!XZFm8Pu+?@!oEsx@xF9ZUk|g4Za`NgAg6dDH=4iOtj$)2EdJeQO%h9b~=$&;a2jG?-WO0=ArmE}al{|4|n z-5^a6xMrY!Dgj~}z%}4wAz0zn>quvxLE*`q75;9rP|$hPhPz8wg(2Y>$TBJxk`Lb! zuHtj^S8!;$gckNe#$HQE2H4CW1DYHTs^j?B^7fpL9Xl1>}i zRg-8l(-ZF}OC9qDQa0p|R-;4>31sDC+NJeRfWo4YO z2L+jhUOh@7w!l9^C|g^pFQlPH%aum>Lh5-mIWaCQr>pUWBNnjnIli7)$g=BY%g*MM zvzb0nTwiNR6T=ovMu4R5bxcQAQqQZ)D74{m*Gc0tjyoIacHqr|zu_?#V6nBw9T7AB zT`TYMafpKkNg-}HV|k+Duz6#V-vh*QWy2P|LLu;-5l)zNTHs(bdI2Xh)%#X;9&Pg5 z6?T6#if`}O@9?1Lm%#^5>;cpQU+QP)bB zv)K=SI63eu&FE|CL-}cm>AB_a1CL;18=ui2NBwVT9Ut-1rl*om0SKQba^R-NZv+pq zOL!0J+8^5R1Z`F~JakaVi5o)?T@-Q)?$Sra2=TY2O=&nC;<{D{A>eH@Y*Hd^N+Cdm zOSYt4G3G0~jH&NHzR3o%+Ay^o+1i=&=DDeewE}$uyW6rBE=H|c?N|*EiFQm$2J%Vn zH`u1G#CTa;p4Ee3)l7cD%|2!u$pMITv=ju0-z#5xnP61N1Bl{gfdC>HJ9A)34X|* zw>`7qU2J1pw)G!9m8r_1`O#17@pg-qLyMlFYqrlf?^@WqxS639`GF2zat=9f6~BYX zUXw1{WNssJ#S_yQ?57N(3vPSaMAgZH!k9cw6abCP=iy z&rG;|K77xY2vZn8vh%%!Kdk4J`F!IcT*;vGJTirBhtC(78uR(o1JqPMd1RDQ zk8W(Si$NUb-J$5m2;b4m0wUi_C&@PTW5A{@G?5-D>UAm*nz@$*dh-4Oin5 None: DECAY_RATE = 0.94 print(f"\nBuilding network (784->512->512->10), seed={SEED}...") - net = DKPPCN(batch_size=BATCH_SIZE, device="gpu") + net = DKPPCN(batch_size=BATCH_SIZE, device="cpu") net.add_layer(784, 512, TERMINAL_SIZE, lr=LR, ir=IR, fl=FL, lmbda=LMBDA, act="linear") # net.add_layer(512, 512, TERMINAL_SIZE, lr=LR, ir=IR, fl=FL, lmbda=LMBDA, act="sigmoid") net.add_layer(512, TERMINAL_SIZE, TERMINAL_SIZE, lr=LR, ir=IR, fl=FL, lmbda=LMBDA, act="sigmoid") diff --git a/pydeepity/__pycache__/ConvolutionalPCN.cpython-314.pyc b/pydeepity/__pycache__/ConvolutionalPCN.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..db22f075889db689ccb43100521136430ca734c8 GIT binary patch literal 7518 zcmdT}TTC3+89uYuVOiK^xf-w;SjT2-8&X3XD{+0pBw!MZ2eLT$GFk5q?8eLP`pjUw zPNkx@Qf(=f@(`<#RI=SZgjF7#sH$SCl~VVijvuP9buegZSLx-&sw%BTTAHW!|If@` zSZr`xwQ7#wpMU;y*>nE${g-oQTX~s-Knb6F>$P8&67olU$rh`@EQNuYA}2YK>m@y$ z$~7Tx>NWM4RWtBJH1}G1tg4mqmR_4`2brap@3E*QO{AJM6VX~rMB52&pej3f{@H2P zd9_ruc!|&cfP;4*ra!*cP zX~@f%yecQJHss}Ejq37qs@qSV|&a&(lUC(ud^O@Ws1$G2>mqAqyS8m!##h-K;{vL)6*8(88!zU-ns&DTxWf68xO zd~WXr%U!;XdZ2Q}mJ1-jpIDj&vi6p^!stepGYDCsZjI1_;PeMdMG)6c{O-<0iGzNG#u= z1YW|-8Z;i*$w)|rwu(w4AsKra+@xkr1VSKSk4eoGrJ5-M{|TyDqvNMS0nHo;suZ=| z@3T-RJSkv+Xx7na9B^)ifqW)fhVpVC2GaE{bfdHa2yUqIArOYgE!NEs36S5{Y{*#g z#YQS?GIr#6_X-CF|Z*J^Uft&}q zO601DyE0RaTn%!y$kmZDPi6yh^~g0K*NEIkph2@JE7b6^XIrQvSBt;3vfAI~}%u#H;cce@E%5|*shBr5zBC&{j(;A*~tuEtLW zau%ewL^=wpZY0N1*Q|)9Z2vI#=pK_WAKIO;z2t%M&ATRH{Ngdp`7w^$vLg>xf?dNd z4euZJ4||8t4Zl77^Woq5U82SB+MaC$Ptp2}p@Amw0kDF%;1^(VxB}5uXdMZU-!s6% zd_i|$L5ct>Dj-GV6qOD$c$T~_IWG}m-76(NJG1X>Pqgh0_+M%T(n6ZZ6d5AV^MstK z5Xm9#9GQZIp%3O&LN>wfc*)ny5sp?t;}ikcZEAa3;{pu8TLcDRtz`F@9Fxx_=pJpy zDo+#$g}|vLW!kno8V?4d%AU4d^A;FXK?n2zCNWpND>X99O{Zvu1vNelH>Gh( zt5HyrQwVKj4bb%&&a4D^whmtvMfcB|(2dA#LV{6`3P_rfJmH)vSRzcEg(oRM#}v&T zlbC`uE6hd?(GAc*L9A>kDuN&w5$jnTqyF(YVP5bSGy*S}*-UC%X2WyU&s=ca^Hir1 zB6q5H-SO$XVp-KR_r~#ip2oS2XYP3DoRj`Z4~QEUowZkzi_WUqx{tdTT@};f^~#TxxtEXr zI+?MPlKRhyrNohO5@+3X^QWZ^i&b^go=ZI!x)+6~7KEK?VdwkZ3!MkkodGlQ3p%^XX4uGOdN z=Qnm{P1k;pRH#}x@v>n>7zX^Mw6?1{cCbXcYTdpvLx8!F7tysaCz(1dJ z1_cU-P>x55tZEBYYSLBeg2Oi+k0dxyO~NTJWzubj`C(jW_M9*TyteWv$xciwyR*4t znvvIK(_Q`%7Q*YZCQCm8=5aa%OL%0`_GA|FAN9sViABUiiM1VWFH%;09}#34_X-@m zR5_tAn@ibfFG%SjBz;I;M1mRE6S~;_&{V;m#)fynF_N=(aWh?)`)2x5rfZIrW4;m2 zlqkGm{;2du>CFST4&6L7zw59rOdmaFFt5sbUmYjthO-4tj^X4aFhz!G$sRe^GE8vO zYnZLyoW1WUGFpd91CSJihAw*)!LYspNaFU+=!-Ixxo{$XiY}-Tjhbkit=h&|^T> z_aPYJ6m)$Cqy`zJ2Dwz%wZ2r}4bw-C8;<$yT?m>%?zZ`p(%Yr!A^*bA$@I|4`5sA^ zr^BZWCNY7i8sz?eP*tA?0pzLrB_jGUNQN28D56{`!qX{`Y1`JoQ}IwtjVtl>)%Tk} z^nKv_$+qifK1$w5{_ORS0bk-AFRsRy9exfhjx7BU$bZepSwwb$^6zF6?Pl<9mb`b< ztr6oD80zC3+xh@}0R~WZ12F*DfdJ~5sl41U({NRMcjW5G{D!9yT^%YQ{=?pB&*+>t?8yrofy@i6uBuoTyNqnqVA(C z1wJb8p77W6-YE=kTHPu%8RIYo;8hkVfVrlN-0Ry9g>M)>?4R5K+YYnM}55x~gJ{Xzz?L!oFakm*7y14X_p@k#I)9`mfS~xM5 zJ~1|bBwQHU{x_(oHFmSuG3Y)ff-74Iig6HWt&amm@i0s^#^K6{Uv2BhEd((0E1Q6M zoL|xNPbxx_C(Juvf~JLerx(@jy#kkr@W^8oTCsvNV5Ybkm?BeLKksx;Btm$_p8Kdk zUxs1yTOlL*p8Vlmn9V8n`oPaZ)z8U<1ID~SFI7KzDJ{+@W>KK-89j#4D}uY428 z7vCdOp$`CDZEYUw`vB4Sm8G7}*Mt--xb$jS{pa-Uc?7*8ba~yY{)ZHgf h-6svdC7%1F{&UL<9Or$LF4S&K*KYkA!8bP2KLH_p`S$<- literal 0 HcmV?d00001 diff --git a/pydeepity/__pycache__/DKPPCN.cpython-314.pyc b/pydeepity/__pycache__/DKPPCN.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3139c654b987081c4a78056204e330283ca27c51 GIT binary patch literal 9331 zcmcIqU2Gi3ec!z=?k&G5>Wi|gA!XCN=y;SRAt@}$GA-(Zl}s^5g^Uwh=jH9(-A2pZ z^Xwk!lYv?f0UA9NMIMUEZGlMbLnIV!-G>6{Tl-QpPmR!8F}Cr8+X6)p6gZW3n7;M* zpWVG9Dax`_WkJr)&dkpK=l}h2hx>YU1=p_zZhvQJkD~ksJ;YZ!>a5q$xuINDi|VX$ zLG{&x%2lP9m`z+r`bpVO&Zhh{#*(v{3kkmqV`?!qt6j+Y*@Mb%WmGAq_bSEAo9g^< zym{w-JQ?==?qc$glI!}piszgfB+MpS?Q-3>T*t8I?y130sbcx1HOsG+>cXuG?s?_T zMv9V(9@bA{4?{838_JuCuND+M_Doke<9Eq%tv#NN#x*&vx5vAq@vI#0X^;0tf(K~&`XXMNtdM@roOQ-HH?(+v0Rex|%IidPP zpu)33X6EAj{Fyl;olz8hn#~JtT4mo_B57@lL5tvtKLD|~rX@I0F_8f)D4*f={U zxO0u$c{Vm(E_0g;!{?^{SSUQUY8g8NaW4M+bo}`-HgWXG(XZwTy8aEvwytr;SGj01 z*RKIA9GaDF!}FNsFu!Jb%(j+=A)0x{jdG3IMiZ-Qu5Ee@7|<=pQDa#T9FYh*x(<#auUd|mX9jZ}KIvO4d^@Ba`hxK1rgN-OH$gR` zwcilS07PP0eiC3UMVw>#o;*6C0N>Z!@JqPa5Z?J{gx$~FKc3|{; zPB7|Tqf!A=NhWKA#HB<1?$INRmc&4Wq+T_IWjOwJLhF9BZj}w&ZZZeV;}x)=!oiSb zUJ5tCmO^5TnY>~&Y=8vpznCX_mutY8#AMyT(Q#6D&0#fem_Q~62|TPn!9X-AIF`>q zK$GK8L}kwfV%-YPZ1}oY$LE=0t^!D-%86}Q-Su%eOF~xz-NB1Sx!e#&xe0z=VlgL` z4F?G30KLtg%!Mvpv95E0j{)0~SBbzRTplp3gXE?5qKVuTJ;$p`v}z*9j7o(heJM}K zFV#J3%Nufwn0L3m0tP3Mo0~6QUOYR;X6G1sGv7FKW%lwM(Rm5Dssa~8JytV3&=Xv~ z=>T?92Mbvh3s@m}uUL-d^GVYJNRm9PT;rs7JWg`p`QDTtDyEC`i}Yf%P_EZ=d40Lz z6VFAoY3n6?ux6}U!1BaejT4a>$YYd!$Sjzl>@xk+1*4!O@xT5RG$&za7nQ4t-~O(mm>7LV`MO$zrCm_J4L<=b zENn3~MbLrpkyxkQk(96v}xv7w&<(?hC)-cm_w zWwJu0l1JN&-3X3M)gUla0t7fRH80%nfHl1-lk>XeH>c{&xEC6;LcJOEluCvJU5MkC zO2MvZV)Nhxw)V!*{OX3Xp{VJ}ht%9T)}q_rB<1bzQ*v+}tCTV!BEHB4!LznLjt(K_ zjd(^`NG>XG!KqprI!@5zI;a$tL&`ms&Ybod&=!GKDiKqcO1V^!^0-|Q-B>7k&;;2C z1vox^W_*17h8UvR{WRO(K1#TOJjMWKaEza6y*In=Yx{1EJkauDHwL#*lQxmmoI~e{ z)P$cWDVvg%P0KbT+b-E^XcNV3QJ>0!;@yHgOq_*iopty-en|G-SkH%#30njS3;z{6 zNZTvRrV%6|c%m0C&*hS$k2XnHY!?Q#AN^cH(AmTwnur-fTklM5U3Od_3AschTVsh#1Qgu-)s*74aeiQd%;%m`f;1Lp> zl<@{+Pk*Iu2xFyf<0bCc6(|@&Z)i6-jF$AwpqghtcvZ2bl2C_^8F?=_ru3X* zqZ9ib`3TNU=C8tRsoE{LajpZ69@;YaBJc$R)y!qYD3C8m_{AY=oM$7S z^h+j8$|_`6?4woLHnMWM7@?Pci<*PfFlu(#T0Bn^2heyVC6MUwisYJDI2efJy1l(* zY|^`6>`%v)Y#)jC{*QVFZoTkg^b)9mbVUzVS&5|4E$?tA%7#9P?QBV6rSVJNa%`)y?BfjT36X!sarO` zmk5PHswp~M>~@?&Kj@2)7A=+#lFJMHZivD zYoiaeLmfb4bbV=MkqgyzXmq3#1w~Cd^vO89LTA{dsK-e)kKZ{0-zxR;6?<%?P(F<;wI%CV`gU( z?4-N5ys?c)u|KwjND>!Pkr2PM4eCWaY$4K?mnA-8^_yv{-<`>%VJq5u{m;-mPW>W6 zQRX*uFV@mUk;^>Aw%e{eaaeT8i1ycin_{O_kz%LxREi?v(5+Ix8K|!Z{o4{vl5KLC zAnDZoplfbsS_q>VB%FHCB}1WOM`d921qJ&}oWNT^Bdy0^`Ze{TLhUEreSdJ`=85l% zKf3mPQHwbF7fqxAG|9q*FknlkciD=WLYqW3@Srf$fOq^9BvkuDLUXZzOnZt)qz=uH+ z1%C#`N>v0sVJH=*C-nturEYU%ou|$l4bNNPmdR~A;uAFrRN&rFdI>K_9*{1xKr$2A zu7yPBLYRg)E<^!zs)U6dVTKK7%4`B4B!`e{)T8|KW{kkaGCC2UOsC2;B))7{U4eYm z3fwEoNaS@&B_q8^o*R{BkfAguT?e_nC_LT9#cXEhB&iG<aJcOOzJa0&;m_ zKHhZ8$O!?31szIX23?zQK_X%JNRC3Jq(LB8L|mq3o*Jojh}|f}iUO=fJO+c22h%~T z6qM&N2SNFC3}OGxwfp^}w?}`NyOaC#!*}2Q>*l@YU%dOH%m31U=_lHyM@>Z+UIptb z>wk*oH>QJ9VIJn!)nhUr7kLR1RT4{Nr+`3mOhl0y(ieivq|`J|HePis^-iOaN=8ke zBG>EG`@4Q{;Qa%4ia-3;oo{`({|M#KPW^+-sGa)pv41}C;}iFnUjONmaevA9@Zz$3 zBg86UF}@-nNZTo)5~bD_)z4KKi8%3gNC`dNh>N=FVRcI^;al*WI_%To9Qt~PalVbG zw(A?cEl*Cb;^aT;AHKPE^SxVd|DgGP^H1Kr3syP#6Yb=qtTKT0l2v|$&TouWo`$(f z6)Dx1E>fy5U8GcBrl=LOQyTQI9>y_ki3>7_!pb#IN?w2xnMDgi>g?LQm>DC{nQ&!hW4}u(Q6GU`q0B&=!I0K^TZcLd7yI7`WQOM3CC`| zvZO8@q72~iX!SuP(xp9XVToRBIHZ*;WY<NNyz|MQJDed$R4vXS&SQ##Tb+~<67K_F!ATZ06NABG6huYit2QD zimOGa)}ksWP{!ma@X7+SjT)~^XUZL@+7@{eD8!Xfc9J#0JbRIhd1M0(v1|c(R4(-m z;Zu&X+Q9D;- zy|<__(0If%VZLyDTqeJRBrq80kv&6S$^pgnuw!3QvIlM*zde2Ty?^?hf7RalsDIb@ zx+SNEp|u($%k@SsUH$$4{@dFd|MH2PG8CGqEZIoA5|8;R9bP(hE09cU*1MJEt=^s;U^u! zBcF;pE~dLmC7AhAN!Bz3r8)&=+XN+O1SK5AtJF|bCn#zVWWoihV_~8RcB43g+KyE& z!50t~D8M~zJGiNOgr5`G7>Wx@;UjbkIS8G4!Yi+Y^#nq))J@d!-b3@hzXxsU9kkwG4Ude3cS5KYs% z@eK{}+&Tqy8#=ktpGl{DV;m-uR-W3(K&Nbw0{b6oLYfgYxgOw{ep!K%q97v@WTk$NJIE-Pf)Z27fAl-K>p$*EFQ=sOU?+tN8wkXoeatN zvRK0-DKT#XO)IIY>c`5ykCg-erVM?ojC`y-|7m7Ap$>nq_|x9e`@N&TROm_W^nVLF B=|}(o literal 0 HcmV?d00001 diff --git a/pydeepity/__pycache__/GaussSeidelPCN.cpython-314.pyc b/pydeepity/__pycache__/GaussSeidelPCN.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..99000ddbfae7bb32fef2c4c24f2b67a52165d375 GIT binary patch literal 8083 zcmdT}U2GiJb)MPf?#>SXe?&>75)Ji3bL(AF5{1@{RNIuyShQk_88W0J$65_%FL%e9 zomt(P6|>Ys1=MJxhoTKyAW~bvQu`1Q0pbKfQ44)={nFG=5z$HxrtyQ(0!1DQP^DY8 zZ~e}l*pT=l7m_=Gl&RRYLpSqkr^fty7Zzi5}`L5g{8TAUC8- zvMx_a=VV_#C|#2D*i`IX+>Z-9KGotUK#NZ$&&B){XtLfirJPIq>4VZ9X-Lu&dnG;j zhCI_(U%dH$G9Kc7s~$frWl}$v@tl!^SfK`}T&Vb#>lpU*svPv>mn=WOZ26^pg}X)0 zJkM+{BuOpN!^SAqFeF31A-y5_vM%|tONt-wmt64Rx6I4qvL4$j*~+{GEKZo+)qB$u+R~qzmL{|h|ZO~gIdRoxi8ua#vt_pewJ?Wi*U4E;e zbTvkHMz%@tWp}iP)TG+J@(87A;_Tnr$s|1N~Kxd84X2#|@Jew1UN%>Dx7JiMwUZ zDVjN(8Qgis(>OD1jeV1uRWdHE$c;)#9hp3logCM4&uc5kvMa~4D<`rm;&I`4cHu;J zLE!9omYvA5@ywSrLu_1YFzETLqFJ8mmpEgZVBgaWle^exxP9BGf$v;SThXeOg5fhy zTe7(4Ya>;UM0CxttDu#bT~J*x9B!OMRN?HJMUtVHWU?eShloNmu0tG|Zn;wR8Dv09 zVSu^>2PtVQ&p_m2JmcKuzi=pCL_@bMII7J-u&T{`NEbWhfY`hcxgmA|#+f!bp?_s+ zQja>H(F_}+E!1)$dj&P(h%80uifOYGuTF~jS*_~W))l6Ox^AdiTZVD4trAmR2TCLn z(mTDnn1ohIILs+%?h^6XloQ#Mcn!?qYtu9Oh55;8ZSu95Nqy?o$>}*#0*$Xa1;*_f zNvsU3)Jle@iMK5>2-qvTS6B{e@>xM0F7*@ z(9iURIjzQg4STkhEZE{T%fK+z!uu*12p@?i=L*)+5*T(&Ms`@RsLL!d^9ob|VfvU> zgBh7s7CQKVDhSduGg(dskz(&A>@JOSA;8@`nw5EAi7~-dCd1bKbTQ-WYVVUjYNT;vM~&V#PN-Kk2D*bmoDQ7 zo|jKX!aFGG(qU;;o(97SuUcU|Q1W@QzIb|W>fV7)-*f;%9(J`9`F^dNYyS!L1g1BM&ybU9#GjZNdGZIU-3$@2R zpJ!s6w#AXfNRi@0rn`yS13)I{T?7y+N+-DdSR(jwNB4R{JSL^Sz3VA~DpGsTdRm|= zp{)XKle+e-w+pmG!n968*SjRO<8e2kp{P=LN4*haLc^zl?9gy3)bK*j!pVhsU7Am3 z>$C8$%fAt=1HF?Y!~?mIN$@YCj~^tU0R$>!VqFuS9CS7DJi?lF4VJ?DX;j)M6!Ic~ zXB57VCLbX11p?a_4v$Pj=7+Jwlg(ia zqnaKYOJp8P0C&B0`n%_gr*NOYz+>B<)?;O8F19i2ch6spF-y2Skj|AfN()MF+g!~;8vgbYi%@}c}P zvx+6(<0MuQMeDM94ul{~_8T{Z(OvVzwkZtPrM9LFJyH(u7Tf1__b^(CGhkKbhw!E+ z%knpgmn}#0Fv)WG?j#+}!}ki}Dcm6`v)&paefpz$oAq^*|B3qsvZFk%Jgn}-BH#*R zq<1u&__y$|OTxPr5zK5qrHAeudZZlvUzD*pU4)GJDQxP=51%p$=Mmzz)@T9`MSp^l*)(P8&Gf_DvLZ%FHk)Iac@*Z=tSZ%zKRt^32a zBWrC(?&y!&MmFM7|Ils)(E&@Z-1u{VPm8ojl%$!hhKaG%D1n+uhWJ*P9cVu3v0rL7 zrEM)DZA*-`AlfIw6IXsMkiU)$6gN>%g#OiJ5O*qmkeYsJf^(x5#GFcy5~2QpXK2)pRzGWCi=WjSuUE8i^vf%6 z&Dgk{;d)3=ln8M3oW7vp#j`DC+POKG?&W&mp8FzC;^wg47z@4#v z2WN&^(!_-*y4k}leVcB^xMd29truj%D#R#4kH`a6v5h=!gA`==K69W}kz2eBZuvO^ zQv^hSA~q=ljPgHn&|v=#muN02)OQqPpuSI%NR@9cuXPUH8TxMKZssqZy?6DU+G_24 z*WaD|`_3~Dlr#VD{egd(> zAT<+p%S9@*p|0b*MTWV&;bH?#e`r*yBM0YSC4$g{*j^JGuFfT7-L0O5&pVx027@!P#`58fWUtAF?6 z-HQ+RAEVT8{C=>{E&W_J7vdcXRpXZ{NE5cI|fU zPp{vDy_|TUoY-wIly3@qc^Am9%w9gXr3gC<6O|U7>Prco>Prbd1(?(oJw2wtmeeh_ zL>?3*S;aL=o;dFUlxD^0n@7z_Eiluli8L`td8O*olFdY>!xhZSq(0UBM4l70EV{1! zWJ?r*po#^BC3Yro3kLYOwc`&?-8}Ve{+*R?uRLr$w4R`zq}0~Aotz@FB6atzrwKz8 z=v{9mtWDZ8u-;Bs2VtFrbxA$_>)pWCdn9$>aWAk?%fwH;MWw2?jYFWI6j{IGN(=JB z;f2YC;>l?ALFDQpx4EwBReXDhK7^F3wr}NxOv7}C0;lb|m8dL@uQd3E#4Y45^{;pM zMB`Nu{zui9n9Yi~BzYR@nmLi%Z23UryV~VuTph_=9#uJo(0VninY+9JYM8)KcE|Gs zE(2_>W(|u3KZxUFV$icA=z=EnLZwnpD(+Gs!cd*%~z|29rbkP@vRe|hVbc(IWD?W`Fz1O^LbIk(Q`v`8p@P%x zoYMu8pCLeb8mBXbQz+mjK(GhJVAQcKGmor~TZ{Nq8PaA4;xmmM-%P@mH_37S zjX1*zgB@satf9~!l{-kHF0_bZ2}*Sy6I>M68JD^dz|i9 zLEm;02|s=c&V><#ucQzeD5pbhKF`Um$$vaj)OuW&<&UI&A4!A%DD`|K4SXaW{UrHy VS?>Eo{lh)kwLRJYkmyOQ^xqqe7eW94 literal 0 HcmV?d00001 diff --git a/pydeepity/__pycache__/SequentialPCN.cpython-314.pyc b/pydeepity/__pycache__/SequentialPCN.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2a8661eac717168427c64e301cde4e5156ec046a GIT binary patch literal 13979 zcmd5jZEPIZac}SYjz=DOJd(%n<)?V0M9PwE$);>eqCQBAmg$wWlJ#|3-Yu!Kk9V8h zJxQb;KnT*73X)hx(3(nW^AR8&pP`MsAbFE((-1N%63e5ho20810W!=tN2X zv@>sa?~9aVCoRxNaQ5xIk9}|6%)EK?=6cFXJp|IfOkREQ`z?h0JHD97Ze&(xAajYF zVK`=p3^FRyOwJI_GGrOFs#cn}4%t*Y;H*QAL5u1H9K+d$T!U`aP0Q^=p1~5egytPX z-oa9}bg)b<8}zBZ!E&{HutKdE^sD~CO0}|?RFM|KIco^#I?ar9nS;qaw^{YPTE$u0 zNXY#O1MeY5vqUEWx}KBNSTZ5R4_#+8e|RjWhA+g_b74tNj?1E=Ko#deGNB~nV#ukv z_XzSpGM<`9X#QuDV}}Nk6H;7MMFZy=F=*0UULXQ5LKC&jz6c(;LfQM@OMFEQ|LiuY#m zr3T(Z@n!hN`2d!yB@|OmF%=Z!=8Q3yfvl?-9mNl4zUVRiF z%;IYed^uOAR`87K=Se@KRss_=!2*e2NQntGCd5Yu4oyAl+0EuK?1(HzV-YoWUStQ7 zQCJi9kf>fr$}h1OWI>WdnH@{Yta?sl_llwvQzzMho*s5oh`b~wqP?C7vKanj!nH#d zT-^p>C#)P#&RBMQnUE-88_4s_Ias^H%t-?O%*HyXWF*eQS^F$7TH7ZUKxv*)L5-XX zE3qk2bI78ak`sqQ78(23tg(a|vS)PPuhFp}kzDJkTzl_zIQ3E?f20-egM6y;GX5K zN6b_L8OmHwK?doUW2(cTLeAO=Ip#mMjkg~=&k^3S(QF4#j@fSX1nh&tXw{f#$RT$@ znH)mUjR2cy9w1Tdyny?OR-Vg;qoN!;kD`i_p}B==nN{ZPhE z+;zVpR<}X8b#+%e^zvZFGj$bZDt#h4n=tYwX#8;2z_}~P`jj;od*@vO;M%?nKIWqI zO<&r4J%D0Cmllaww6qp|X?dnl1IUd~)m)7VMO4FaQIK)x4a=b6fUoJQ(6yj+$8U6Y z8Wi6SFIsaU#n<#f=YmoTb1`JQWw!OUt7ENfqZYvrtbR@sYu2Qs8j{G!z0kwMNsRoI zPvSNtYTLC+ByrnZfWH0jleiSg1A~oCz|JL`i&p=8fxPr9}ZmmP&)y+^E*5!r*6==d? zz`rZI%BPG10^7dMrrQjRzD*S{sWSq0W-KmVx9F_d234tkHJNe`AhfDNsu^wQKw%%u zvf^dmZC7n>gvRDbJ_oVnr=dG}7l4Pe+#^scpHA&ar}o)x^pP6{)S~GRqf><}B%(=V zv+xBmHhxZ3sT=y(DY|S9<4o|vLD+ug8Ot)GDox#CDBNd4MvBi zNl`NEHj{&(?A)XYXPMW|te&d}ZeC#{pIr7VE~#}h4W23EW)9Ed{Se=3JN9~TP#a$Sk0r#a6X5R zijo#ARJFMhq@wb50GG(eF7Jvf2uA&qt1fNkXDXIl_3$>$ea*GvYFKbJ%sjv3YW}#X zW!ihEqWVhe&l;w8r@g_IhRq8Nn-?3lEPJ<152wB5E8fNhZ{uxm^IbdfH)UMJUAp3` zUvSkgxf*HjjSH?u=qcNG(}JsM$;E!`tDkq(uT@;g!Bla53o`#_ipxfiBEPrt>qCc) zP2ZX6a^Xtp`pPapi}^zco1DLVmB9p$!n-z5O9R=!bxwf-9S)`@E; zmTNa>5ce*aM(}vA^2hb>)qlYK+ldcOEbl%}tLr4qS1wuAO)BZFzgmEN2I6E5n=*bLGTeovDBvx-4q%Rzgl+Wi%$ugl=|NW$$V$WWZsOwQ`Oz z$J&nVJ2t+}sEy(b!o^Iz?4i=37wenWKv?$z0C@29G#nTEOL%D zUEM55qXdqLUe8`JE{?-~tFYh>+e95?tVpc^*iKnB$)3&acwM2hy&f5jrjRR7wCKhf z11w!o`9;jVgdh$;!Lgg341IUCg6CbUDBM;7MVAHLN#Ma~Any8^ZL_=QQn$L7TzuMB zIbE_=JkS|O#iJKU_Wx8o?k`;VEC3V1ly98smi!vAvU4sFF%&cO@KK@LT42-`^x=j& zF3v+?2|zdJk6E45 zvpX_O3r&8iB6v7(ZT1S%(|X6zurC<>7UX zoj=oUDl4gsFME*fA^A9Fy`v6ZY4NV|?7_~oDx`AWyWY` zom<>yTd<+B$DmW_;HL>UsO^TbJ%#3^j`f(WPN6cQ=}Z?_0=2!{QQJT|h3;0GO#0`w zFU__`59h;cp^Nf8A+-IS?Dm|a!wn_Spv?<&C6%Dhk04{ ze+fok4?G^qYiwlA@m>Iaks72%SQj$VbN_m<8nB}K-!axPC^Z#RMQY}o&B2aCuMOlN z@ZhP9W}hRyW-j*xtG$$O+G@$GwK-R6=D2Eet`eI!S7&an`!~TXTKE>e`Eh7ud>ynC z-^{nZEKj{JF#Jh`?N-}u@30zPcZqq^Pv3UEU= zP`#3G1I|GoY*8{tRp4a8y1R=i!mC*qqz4wma54=%s2=6U@8`HNkMeuhw!_om$V1m} zy|v+LxS+{>BL~*@jQne^p46U4r+n@dDR;JiKHg)<$k5vFoynJ|^<A@Z)|Rj1+X}d$4lwnF^0yK81v5z}x@ZcXABVE6zD(H#jYYN8zJ97(+ z%$GkFt`+*W74&UA2)h&VD%Xa8W@HaY-}+WSyD2#+pKY)!Hj|W9s+9aO!jc}6)t#UV z8b}OE8%Z<$8xHkb8%c}_bqeo5q35s!zCYbY$79}o4i&k6~^VmGbIqCSvbk7t)8$ijqPU>pkAVKC$I`(1Rj`d2pUE$CN5tR%!_ z1&58jPI_2Kv@RT0!R;D_xu^;oga71(1h|Bvq6FbK5X3`|so>&O6#a|foCY#M1E3!5 zB_c=|-w@S1hCm%4Zm;oaI$JiTuYvKBfHN)ze)Q^urG0d5W}l?%GzNoIH|j2_O+xfV zA%a7oHT!7=Rg}@C71wd#7mr^MCKcKVt}~2NrK7(H<2;9?WDh-egdG(D8HuNG+_3T@ z(jlg9fGMHB9gu1=+l4B~KnpmqgO6K{Pi_S6%dSH-)a#kDY-DK!i7ChE%*I%O)g1s@ zfyXbABV>QMuFYjXL_IJdgA8v0XxU=FNG{o7XMp$+DsA9~ebfrlVAGgkjp@>u&>?vj zbOgZ=mfp<}d=$N?S(GUSY!H(5m%DeOqwk_5Kc$7%oa@6e;Q!v8@g#7y@>FlO>NNDA zumC=R5zaRczx&EN2W||^w#*L9K03$Dwax|Ro`2K#y;pumJ$7FT)-hAtJSRt!K*PwS zkT`t=ee@_{RKQO5Qz56vD`Cna$| z^ks`ArAcUL6E4OS`RhEPFZ^&9mDqe^~sY{wjI4&#-DQ3+=~6q@bDWGtcCB^f*sG9I#I+$Uu)afys) z3>gksL@q^ui=w$O>;@N>dZ9HtTzrA&8Rw$esSiYRgUIS1(tX#udm2I_k_x@OKyM2~ z1u2>XiL54NFk*BMw;aJ7k09sWK7p@M0Gbo}r&F;_h(bbhfPR7Hl22ioL%$6Rbr$>P zWNaQoFpNM!pdvsgXwHEnpU2m)AZP`kIdw|Onmb_34qWXS@Y_x%$vDU&3Rn^wB~%l$awUY zG&IsqKc;zU=$+y9*O&;zt|4{7&#d3@IpGFUjf+u;+B!cTR+8Z{K~~V>GX0YJ-5>5j z>m}W5>B4wnRAt4HeX7{W*ZIzyBXU0RS8dd_2ECe1|3~Zfte_UPn zdhE5>0&2;{=HbQa;pvjJ%lB&O<@j<;&oGS_*lddWL@$J?~% zWogldMQ_&~Z{wo3Iq$W3(c6-3m=4r`T2&J8OuOzjk>gC1S=tp{I2QSUfB*12clcK2 zty+dcQxt?KvX`P%NqT5kH89J>EzEQkI@ZPU#8 zHyW?le_Gk~la8JlcRJWKGr1fLtpxiPf_=AxkKDDA&W#xdY3jOJwcPX6a^uriJn8bl zN_po(dFSlZQu&sTn_I7V)8)a}%U&y+nYvMXrEICZA3L60sSRD(o$h{QrMrKjyMM9! z$%Vj%D+gwd%?jy2a3#>b5NKZtbgsI{mZuhin`YdzPo&w-*~w-0ks13>>)ET1-s#!& z_Kv^WapUmK?)h!|{*~SsF7+Ir8NSoeJ=cG`WBV+VZfl>j|K$_uogKbW zapQ&WHr{HvCH$!4{l}MgANjETC|vV)?tp9Fj?h~NuN}PMUT%M4X5XELrZ8u>lCO6bR*0;7)7%TV3Djn%nkv|MmVGlgmB3S9*pPdWPmljxM$GE3Ic1TF=af zN9Uu7#n$AE^G-|W?8~=Xw$AKMZw2bL%nrR-3IurmyHz(s%bN!>R?D`gUpBQ~jm|ND z>;JpDcj{)Mi%mO!z`x7?kcNuPA3brCUwrZ;ge^9<%|5=|uxU+f zQ@Wus-2n8Qd*sc>XM5((+#H6L+<9dF=<)gf7v}3uK;sG)8dv!5PD0}i_uQncIx}n` z?s62^`DzFQ;OLdc?^X|6$ZketKssJM{$JD@`k}+e`5ixO9cbc$_V?{i0Q|rUn7&6Py=5lH#5Fzpy5A$ z{#f19A?x_dfxkZRv#BpX4SoqsnWvA00vIumyFdImz!MHnB%`S~=HX`pUq}gYqr?}6 zU)R9NDIQCRi6j&|!{KN$5)R8O^dsZxgqln8PR#8=Z~#FV!5D%h0=)H>Zy})bsDeOz zFkWMkFoXceu~7_Ig1#&AegF{><8eTlzlqRa;ra-fB^Ro`YZu+p>7_CScsZb;;q$vc zfwRb6i^cA`=YXFu{2G79aL)yS=D)_+^?Oag*Q?0f_Z$#Ki@7$;)nTpz0vMyA?DDmy^FrDS?wNDXeV=>oIp?0s1CkQX-YMuS=C5MS(=%qimpQ$e>!WZnY65pBuX6- z;;}QU$Dj|H$lM$`PYg~ZhUbb!cea$P>@v9)53^i%x35%VKS#` zhA>4lGeVXs2~~%!>z<;D(iI#?$}N?M|ef0G(+``gTW;B z>p?vwXO(U>J*7}3olt}X%u`9L4DVFaQX(a%(~73|0-jRT>6C%kteP}ZLPAaplS(i% zqZ)>i6yzyGp@NZ8gejHkh9D=Cl7=(rj6qdR>k`;7utaMeSQFNAMa|^&Agj1v7cw~` zn==G9t(ef=EzGFtoL*XUmoS@B6Dc8?QS|hFLr|`&FjNX=-xExr)Z-5m$}ygb)eZnh zK#Jq!g6q&X2uT99nVjKLAjf0eYeaO3?p_z{mFI~Is+z$`*+`@$UA?B5KB^cwnjV4U z;#Si}%w_tfGIT~Z%nC_LXtJ*3c1eVwYT;5~cQwHz!71(#N z-^#B-6_UV-ItkeWKYq0z5Mz`}H~y-3>05;qcan_fMa zRjA2Jl9~oRBbg!CcHNMyJ#?EMUD2j!2>LbyHrIe-IQYJOa}+ctp$%BrQg*Rczigqq zpe8KzW4_|1f0^Gk-}n*Vv2yt64bLNf7p;N94MMON2sqH{_W*pp5HL}2Gw1=xM5C(_ zn%PP8ZcFuTOZBsUcm@Lu1{n+ibctaGD;TT<=oTYlRc{1Du$rPNF~e{QiYL$HFDVp` zpp+_Nvu0@B(KG|avuX0q;WT;OplVXVRyHn(td>;O6zUMGN2~#{U8Evf*o{~tVoiuOld8JH9>n&N%IZQ3U~|tis0G7Zt9nSh*i9-MGL$lET=04n|KF;CZu`~w8=pt;; z#K1&+qJ84(#CInCcH*BzPrS0rsRCco`(=CmEpX;x{@;TBVQDyj(YO8_WUJ%Vqs-x1 z*BnrwqVQ}=Kt+++l=|GDNlKjL+KC9O-bji0*-3U<;wkQf-)bj-Hqt`o$T)d{C**Rq zNM7Quk~wglPQknaWH0PZJNX^A%F#NgyhK24yLz58Ihm>HeF76&GjL>DNh?>g^m((d zjOHm?u%07Y227a#d{42_Gtjk;a`FVG@}Ta}(%D6B-ncdM!?QovJwX0?Rg@dKHvYo4 zrcjv4fz{HpV92NPU>1vq9c`p-XS)PBoy4qVY+pk8B2(!eRsnULf?cfht2UVI;%b=c`w3J9)mLaE4Q-352X$Y*5&Sq>w~W&9Vcnq*qla$z ze-?`ViTuqYE0LPVk(T91%RfZ+e_SuD)V{vzb~Qx7WLHNEKJr}u?op{AAEUGvdfCvFU{2+uqg4lWA^-yVM4KeXIG^ke1y%)QKmmiUdK zN8#q5wY0PER(pzqjVJLJaH=ee7RbP5V+LF*YMU62qWam{q8x!*dJ_OC~fmga1oan;SGxp+8 zk1&X5=lJjor^R9Lm}j!8rht>%-5u1GjUKq|zJp^sc;EmytqJ`wxbjorZ42OFqJy7Q zvPwD$j&B-9R`uYdf>d2Gz_kWn6E6Zz|1c5&-kH>{FTzRA9-L1->Y==bREU237~X zT*ro^kK0a|)o}Rfx3_o6WpR?J3tXV%x}F5w+*^i+c4IbUbg5v*%yYdNdBpE7P1P}l zzX0Rti_qN{WY{oNUC%J^$~bSwZMYY}r!)(w#nP|Fuxbis3$BFM6ls>-7j^m)bZePj zTceHu!R#!Lh!H{3>z)ucV_+$+~Y9c>$z5>x#QZaVs{Z^*6FpaO53- z$Cg^t$8OTPS&ipcQ)HS3eIij%8;&wSgAPtViw)oa`)rShj$=W&ZQ0Tt;{cxUIkHl> z``fR5>oqVe;p)fX_T_NL>JWB_G0vfTH`g{BjPpP+Y9l|C9maVaIc!& z*B~5WC|R9ZlqH|(PG=3%KQc5xshl@m>8$Bz7uvpq)*bhYopwB?`Y8am$r*$g3+El= z76#uuwQy?5^;U2x_^=rQE=auVelK)4bnp24FWr0T;i1zOarxEPY~-S1wY3BfHG-4C z5+U39)eC@Z@>)DjJZ2PYZAy;)0*&!`rEVuo?ASQBkndgV&6Zoc=@gT zQvQe6-yQxia{K{*e5=y%LqjH#-vy+cPw^`W6^f)sWD%fWR;+v7AdDW%qhX9=JVT8I z?3Cd{ClMqNBoUxbu(KLNZJ{3lU`t;g3Q*rd&6`aNO?Si}oV|1Q;qK>9y87bH&CCV}!pSQi7M zbQ*dFhLj|!Bt$rRNruvpE9*tsy)ibR06GG6^AF}PznNdi|Jn6Lkeh=K_=8*J2E&U? zZr%mtHzqf~4pru15k{|w5k;?v5k#NJi-BGq?k2%?GKGiH^eNd)BBirKX$CPqs>+%K z0li>SNyvF-OiaI?f~$t6FkMJdsmJ_f#gO?ure`vf(RP-|IP|6G05FM+c)|Au@M);> zj}PBG{09B@wKuLk47C-!SmGn$$VRds$vmm9Ed&q?V&pD_SeVo`7Ag>{L@a_>6^TOB z53oX%1lyk00A}qnuE$9b_rYBbueJdKacsvUBoiFOyoM*H`|a8g-2x9k#$CG}6mw}^ zkcFunT;*nRnxS?xo(fE3;T=rNWU>Ojnn5iMpJXULJUB0CLQ>VU;6(KVhZIejX4f(Z zgLkvn19+BD66~=DA}qqik`nBUUF-=`l>HdL?pQDdOMz->SsgJ00rFVKxjVrLD|Y3E z2R*Z~jLL;Fwm%qDhuc~+2{g_3-yT@Z{rK6B_&5X}ZiKd;6C8t`lkWlYfBKv-lTXjV zH0ebEI}0l+AsroVTYZi(<2}YL4Zd}1>C|1P80#g``t(J>87Q!qn)lhyuxpqXDP zBSPgnWXx=bw!lo=iht?t)*r>*iT&k)cQ3z}znlM?>mPtN79a59Rw?^`W5wwuSR&N` z>?~ieLoNM9izRCy8jf?gRP$ETQqw!)kIuex_F?Q8iq9Z-pIPWZZu!jk<1^=$;qSck z`26(p`RRveQtKTho|LEWU>h{{-z5V?3cNQ?+Jv!Yz*!*w*{JV`u zO%GMHT+BUvz4k9&{Pfy4j-sGKDjXe;)#BS^TAsld)u1HJWRf`zad>OJoRc-XBqG7T zNx-|crlys228#WX1OYoqVy}}Fqrh|s0Y-c%>JxThrg$l%=mS%9V(3)_c(~{s0J9FB zI^i)_O-SHYQ*|;2;XCNNdfoGJLeVrJIWNjq7Q_x+ozh#|v18U-8!BpM%keDfZvgoD zAA*{H>hgK{wMu;cU4`52F>Z~Am#zQA=eISHgIMV*229t&5Gh*)L2O*}LF@;yjx`@d zELhlLjfeX%yBn|ZD63dJ$YISsMEll!V0`}rE&W;>YCt58v2v)z>RK(>xwrg{FtC17 zPF#WzCq;=iy>+r+*4rbOiKg0TkRE$9+O}eL|l3gf#sliGD&FKl3CxZjhT7AMfc~ N-qZJAgk@P@{|gf0pUVIM literal 0 HcmV?d00001 diff --git a/pydeepity/__pycache__/SimplePCN.cpython-314.pyc b/pydeepity/__pycache__/SimplePCN.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3408e8a01bf7b4410e00149fa456c1789401cc0f GIT binary patch literal 13851 zcmc&bYj7LKd3$&R1W6DCNst7|<5L91m)^1{*`{d8q(#g0Njl1UK}X<7!UloM-9ZxR zB%@@~37sajJZWnxO=>DLuBra0knL&2%~3+uPUee*3-m!}g*=7lAbMPv3rVs*#X?#TO%4GlkU|C|o5M7>*er z{fx@gkqd+~4VY9jeOmghs*To}2dw>e)lSQn0Y|@6b<(nRz|~)%7WBJScYmQ;*k7a; z^%twf{T|iRU!s;kABM9Hc>7D$(mGN`>IrABAe`eoGt_Dg!s9n7+0f0j2)82NHnZQFNthlA_CB6kBjQ%g!~dal8nYw zc2rJGu#zl>qY9jVhoZ_+NfINhG2x&vDauNxD~$Mp|Cns>kNC8D0J4K{3_Q7D+W!?o zB2XJ3&okq25GRzh?k;a zc*>%hl;g*PW*LuNGe_g9W`^YjO`2^qAx{XZRu~F}V}hdK216kQ=W4jB?dlp&Oo&}_ zLJ_;W0DoT;!>ZC15k)DgPIgI?M#R*^T`-``$d1#^QUW^fCYZDcZ6MT}e1Ycvz+w4DXKBbMQg|$Be?P0fO&( z8p3UCNcCiQrsi4ZBqN#68I(XySa@==kdWb>yEEhJ2uB*pP3CyeBIBpEVj-y}LU|yz zDoII{HAg4}S4$phF27i2xmt8;w&_`Qp~c^vYm5u*^$f1UYJNLh{i*sq5$|LDw;cyRC8~pn5>zi zvSt>-s%DK%j6?*v48|bd;chhRXe=S9as`xwCfN@Uxe~GrvgzQc#@Ya62pr{k$YB3# z#BFum$Ltqn{6nfK6EV-~$fF(C6DoGb&sfc*W(#23pdx)#(ehHLx zSlMA}BcTMMsB{nt2&;2O3prs2q<|wE_73kJ7CFLO+p}$iaLm5UJb)@F3(Oi530mbg z=p%2(tOGL51rUi|5>P5=o?JN;5#{Km;{c0#?Bx_9kx&eIM9?8O;$W0LauBoJg!p+n ztBlB@I3NRT!jwU%T~MQy<`617D}`tVqdxHxXAz=LWy)PK*O2m7+-|&M{y{KpCC;kf z603>ghDZ@2?`I#D9+r4(6CUP(HBoK;U*(@!r(`rKJHvWT%f0KvbG`~Jme_QrDWnwL%5p_iD!0aICC9223YGPwUi0S zb6cYs*rEW+DUI$oC4-JnB(4X>p`~Z-;j@pl5Lv`+8HqAgcT?@T{ z;>#*uKl`U=r;nuEB`fa61$X1@;k)i_t7cMOznRt%2J~cb@PkfQBapm zIxH!Qf#1mTi>KfvA1ybHr~B|gF9Wy>j{4cAyN+#};fve|Ctv+8 zWDi0_5G!OW$&5m02-TqtX_M0P`;SH;XQ4vQ+GPP^uLK+jY~m z9N3k{x_8Y#EO@uzy~96w=DlZ@_diGL7EYhlpBF_sP6s$5v5_(e?N~hkC{8(^f$ByN zc;*NWI3ga;hLn#_Jjk(pfDc(WbcR>nl;e>cBU#R6WhrkqT7b)f;F7GvPYyXZ&tR;m z9cea`Bv8MG3XHeh)94HQ7$hq4W9XMR&ghxzw`mW>MxWd>uQwusSw=4NYoq<+|AG{B zXa%6<0H0S-m(>bDL7tF6(UX#@@uIg_fo8@;tvD>i!pRuwy?VVnDMfN`MVYq@S!#vJ zxIQE89JJ{y?`c?r+>hA+WSUzAMLHG|!CV}h)R>Th1cK6@)uoE&Qe`mx;De&1`Erag zS4&aw%POQKipLX9XDZiHd z^T2d(%I#kXbT0(D7Xv$&-8-iTQ*O_SyJo>%bJtyW-%7l-X$Nr@t~ja}9MwyXn(TbF z3y#_)2m4WR^}M5cGs8g4p$y~OQ22u}3=7q6e{Z%y?W>F~LyrefM&}b&`57!9!)y?< zXE8%<_t82|FZ5ClLPmLm!`gPAKz6V{#A|yf#TQ&H@oC+*hGDwIEcsl>XolunX$}p-*K$?TNnJTi^ait zM{qOIA$$;HR&PW02P3-Qo7^a|>QdUmp($$R&=j?DcF1jo*W&GiIjY0ghbH(KpLh(UbBoO60t{ z2^l5lVk>CKko}^d=+&NAdR~)XpL%U-xu79!#TFZJ7q73jW3_{n_|i@+xrnDUU4SJw zDXU5sVyOsA#aQwXZ+W@|N@*`~mEA9el77mle;LUI1vq8*Y6BF&q|GV@!_08w@R8xM z-I?ASY&~j4Q0bjwWLdm{gU?I% zY_gywOJMnQx}Fka;uwfNg$4IW2X&aR;uSHRL>rh@6YRyDfNBk1>~zT}(G9ydd#17o zwe&#cQ7nyNHV#?7-P@v*n;9IQ;951Dcb)J`j|ANfSIH*<;;f$8J=;5%ywkSi;8Vq= z(*>JJ11Xr2MiUesMj9Ip1gAOd2!+X_|zWbath__*Cf z-}F_YeqyA|D&7Crb}obLQ5c0!oe))pRmVlZf{w5dh5{%lG0;L3tj_F|g`*TNiSi^% zt=$7`#}32*VI(FhE*ymz(;5*+1wf@>JKN35BA`kn83w$-$;B(+3}LAfBt|+C@z^9g zB904}q6yj6stBWEI~z~1f&_!EM8Qi5xC7HX-VFnRQAZTk_y{0WMwtS4t8RL`)C3Gg zY#tNi$!J_$KM&i{!Cn$#QPj^O?4_sx9c2-myrA!?F<5v9Dpu(4Ou+t>%i!{C&+Yt+ z-JRWgI=VY|uww!c!58=J?bzGh%qo&7Mv^*#xJILJ1;FZx8$xdqp%IROD;4n{x39`c zksT4hUkJiiUh;C&a4Rkur zF@CK~x5o5V97}~$o+Ni|CCN1_Tz6!wcFht4KaOTqlW@aBrMnyjusoVizX-jQA45ik z`wd+DYqODEJF#(T$S7kQR%GuOeMc ziyl&5nJ%G4FR7@$zUM~ijqt4hJ5MZ^Z%>!fdLOC>Wwcn1#R^*V6SgT`iN$mk>I&6Z z)Fnz*t%hx(g$$VPUgbe$&sP=KU7VvgC$Pk@)1yu*|AiWsJWfe zl)=XYv*FH$zr|Oy=iy^}o*zuA1X=*&bYtQxjdg)*+b31Q&WuInb>+)pXJz{`v;>%w z+gGtM9`yzGU&q(ypB&FZA6x!O@b$n$@;-;d_>=(;G{?B+{Cy7{+B}M{`D}9m9yaDd z-D-RVu%Qj0UdlHD=HRW$hGeX2orf0t_HO71uSQ>>9yp8)y!#&3k7DBwa@d#$`KLCI zLl@7`BhPQEv*9W@zkz*2Pj8+X@z*##sVNUm`Pd0IC}3yP{|xsSGBmLHdl&L4YAaf) z%&rz~*9|ETN?VQ1+v`T^=Iik`>DZWlI>&x!oYDq7L+9m&lsj9#if>$p4OLK6y`KKs zK`t0cY6kxIobi@>^5qa~RmE3w)rEu$@QqvzJZt%8eD1Vyb%tc910Hznk1{LWdZsd% ziRCNSuK^@;J*=!ae=S@C%xzgWxA_=IC&X2*5&w*sE}*`xwSXpra*#e7K`Pdfq**GJ zyivlE?Ig=Pfv*Qh6k2OY9sL^&_Lyr(lnJ&7??9vLgaiRXV0lh~-MQYcg61AZ)PS)@ z!-~*I7E~5|CfHkH`!GPKqZcf0VN7)O83y)1NVqf>>|~!6;!v+!)+bc36eoIP(XqG~ zIgBAeV41<@HPofPwp*QHWD(oPr@%a5#x?24!?fMA%l^`n# zQCY!dQxAoUg+%*;+Y0VVBH$GO!y1k2#N{}PtCYZ_0D)fAVFfM;aCA^Zqn&~Y1_1Qv zEMY;y=$A;w=%v1`&djIjZdovS(MZm`#KGFfkf2T-uyl^rZuSuEGYX4T+o3DmDRl^u z7lkk`f%fd79aK@CpunYd?W_>HEKDkN68r|NQ`J8N>l{Z=Qa8c~emxvZMqrt6@*=_^ ziY|eSuLV31*(p>(23UYK4}oiHY!Y1s*>i{lJ6%(zc9upK86=E1P#9#gil%!5;P_Q? zk{tEu++6lTj0^)b$nYk}O*^eu$W<#y25*MiAXaUU8K}XcF@qY@sxiUiGRijyoiTNG zLu^{)ie^%#bW7j3PaCysbUk{{Qa*LnCtQNaDqgRul)wDNdow%awBQDGtb7*;o5 zH~8-7-+B60-)#MC-|U__X0Bn*H~0LT#ee_#-%+3am;F`D)Na?=5pdtJ;YlHW{$$48 z!BXNn+m&l!&qksX=Rbe83m@!M8LNApH&oXG{6_)s@i2PM;97Y@do%cbkSLWd-NCwU zQp7tTr30+SxHJo{2O>$~c6u@?5uUnR!Q}Vl1MZ#mh!{C3B__cPqxv|MJ`4)X$U6967@1QIWxW%c<;6rau304+ybv;yj%>#aOe!LyeIcXK zC8OMwLFFZy1LL*uVCg-SW({MIk&N1;W~KfQ%?aeHe@MT*8l-JgB1CEG*N@btWIm|+soxm)2xiyTh7a`N^=+wo1SSCavuGxUGz*)(svCXF6TnAh7ZoT?(q=Fgf zM;RZziY2@PkuPDEbJ<_U*FVOr2{O&DgGtt$aRXvn0cCI@kWSzMau_!X@|SQBUao02 z%8nJyMJa}Y$cT1DaA{UruQ~O3kd3&&BMKx4aaywG)<@}hr7ad)@$Fdf?O62fns$B!uF&Xf(FJ6V zi*b{k0~(+x>RW&7ND6%iDWbwht_9ADACHwba0`G+bC{xG*0YnUBO58xk}2 zd-W}|FW;@-HPf5g1<_%rSrG{eIOuRWp&r z+Jisj-{pTqL#yUbow>~~9y$wwt~HIbk1Yo}=7hySH(UTT?1q&BG~k-GQr~#ng0F4K zhsaP?KYMi9*LH0%Ro|h1ZMlBmwP!x^1zvydwdYd)23SdoZJd2(PML4qb?dR){GF$6 zKl^_9{Lq>Cv!VItF3#5qX*(%vS|gUS^83RKscK7)Fr=jN_2SoxQ@+|%pe7Xnc+Ty3 z^Re0Oa~Ez8!bu)HIe+T(e9sH>RcByu2@8Wuy!R(z@U}H4DJoA7nuybb1Up|2Av+ws z)BN4)2@~mMLU6JSANNcYy-1y93o J3BJ+U{~HGToRI(k literal 0 HcmV?d00001 diff --git a/pydeepity/__pycache__/__init__.cpython-314.pyc b/pydeepity/__pycache__/__init__.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0da68a4483dcb43ec4675e44e72fee662bca765f GIT binary patch literal 1112 zcmZvb&2G~`6oto*9Xm<$=O$@d5RFQOnjj^mONx-7NF+ikQL=`S8;=?n|Ag^POS&>F zKwWjk8?a%^lQgSDmLee`_Eq}=%%rgl6|*?^d}qevv32KRCY?lV{J8phGak0KFADLQ7~0AgpsQw#23g;@rntv1SZnb1$`|rUcU5%PqMngFN@~R=gR9 z_}nWkrKy6tj^3zkEqrBEA;I!O#G!*E%S$0e6;doOM;tm>V)%PSFw4stBt2q~(NXZdEtp#zQO4?>D6EVFzo;?O~%jyj3g0u(#yeN;`J%8%;X z_3f$)H4YJrQ z{^C{pW+%}Y?N{@ZcLR-aVSR#VoQuLFD0G1m-F{}7hYm5G>(J{H8fb?Eu(gAGIzxOg z0^$W(eF!c-;483ao{}JAj2zuIdl>p`3nR$7_5j=ZhXs27}p1*Kz(An#?T^9=^eHOyMD;~#^f?42*NZ|^i#jl)(ll7f73Gg^9K~);9g@G`~E_Z;bOBe&+8qr%iH| zajxR0*^_GW%gM(RUz)1PbM;L1<*Aamog|meYiBiInWoa`yJx$;dL@-p;wOc!vV;Bs DgNXK* literal 0 HcmV?d00001 diff --git a/pydeepity/__pycache__/_backend.cpython-314.pyc b/pydeepity/__pycache__/_backend.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..94eab55a252e8a50f8f00c83a4c6938ef0d2f3cb GIT binary patch literal 443 zcmYLEyGjE=6umREd1E4W@sW(g6d{lmwNnI10;Z9WQb=Uo9nHo~W|^HJYawE3XDj}I zf27+eSlKPaLbP#K6FtS{KF&GZQ_aq$L80Bd9~2P4Yc$NDO`AzWHCM0%J=8%h*hdl< zp+ibQp-bXmB#xIH1t5tezm}hKc)zZz8Hx}%N=&GG@%ILY_QG4V` z!--TT7lN76@hKPoUMFhs`B+dxn0;F+f^w-HRAK=LqTULy;WcG zm|#Qi^C8m(_t?7L7JQ)8o<5DB|Ao30)klYz^YwWOHkx>>=aBowUkRXIY97E;j zWclMUvG5KAm7lWt5JUbDPYBKxUP^0W45!LpAyKzaZ(ixH<7v1_6v=IBk=(_K0A literal 0 HcmV?d00001 diff --git a/pydeepity/__pycache__/utils.cpython-314.pyc b/pydeepity/__pycache__/utils.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1a8a732ecb3cd961da349b5fc27326c26383b5b8 GIT binary patch literal 3932 zcmb_fU2Gf25#Hm^@pvS~ABp;*DCum~vRNx+D!Y~KR84J1Hf6`Me3s8YqRgDgqjbsf zc-=cDCDjxPnj&)ZU7`65ws67Y*av>x_dt& z#7D%^fEq#C`Syu;O-5o#*NrV<; ziGGmqe2G6ao*;D3>5{CLBq8C^sR>w`g@UT;fy;_0%K|)IlFCI{_nzWylAaWPtXb^H zg;HVsbg8IFf+kqJ@T6E2MkQWRg}lXT#Vl4R@}extvnDe~pwHtUZXxu;8-Mx_)GYuG z+;kj!7l1tC#la?zOS~JMxTX`=by84Dxj9vLtD2ywI*pD?6L~2sL7mPEIlhvG zye8-#b*@|}NPZf8`{Ed_{N2itZ^$C0;u+q&r1f)7{2Qcg>Va z5{fxlD8VAHsW<@p^;BxER1{LMqzW&l5CmQqa+;dT3xXnQm6TGksj?~TbrE$7U79x zJf?sGfzY$Ht6jLAn7Nq2K^NI0&=?nLKEyF z0+j3N3IfXl$k^ZmN3kku3YD>MAK-|imR#|2U}6_urlz4yU;Xu96BHF6M@@Jda+_ia z&gn-tmUE%r$@x(4;%L;nhrIST`as5O>t}p6h3>6q*zzJ1*yU38stn$d(QY1My-1rL zw*1e8ZHc|NruDwv^{xMvR`fq;`E6Sq#Ri5g*em`?|4T%cC_Bo|?C$CI0nR_sVcTkq zYzyaPgKWt5T?u@HUYPUk(QCTFINUojh;9N&7G%B538`n9wi(uqm8{$0Y&)_cZk-Q7--#-2(^IXeKf^FkM zY%~DacCM4{Kw~G@g~uV1?Xoe{2hRT-059mpnEN_A0c0n}1^1o^x|7|=FJH4KY#dp3 zHCrAf&Lccv+iVa1vv-LQ^Y!K2^(ZK@_CCI;EmxR;S;_ zWm3*8ld68hhv`xCtp*8)?$*##C}AIwa@ET)r_4>w+R#l9@DQS_P6>`woyqSOu#$kv ztJ%s()paa6RH%{+!@oH@f~+;#5YrSN`kD@jT6`13PiO6#E%`s*%EqihYKo*6Dh=R z6vRnhr}%t6i_c$oLtdU0^sp*uSxMj_uJDX91>#(fN)%qsmx{Vi5n!=wViv6tkh)Gc zu&0w{C69obh@nbG&>+qg5Jw5(HxAqQntWD6&IWXR%5=>U=Q{^k}ZfDLa8>FwOxj3Brr;;Gbl4GfM;f!oPj$%WJPQ21BW!q{V%@4XA}T&UB>*9X=+YbUw7FSK~D7EY~s zSLbUduGF~ewJSGjH>S6pgf~k(b2{DRGa7_Cx2WjynTJ&W78R+r4sB3Jwy5y(sg=Sy z_mFyJi;8VfaYNLe`^Xo2K0xRL}0hdZ^=B)E6Qb(mOFgM}E;+kM=x@4m^ks zto(GHyFXNm4s1ln7wHCjem&7(e-Vu>fA7xp;%L3Af9=ZV!4sQZFD;&{`-6}C`yTlB zt$h2T|AoH=+qYuzj}spy9>r1*VyQJ^BQ~_;sqfprGW7c;9|Z zJnrsYe6{XxyFGqueEG!U_(T6N{?75wq6e0UdW=~KuU!75WA)@3ad&j>^4*bI>^L&_ z4gAMl@sE2y>Rt7$b*vxQ==$LjSr5fm$j#8elE2=QSiQ6szI$b}=g1QMIMjJ3_(8DV z)?1JF)Z=|C^y-mXUuw0o-hR)u9=ONdKUcdtRXd-l#iqACAk6#|xWbVqVbDIX9S4Ev z?TfcA)T!EG+!=Sx? z$BT}A`y2t>evE}$y2RwvC7;!;reLoqj(w6dX@Ndg9I7sZVPV&o1Z zi|kDx2GD;GU>`DQ-UjOX#Zta32|s`r5f4|r>S5F&*b>AaLF7*$_E*5{I4Hug15oz_ PcROuHsc-$KVHo>29Fu`d literal 0 HcmV?d00001