From 997e54e3bf15abed990fb5affb769a579d91cd4d Mon Sep 17 00:00:00 2001 From: Philippe Tillet Date: Fri, 19 Mar 2021 14:22:50 -0400 Subject: [PATCH] [DOCS] Added non-tutorial documentation pages --- README.md | 11 +- docs/conf.py | 34 +-- docs/index.rst | 15 +- .../cuda-parallel-matmul.png | Bin 0 -> 9749 bytes docs/programming-guide/halide-iteration.png | Bin 0 -> 12603 bytes docs/programming-guide/introduction.rst | 69 ++++++ .../polyhedral-iteration.png | Bin 0 -> 60567 bytes docs/programming-guide/related-work.rst | 209 ++++++++++++++++++ docs/programming-guide/triton-c.rst | 83 +++++++ .../triton-parallel-matmul.png | Bin 0 -> 3115 bytes 10 files changed, 403 insertions(+), 18 deletions(-) create mode 100644 docs/programming-guide/cuda-parallel-matmul.png create mode 100644 docs/programming-guide/halide-iteration.png create mode 100644 docs/programming-guide/introduction.rst create mode 100644 docs/programming-guide/polyhedral-iteration.png create mode 100644 docs/programming-guide/related-work.rst create mode 100644 docs/programming-guide/triton-c.rst create mode 100644 docs/programming-guide/triton-parallel-matmul.png diff --git a/README.md b/README.md index c724f1519..1bf341821 100644 --- a/README.md +++ b/README.md @@ -6,4 +6,13 @@ This is the development repository of Triton, a language and compiler for writin The foundations of this project are described in the following MAPL2019 publication: [Triton: An Intermediate Language and Compiler for Tiled Neural Network Computations](http://www.eecs.harvard.edu/~htk/publication/2019-mapl-tillet-kung-cox.pdf). Please consider citing us if you use our work! -The [official documentation](https://triton-lang.org) contains installation instructions and tutorials. \ No newline at end of file +The [official documentation](https://triton-lang.org) contains installation instructions and tutorials. + +# Compatibility + +Supported Platforms: + * Linux + +Supported Hardware: + * NVIDIA GPUs (Compute Capability 7.0+) + * Under development: AMD GPUs, CPUs diff --git a/docs/conf.py b/docs/conf.py index 5fed1334a..37b90ffc4 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -30,19 +30,21 @@ # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. -extensions = ['sphinx.ext.autosectionlabel'] -autosectionlabel_prefix_document = True +extensions = [] + +# Math Jax +extensions += ['sphinx.ext.mathjax'] # Sphinx gallery -extensions += ['sphinx_gallery.gen_gallery'] -from sphinx_gallery.sorting import FileNameSortKey -sphinx_gallery_conf = { - 'examples_dirs': '../python/tutorials/', - 'gallery_dirs': 'getting-started/tutorials', - 'filename_pattern': '', - 'ignore_pattern': r'__init__\.py', - 'within_subsection_order': FileNameSortKey, -} +# extensions += ['sphinx_gallery.gen_gallery'] +# from sphinx_gallery.sorting import FileNameSortKey +# sphinx_gallery_conf = { +# 'examples_dirs': '../python/tutorials/', +# 'gallery_dirs': 'getting-started/tutorials', +# 'filename_pattern': '', +# 'ignore_pattern': r'__init__\.py', +# 'within_subsection_order': FileNameSortKey, +# } # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] @@ -107,6 +109,9 @@ html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". html_static_path = ['_static'] +html_css_files = [ + 'css/custom.css', +] # Custom sidebar templates, must be a dictionary that maps document names # to template names. @@ -164,8 +169,5 @@ man_pages = [(master_doc, 'triton', 'Triton Documentation', [author], 1)] # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - ( - master_doc, 'Triton', 'Triton Documentation', author, 'Triton', 'One line description of project.', - 'Miscellaneous' - ), -] + (master_doc, 'Triton', 'Triton Documentation', author, 'Triton', 'One line description of project.', 'Miscellaneous'), +] \ No newline at end of file diff --git a/docs/index.rst b/docs/index.rst index 7d678df25..b5ccc1e88 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -15,4 +15,17 @@ Getting Started :hidden: getting-started/installation - getting-started/tutorials/index \ No newline at end of file + getting-started/tutorials/index + +Going Further +-------------- + +- Check out the :doc:`programming guide ` to learn more about Triton and how it compares against other DSLs for DNNs. + +.. toctree:: + :maxdepth: 1 + :caption: Programming Guide + :hidden: + + programming-guide/introduction + programming-guide/related-work \ No newline at end of file diff --git a/docs/programming-guide/cuda-parallel-matmul.png b/docs/programming-guide/cuda-parallel-matmul.png new file mode 100644 index 0000000000000000000000000000000000000000..8050ad150af0e379d2cb0dc7da5cd988f5041564 GIT binary patch literal 9749 zcmb_?2UJtp_CDfRkP$`!X#z7y3kpb4Iw%SVNDx9Vq9DDO5Gf&wC?ZXz2_zsjhAJS0 zmLOd~66u}LLk~Sb0{o(HeCt2+{_nkczPDCR?%LPu19lpbl}T$bPMZ%=!$Z5WqEo3yPBG6bj8<`Rn;}+=!)u^nhNxXsw&!lX;%H_TD{bl zFJGFQn|peChKGmAWb)+Xl+*# zY-wrP+S;0)o*o(+YHe*@TwGjOSRj!|+uPgw`}_U<{RamJj~+d;x3`asj7&{UEh#C< z$jHdf&TemS@9gXxA0J;^Tl4q#4+{$m3JQAl>Qz!wlC!h(%*+e|fpBneK%r3O<>fv; zK3-m4PoF+5EiJ`jvAw;$(b3T#KYk<+9FIucesA8qadC0U z&CLxA3`|T+ba!`eY;1h??3ss$$HRvYBO)TYySt;JqO!8G;Ba_cT%5PJcV=d0PEJlr zO3K{aTzq^y6bcOp2uMy&e*OA&U0od#iG;ymPo6xXP$=);zsF!Oj~_qAe+4duqRHMnkha*PhpT98m zD(lgf<@7XV|477r=TxL>@Voh>93gZc`Ay^_+CQ}uQi z{Mv@^nPuGLuQzyCMHncm;X$6|n#vI6E(WPIowg0{urc@^b>0sNYc*(a|zz{gdQ0w#UR3jv{@PpFq6wZaSX_5clOSCVAB9*tHe{A z`4n8LP2jB@oIgt@Jz?FkE#V@s{iUc_t>7m!;UVECbSQ{TuK&ui@`W8WU{u%X>u zAE2Pb^nZAJe~v?a{_dLNO#WQE$|Ce(n%5FBGJsnkNW;}ksTVV4%}SD-CJNcQAPijV zW*gR5F$(TK@K#ge45y@{$3QoCmDec&vuG2!DMX*h!C^+ZksS^L!D_{GS@$ zNW)CY_Fyw9?8|um1&N#m<9-xi%CmGSQE-aUtknZyup}|G&^9w+c+&CdS{=ZAoRx_!4~J_yQ;UHixb(!;99yH$Hu31&6`bTH9Tf>Y6WKt zC-bt&;u9|G?oPm3o(YYcpF;}}kGkr|oCNhOT3Ya&7HJ5C6U&rfjYa^;5BrJYwhkWa zY?YO<2stZQ`|N?#U=xAOI;BCOOft#RyfVBNIo>l%_Lo$gxi(mjuDYY9U_`pMD@M@p zbEZTybEFVPZ5gzCC`^jGqGZBLV1ds8YSEovvCIWR3WGK+#l@Nhl~X)!Ars^eWgN`k z0o`T;N5YdZesACSIB#I15B6>H`edLz-CmE1+jMzOFjuDBg^4J?-_~`&-`kytQH;s| z1QER=H__8+o0df|9ap;xt06&zy@6r}o706$ady;Fj0-Xb6ahr699k~@f^#t+>G672 z+^dT?flu`bE}skv7_V=&m&|qMR#3J~A{p8)Ho-xtSErxcpp5S4RXR{8$SHzS&%p20e6p{m@i=ZDS8%S}#tWl+2a`?iwWBct5sROGw5 z6N!eoBWAN!$gyc<#UkHe{$HRnp6~bD^OA}zk$sz(F?fm;F?ql)=_(KmD=Av!DMq*2 zZ%9OYz#^cnXwoIVy_-wXCJxc@k?@&e&g$T;<|MMpa=VXcSqkB^d|9gXNM&{r=xwgs zvV_=fMs*i>A+xR>U87!v)RP^0Y=c9IO)i>Y4duZ1BeS=eWWvSrbv*7ZnYNr~H#(}= z203LOrUc}xE#t(`?(6b?!7cANpo?A$rXWx^ zdyS7@A8KR)`dVT%#p_y0y>sK%1-8R@jdksTDI06WpGODa_Ng&VrZ5D7n zEFXzCyX&kB#E8AM9On@(0rGgZp89y2xpKq&<*0=KJa=b$Al5_Jbs~%(DqR;1TCrwL zC8kxv#2+S+&RNH-#)coIzU0`ChII*-JmI?~aPY-@_u)l$#zFQX#SPwRlAfE2%9n8k zu-8Y7zi`oRLd@a*eFoC(zE7%1V3lekLAAL-V0c{p;F-mqbMvlZKZtsJ@-%z5pURu3 z>;C+mecqMU*Zkw=M#gXFwh_|I$rIyu`%!w7kraouJ#*|8NSfafJIBu;pZ+5%H7@Hg z!NfZ-XdaWGTJVas6tNF=a~n)BzAMoOD8BF!JLP7@uWXevSUrl-btp$ngbhBe#zDC^ z5H@!72~#~}%nJHKP05&l!e^<;+7vvqlD?leLW)A3ORv1WlBw8KI~~!auoce{j3fJ2 z4mPw&V^YM$_6mxw!%hj9dcxsDF7$)d zBgHfnPmw94GEm^sYp3a+yrc`0E~2BO@1MbhHJG0@sw$51jp8BByH>#+-~~y@c3&dj zl3sDy4V(UpWfS+>yqrB84!!jTofdm+f{lX+73o{r$}j;9pS>ogY4rm(AHT45cy55I z4-V|C0#PZ*&z-VOSb)J2A9;g6K}Y*nTX#yWa|KJa85(53+WGPg@mDk(CuRBuAwJi) zim$J&+ig0Ns>{eHlz|9BE$nbtg%}?-WtfYi=(}yrtn}hsmUCFRnb!ypXp27Szn#(y zEXMEbMRDt`iG|NRLFAoN!YYXbf1uQl2bQUB2EY{r-om+9=PZ$aZX~P$WyhJU(d@pS z$`Vi9r7lrG7Q0DHk-asxwJBl0y<|-X>2$3O6T%yuT2LwTP5D^-v7rHQ@&=CA%Vqf` z2lc%WRJ$fJSh#_?5Ew&ZSJAGnthhPi0ybANi^lY}?@E%*N(01*kj59mGRygP4f7tt z?N^s0=#$R~4Te~B529=FmYy#>bgXT!%tztguW1Vtk;5PcucJgBtBtM;mpW@C8e;Ik z7}dZsyL9TDHc|W#-p)MGrM&S@7PcTRrWp`Zh;3>mG~CnR|ijD zz`%sU&GdVSZ&truF|6&3>i zhSp90Z58on$A5iBCZRcgExNg^n~$ux1A~v8t@*SZ-_9uVHr(>n@EVv^g=h z66O3p3ze+{=15kH%Q#wrl5OJCt2Lme_PxG|TbVGNINlp@#pJJm?SF^F4Ykz7o=WUE zphB$wRtgpt5wzPxr)uq9P za2oRL^Pq;1&v+lo#`?cnJvmWCKvqD***|sH%(dATrS5F8zUx#}U*FAcoj&lfPK~St zJ3E_^^w=!!G_V({62LFMge~0`3Rybsj%V)-U`M)zOU2n&&AH2MbOWmI*JCy`a z?cg{&-0me2b4uz<+xTDfd6c3{qWV{^l=WU8o#KeN@fu7LbMQ4DPQS_ANR+D zR1227w8@_V{ZJu9~hRL(u_0R zkq1d*0soUYGJHidq+6G~B2u+Zif8Y!>Ao+X?(vBY_eZQxYm#6w1i7n1M}}|Lu}N~* zPNm_`k=9fKflj{n6SS_)RKPPLq2>&~GaH=1Z_GwD$fbwIY#6;O(O|bv`pNJa=v%g3 z>EZgCZP{}3JD-{f9f|*%z2%H56w5R^jA(p_sLMB(c5>?Tyg;@UV1d3*?}TA`8<*7^ z8sqf;x~5SdahS!(gq`!%>l7vY!y*PJvO?zALNIZ2KjaQ?|Bb%C6J!6`&YV-2fymU6 zKB=|FnXU1rgfm!wI&BCy&NB5yoPwm@XLS$`*ro6kNV}KvVPA63L4Hp11eWk|`Dkz} zjyd1~f$YD^(y0jUvwq$9&Zi+vUjjX`t9!bQmLO%kIBZEk%kXes^j19O&(w6Vq4soDr zLi$t#-FI0g&a^!kKe@XRmkAo<4$e}#c~Yoxwq}=oUB;tZ@Qe{GRb^S7KE<>Tl_(TT zNrU=JY4oE|@fcO3CAA~g6|+J80JY57+}JnzZ}0pUVf~HR(3zWpj(Jr!=Zk!7nu)XV zb{NoCN%URFfBuCv3Y437Tg|pFc5R9zD~^y_v}P!MaFu2C>BzNOu=bINz-k+Tv@<%K zcqQbLkJ@Bf6nkZk%((c?()$lpZr*b5nf+{4?W7aWp$e?! z$B=QGApqw0+sYPqGNvIVDAYVX%sNQ9L50ZtuA6Hob7bx;;a8!e86K@>`;C^az?fD4 zF%Oduys*u_=(1PZY$(uT(>ds8Yj$hCVu*Qbci-BY^@lC0o1}aPC56+X-X)9dPK=<@ zDoV|aHD@JH2rhsRM2b8QPtV6yA2(Xve}EL)5=poqET4h)mTa@)-bF+zk>HX%{29b|74lZC8D8d((Nb{ct6xmNC8|Mzwx-ABafI}K&9taY?mjE zT__LO>~NPHDb^ivnc~5DD78VHaUWm7L9N4+sRb9CvU=i=NY=XZ9Kn)d4$0KJR&O zP+6>ddA6ay6o^Q&EFk`ju##m#RDqy*2MuGA6Ft|4JB^lZkrSDljn~DCg>ewPRxckP zhb?1Zf1#EUa{zqE+`@vg<3}pX2W7Gccti(pTp@IravoOg2FSzO!Ont&DZsAwJff^4 zVYO3gf|aD|m6x;On>kR$V^NN?Pb=re+=aZUZ6AOnNwjd8`1)Qg=_r8oU%~OM%55b- zDsjF+49N%i>?!7Pv-3_UHnE&n1zqub>fwdhi$?3Qju5P6FCe^AtNb=%LP6|YImf^WU&56U4^80`^Svl}q~hsLWRw|Bsx|7H(6EX8#u+WVsn3>5K8;+(-f|dZp@1 z=fFVaXyuLeGcBlwp!Lv{mL#sL7ZA?&xG}$|jgb~OILDG&-g|d)3#V69WYODLRICfd z$#(D}jg1Wr<6I|+rNCw43f2%%qz4>v2u&lV1zcRIGK&8clq@=ud_{;CsgYEQ#%Q>X z8qelSUto&cry3EFw*9lU9kEO6FE={uV?ZLVn!J>L zVR~s)$897$+ctc$M_)1^&LSifU<>0pTMcqjNyHpr0|_fs=;r;q^9o>5ih>hE(cRB2 zs{1~n8;nUqHMt(^CZ@iU)GCq=*3{Lij1iE?=ITG}Rg45%%7MRF1(j=m15KX(Q5$_Nx8VoTLfQR0#$NOEO#Jo*iLPotGYN?&QK*em8m}4W3sZ1 zYgemG9U9x;&2@=_{Q|_m`YS<3Sdc4n;Edqj(iiA+hy@>_|2gUnlCo z2W6;@OIbrN@wRGXJDF6>sc^1|B6(RQ@q%jMyU^$7Cv*lq^EQsVcd#Ov7=LBLHw-$o zfkweAI@g<-vNRFU&DY_W&%2YAB^e05T*voE1;vrPzUOaCZLi`Gi#ogHNlMJ|Q0@rn zhA3APom9|i&A=1nDxjF#sQ9_8A#UYJ1m-tCBD>!R4x9Kz>R<_-cZgGKJ_lxs%0+W= zvsfc@kEM@i1P6xgF}+D>ot9&7gD>89mdXZ_*vSz z){T=!`=O|1*&59P`<;V0N{wOnFa@_deofGp-5)fJ;?dj*a_9(#9zJbPqV_`(!rml3 zp5+72ebkCX=+zE(_a*LwEFY^r)B_TYlW`-Br{6frLst9nEgaLWYlPKw9?XX6!c~%! z%e*pP)$=a=Sb zy;7VgxK&oJ+i+op`0@2%2vv?W{qtWc3^YB0-~Mcqf6Vq#DM5yo_OSb{?LF)S7l~vQ}vtD4@#Xo9S95Z5Z+j_oA zgS92n&Iki#8{-EwxVZWlZJN$`f6}{?gIy1t4>JEQnTD&2pD_vhAK0gPZ*21l1oRQN z&GrOOkJFaJZps{)ni5$DZi_vewkGjnn`6C*m3cN{KJjzd?0zpQzOTY8K)3N@jdNn0#F-(Z{ zHpZb%bj~`5Zx2LziI_w72OAT$D!Q@!3TBtGlJz+Y7UQxBEjDJyUv#5hGJ`G=Bh76b zwrV3on1Qe8k*uqA0E6*3dYWo?3U2W61oGBn&?%ZY3h>lE!UyL+1_aR1Y1T);F`9cC?>naH^>xut4`t1gMMhHwq9(_ z)RJHYzrw?xkoCUSo4`RUBrE8gdotkbX5X{&!h2Y90X}3vL`OIlT zsG#vxBXjH2k~mPH^-0U)!Mct$actVJ6xed*+>-T#1whO(JdkQ=4xETo*KYjwq%BPFGbOd-4Xcn~qK6MzOvRuo^ z&C`D6H_VB#lG>QVsmd9>P|cGyLRkcPC~PzG5Ym=^ctI(onWnAc>i()^Pvc>bsT%7W zk;}(lNa_tHaQ(xFBws%*0jy9}ZytoTJ`K1@`?7^j&{y5m_rZN%I|W;BJKBLx>avui z*kx%kNjXC)X$2{H1xb0a%Q6a=FMoJ4yz_?vZXOSu?EU}wfTl6qo3sIRTI#@?g=+Ug F{tsd_D#8E& literal 0 HcmV?d00001 diff --git a/docs/programming-guide/halide-iteration.png b/docs/programming-guide/halide-iteration.png new file mode 100644 index 0000000000000000000000000000000000000000..073634677367f160445f27f433ac8a0a4285947e GIT binary patch literal 12603 zcmeHuXH=Biwq+qIm{1OaiULO@N0lHH=@yzcIRhTpf;*vg-TxxA$6W&Nb)qjk2QbnUnM<5eUQ? zl-xa41mZ{|0zpBQD*Y|n8rGBq0X3vDjWcQJk{ zJwyG~;g&)MeHN~5!L^EtW)zD_@6EE}k#G<02n`E%USF5E+ctJco=6@Yt z0`F17<9NU2fJZ}I|4sTLyvVGTkv@`{nUwU0tE;PKE23IPSvizm&_dZXj`=jg`|_90 z&VCvtxUbKVoK%Ei)99#?fZ12&$B!R3I|Qu3_0~Us`J!_B_VYV;?(~*BnB+_%7?(F@ zd)nZ8S!x-io4NQz@akn7)p^3@#YZYJ^Cc#&!RxiH&5r+JAt{vP2t>Wyd1H9*;v2X% z0&z{n7arpw%RgTIzrOTu$M~;}`2TMset9L|YHMqI7a56f<3S+wi^j$t?(XbpGmv{z zR6e9mkn*aPJlscXnZ0EZ3VL!?s_sN2D%PT{BKEC$GOOzIVeJPC`H%e92uV6u90gF=S{aaUDrZ@z)p z?(S{{&o_l_@wEEmSNX&exYM5Kq~0ZLYs2?pDcPf;t$D#J_+X|zosN|J61Y`c`xj07 zrt3K-y;)y3ng%M}vifd3{Lt8VXK}Dh>C&Z3nzCeL$$V~en_t)y)h#SCxr^kmwNh@H z(yXSW>u)TN8b!X`Z;({kna0bTsTVmgVKES6nx577d__i;FQ6pHARE z*45QLdh{r6ur-*W=V!4-qV0u?7c(<6IZ)kiugShkOyo!T`}?ate3-<-dFz(e?CdP< zba*(*uY3Lc&&GUjgkCASvN-qa>3BY5&i-#qLUzWsY($dmVIm=^@Q(eha zh!sX*KiUkY^BGjiVI_$>CTAFs8`Z5uT$_S_P}N5_qom7X;jXINM4~eiUOV*e&G+z* z>M@)^AbJJ-3;lBReQbnd>G%zE#g?mkCLSl4cK4(@ajmEI-GH)uO2t<~d+N6<=g+6T zmyaY>-#Ssdws)NVAt8}0FJHbi*bt%eyW1YYi?$89d`pqAwU{IAxV<(`Sl^f$(11#f zK=8>>Zz+e%+snzlj^H=Szid~%nu2!UoMZKCQZaot{WTitu~)*vDJ<9<4|eL(r7R)qehxmq{LbLFp`l~d9s}NF~t-g^}*VWCQ(bTeB2q@rMexjHsNds`l=wC>Y zsqeG+w5mVfAP0_E_j2y>wzjm!Mj!gFf{r1h>a7WF1~Dv~k(5-;+Lpc$B~?10-^1Vi z3uEC>$&Vb_Xue>SiJ9;3Kb~_nwj=#1&MA9!eSLa=J3V zbWa?C7^{GQ%ym2wk(QFe%*x7Y+30(s|5(mTwn$kkm9@x-D`E#T#tI&ufh<`ErMTG- z>@9j&z}s?)CHnDnDke$&GUuo~F(fhX-w6zm{B2Hs8g^)macC z)|ePu#Bb@`&~hEwvx3AagLURx0rOi=9G{s`J!_^HONm9MU?9hJ5w&=kh5)W5SjWv!=`tDPJz zU~cs_>V{@ZHA{w)>D(r2(n`EeXPs_ueFxPyQa7ewo4|As>hFJ&mX0wQWk)oSl#~>b z1dg`Cc;5IHEfZ5gmn7jqa&j_lOQP%ar9Ax#)p=Y&Bufw^2Rx!CzB4}Wwd)SZmEsID7yB{V>nTG~1_$o)Edol#~p1ZT4f?|bx2GL=0%R&{^IyF-eJN?O&?u9># z&8;U}f{;!#p)8ggGvDXFr}_<)TAMrE+;(^=QK{Q3w*JcjZyn1U%HBzac!+?^ZeV6+ zE_v8+WOpyZ>+<^AnxOsIAMs+&ofD5?jY}cS3*%R){91w;=7$tfsMXBDEZ~6JFPRsP zr7Ol~Q!yY#%J5&KQ(5IBw4*42F&?P+)_8ZAsXcm>;^X5JXL43*zmV#R$U|c;xZF0N ziS6To63ZB28{MGkmgZ(oDXBWm)=?m;dDHGiEef%g=DNX+mv`nLcz)<}Ak|HKnLml~ z-!dwp#^dw?W`VGD7F3>YX}9}w&2}sSN|~-&h9bsx`Dby4;W{j|{D}0AKdPQCI@Wf6 z{#-soYz<+;?QPD_3qW1-ZddU=^F2?mY(2Q_$WaIJDPN-?X z)eEn(DaL6}RzP{C@=I;h)`>2ex3}e0BkhAQUf=$o1h8s)azkAXsu zx5Ie1J-YVd{mmS;-n^!0)`8U?Q4$Aubz5?7M7VeE;vY3ABAXHGd(DhYy)H9mU$fP_X-xhM-!aPDo=J! zX+-0h9;!{YFJjYj!(=tIwTWXP?R<90-rj$px-0gJk?1*>=1HNI@5I92A!XXliZ#SC z#6&T3?;xJihKBT|@7>GP$k9AWMfDL$2i1n!zwk0jF(!-ljU5VGxmBU2^Z=FHU;TB` zZ=)@dF#GGn3oMJ2lvhwplyWjJC1`y|JyuD7pN>d1)vsm_baFJ41|vgX0^3cdG09^ekbyPD|&f7n8}wbLW!g z3+fDk5%n#WjoKRDhx&fJVPj;=Fu_LsB`^7`2O2AC_PrwQT(HnIBIWJp`}0blF6zQ*iH$Bp zQ6cWMO3l4EEBWnUd$gdtTa}fC#bV}LGLS90_mDQ5Z>nTmlG<3yZ_C}4AhfjhaW&Mv z@9V>H*2)_-JJ04MjeQ%Sz|b@|H^6 zjShR8pqYHsPcZP?PMIdh)&OI~w4?c8nVpuISvpG=fZg0k4Z&_n>hb4u_<7jxersV! zN`?p-m(aisL4^afRQv;_a2BD{cDbu|dCKj=xc)Er0wu$wuQYzFC=WZPWEq7R+!wqq zub7w~5Nq7`R9V^!7o5Kz@Yn4-;^%X7bN2$_sd!);&jA73G40qG8ch5=l5G&rU~-Ug zDy~dvzIU?Ujk7T;D~kjvtNZ2UB8xvhOk$^po~?K65edtzsQ2&RUjRfV9vV{a%&S5I zR-XdQQ0Hri6tYRio3nA>%r0&I!e^kP6MgdJNq|dXGyv1~M9y=7<(X&0#pYeQ3}>OJ z@&83k4))168jZBFbpp6~6*UE(KYunqv~DrKuTl{0x2C#Ka?1H-qy~@@3odh6!~9 zBB|CMM|le~_v`yf)*p1|7oOS8x))e3JIChvPlr1?I`-CidEskb>6KVCg)I0Arg z_9WPJ{&uA&{l$pfP!`<{EqUyHd3izWZptaM#G;!ixX*2E-MnQ(DbRMvVd;V6cpVZ( z^*JaI-jS)4RTbO?OSyP6w)vuT-VHDIuaREf3RtJPU(_{(0l9!4*IEeoEa&ABxCO*M zl!`tp1_8MB>0IikCN>jZ8RkZZ+Gp<2-{+DNc((n5f9{1S4J&H&$)0+SWZZkb<%6*B z8~i94nO7}=m((VARa8{m7E5t+EbYJ!omv^3IiLf=_U7scka$Nt5|bU)MFD6S#JrHc zkT+mG6x&YQ`+HjBjyIuYrYi3+m8o`^1GR?}qdD(xqg3j@8Mqb^8=Ld#(zUJ$D*r+I?x)jCoK)j1omzb5B%-tvFd zu$PNs$turOK3tv}M@B}@&%I`}hUJ156k2?AdvQ=p@8gr?qrg)yc2c@0wREHiK3f>v zBfRrtLA8eL=lb3v=Bm)p@BsICT4g{19nR{r*bdE{NvXsAb=po2DOio$_OLS)CFPv1 zV+cfoAM-+T{-G54=3<1#K+*JFRQI${04?rkvgCoEPTWG-E`h+!#@2#0QHmALyK^Ez z#32Pp{$mtW=!X)JY=zV&=Gm)LPw`t)hifIl=}V>8-x74oY~u}G#*g8*pf%%c05K_; zd7;7QZj;d3%ZjyC-a}!!g!Z0ML;CcTMl>Ben*Z3bV-OBe5G|}ISYBd+$Cge232bvd z2sSpA&iCq-9XOO=4>vEMx|0*@a0D-g42nczK=YTORtSI0%(tuft?7I(lTC5_$KUEB3v?$#Q<;#DXT?Y@JTI@B>16A~&N zKp*Vy&mpM*duVuqR*n$Kgo9#2lMs5L=qDo^DoEFbB-7}jFCkJy3(Qe~gI6Ng7G#b~ z3hR(F#pSvloIh(6?&s< zvcWT{A(GuS+ao#9hUzk$5CRoa{A;!AI^WJnoG4m9>L@~4RW(J+$U`I1i&)X8qD;;c zbp0E_%U$zo@;Ejvy~AW<2EDvLO{^;O%J<@r8XuB$T=8@3@H^EG5~ClJr%S`O)Xo@r z&B4~-$+Ksjf%vWP0ajUkOL~s3Dk)JnH8thp;!;&q4EBfA_!=$gQE6K~etegpmM4B< zl$ZR#F;)oGCK~4Y{h%ujnGC6=o%mK+0fj=fwYOJ|5kME!Ew@i5Ze&PRT&G32M4Bgh zXaZATcOB&h{GRK}5%Sn`n!@A7S_K3IN?q5@hV7fqvmEGJbM_e-| z?9|UPipx=NT|aMpcWAR7EtD{ruVx34S=Fe0?{&eR5pL{=fHG4pVM)wpw6>=4N1<_3 ziSx1{Xbxhns;a6ReOg9Y?%PZG!IX%W*RTgS>CT;dc>A~B;}_b=7#}SSRcN%lqGHG% zZ}QW+gijGgWE#naFy`Ituf$lN!X^oGGiN+9T&{P4;AMj2S0itEA{c3 z=70mfl2{c+0z*>rlGF>>sgmsF{!Ox zTU~X)sP}^2w}J@g$F6T~_CYYl4>tLoEpYta0x`wWSkKAgXyM#~lg8i813KYFU^>gtl{^|D8Fzw~bdd7*64%VQf0*U^77OLByLK+8_tMpsGtgjsu&6PkL|@Nonn8V^csb z6NBi5RL!Qn66vB66AMVEpDG)YkBjpzL&?c~6T<1ILSKI$3&`^)_lz|1thj&wz94Wd zQ*(29t~%v8qr<&OIJ~^TKxA)*QsOB|ouvyont9!DX6oZSE3!-;Y3@rEeuJ5==b)ms za2c_%qkxz6lc?-smkShmPKXez{@m#4uEYYdrSiL~yOZcvxS@0a-7F+aAC&WWLTA*C zyEp^qcnuZfw|aIp8b)3RrJ#c?UE${D?t!f5Y-IEmYV?l1ucYKmwE8GaD*NO7Hi$kn zVXQs z^I??G$c=lpToJ|2-7ZTueq+-uux!**5BPIWtp|Yx1*%5wdJP?~uDVXs+2Z{o@oAjP z*CKO=DrWI{uTBoJf{h6=w@|Y97kwj328Sksg|0ex?cW+?z1(vd{AsmFAiAH>XhDvM ztCD>@j{ixh^FO(^u{h`N93W>{d#~dPKlgQ!>OJEe&YO)&vl%rR`Uj0&xYl`3j=^)RrLrKF3hA@5IV1FM?S6f__!W-3ma zRN?rOD9MHFe|@IrHLNag{BgnPS+Z$+#B6@mQV|F3T+gpBVM4BJzvL5eaDT?k31bl{ zM0vDNwZNXGY_n0>?#>?PNB2H=3M08EAyrd&qt&u=8NvIRCWl#dMpwy4G9kClim!Mp z#v?VRYGfH*?}2z-{5$QC3g})WYLjC!_8VAZw~DuOSh@JbwTyX?`5`_{`~_R)M_&qd zhvSAdO`mP=9V_jmh`0_8&~X!wLJDZY zZOytOH6o?;Jx;VUds5@v?ZZDDJ|SoyGgrcKTzDAs?%jEb?IEwoi<|Q}CAIViS@why zl49mX7#z)gO>qIr^4rA3#3#bBkeYpq1c=#s0>)AJbeLcR>k~&B~jUGM6^i z@R&7G6$h#yMG+nL#A8wEU_id0II9UpA`7RuxIVZU`7Ea zx2*uojsFKyY2XdK-EaY}@kn>n zFKD+caryCvp1Y>_?l*RI2b%n~J1J&gqsF^)5mrEc*$ll7Fbav123RMkN!HL!_r*j; zM6mfaC6iN7$#L;Fym|AclX?I1wGa+b(TcWeQja%c;F!}|Ke#AjpR~0!thabz-j$G-AmQF`!i-pAgGUFnbT&j>fM%uw zjZbhMHBWGrA%XNqyHP%r+j_?>%%Dx#mvQ@YE znJRs@>j=`_(uCMy!MX&J#@qQa2#~>7L??zk;S&5V>XZghLSPx7 z{~Gweer>Pu7*YDiUrGU(&JlbD*`5b`fYJ~$=b4!Jv-UmxE=<~slsx%)6TemNFj-zOL4B^KXP z+&WGs*(&==D1*%Z+ixjT?UC~>m+kgDTx;Amk;?@OW}R_aJsItGS%B#CSIPz#X*+!% zK+{3ht!Dz&q7$biv8i5KO9$fNUf|LCj?Q}xT0hV8*O-xD-*SE;@Cz%IEC z-8y6`HY}_!N2`F|YntEMr^Yiwq7#8Q*Fc&D_zA^%{@E}ThF!vXBFN9_sb^e5dwbP4 z7YA`VK9_~8(xAJ{(krut-o&d;@*1=l=BL6yJafJcVgR)JB!9_eC5a^q6~?Ap;L7GtZ!KYG-XCI$Hm0x8Xv5_hm-9Afyzqo7B8V6D9$Ok|Hy zP*{UPK2YmX1*B3#Svd(r7jDpBx}*-PSzTdS)`fzNXBZ@(mESo*MMd?r_LgNgi_MS1 zhLpz%vp92Kk~G6>P^k{>Y%$C;q<{K!4@?jZsr{7~`E^7$yeBEUYxmdWxhIYw5Eqgu ztCFD;0x1i2ki^DI?M4l;>d@+d&}RgOB^}$w70{zdig~5eY)VwrMGqiymGk*k+4|RU zA8qvVbc$t2{3C5ms3XB;ngM}OXOjLAgjpQWfCLI3w#5OKnil_L2~WJic;k_8^Q`y_ z=svMnY>CaFFoTHw*sW!KYViLc+{1^Wgw?#153*W zqykwCXHzz;x!ejZeZDO9)4Qvyt9uK@T~J$fjrXI?6TY{FbC-G^xS_fsg?p=1PT0CCng`P)@UfjJ&n==UZ#82pf85?e%d6AuVNz@2sDZ7|2GhV!AO z_PTf~oSKE$CLzk~r#Cc}(=pagT@u^+PMuJf;tLj^x(5ZHLLi1qtgMHrM@L zzQ!E7_5O#JzJ4SM4LA}v2(Z-;CdETz=543Rpj${IyOHYMl*3>{k4+8yeARMoRii{4 zoRKs5E19ofQ}SvT{!v>yHQHEREq>;Th%(fpV$cp|d$YnSotOEt_DAXg(7%Bb(^a($ zU(fONJ&Kx=zd>ROkyCJcIFzl2U?zZ`jIqr~OLO_U*sE^|oE=0ztU9p}3kLKINm^Z< zY{l$tSlSGfpn#UwF}g3@z{lYX-4)a=?8aGj#9${x^CS1!fYlR%Hv!Vzht9kLycTrI@ z;**>x*be)=rEgEga0ucd)(L8r&rGs;f98b@`Y%b^I29=$sG} z#2YubIYrhuKKy)OU|_(K*Zz(}o>5(`Qv@JEKvV;A-a3Bn?!rx@%~fVU8de zzmCo!!h`Ar?z}hL!t0X9-Xbol9ZC*h--ie%egx~r3N4*PL{6N{F+mt!{c|XTs4@4w zX^sOS{%(+V{M+w}s;NWK#jMD8Di$!!(N{=z=s}Jc?o_$k$g8QTnZgtitl5@gjI1v?NkooZfFbbH`O#O3mEDN~xBtfK8Y`H`v8ZT$C9pMf-zjGu<$yFM8H%Gg@|9F4D@PK&py4b7F@x}t5s zvKN44#m2;BLpup*25LcxRUa3Wy^e{;O9eAe(mf3cF^Ws52z55pNlMGycHQm1RRj6l zGaz8|{C&<+ihR1DJ>R7$C#S5d8^baSy>cI|mcGh-m(wkCDjVDviA0eye(PbaJiGWb zHL1J|(sTsJUsDy=u3alN>!1hZ?E8Xe2N&p19!y4lyjPxDQ9&WEkXVkqf4?0DGlyn{b z>0q1}%yixu*XOkpK;KFnF^0e9ET||czmeL${yaTB816ViPTmJRT0SQ1 ztwHYBxJM{e<xI5S8r%%I-l$~X#lIBd!)v_uhs*6JQV89Qr)4a zshMN?d2mrcx3kUt$DvtE6rM1~q!2p0lE0K0pmxZ2n!_)0&?q@fSt^jmQ8E7S zsckugg+Jau5y1lbG70TKtj5By%$+mzf^wpwq9oZ8F}4Z`ZJd&t3L`o^BoPr?;&td5 zbVamC(_GJ>!M1E3(l1ibd^MfNqnuM7IKu^HrVmg`8^HRsSQ%>|jUK70Vp%M|r$|@M zrpIoKy*R5Ix6rtMa6n5#gGQla7S3uYyR0-j0~%zKfxVCkkjl-))m9SSOfpK)dTdQqTZLO({b!syr`O;=9C2d3>s1fMQ=EvJQIu@g7S)ZqO> zdg&ic3eeLtAf4=OyP^eTeptct5!q@cQFh27|6W+U|2Kw_3LlTp4(ztW>7p3`mg>g#xP)i{)>PfHu_(K)BlWp{Ns4~ zKMKPCM_>8^><#LAc3|LX(0w}QE_%0$dG7BV8%+g#9FqYMnwp$U1?gi3IF)8AS+xw% z@a(|dcR;#yLZLm=uLlf=pG+~Q>#co#eK{m0YuiNr`i}q;kD$B%r<3`_, `cuDNN `_ or `TensorRT `_. + +The main premise of this project is the following: programming paradigms based on blocked algorithms [8]_ can facilitate the construction of high-performance compute kernels for neural networks. We specifically revisit traditional "Single Program, Multiple Data" (SPMD [9]_) execution models for GPUs, and propose a variant in which programs -- rather than threads -- are blocked. For example, in the case of matrix multiplication, CUDA and Triton differ as follows: + +.. table:: + :widths: 50 50 + + +-----------------------------------------------------+-----------------------------------------------------+ + | CUDA Programming Model | Triton Programming Model | + | | | + | (Scalar Program, Blocked Threads) | (Blocked Program, Scalar Threads) | + +=====================================================+=====================================================+ + | | | + |.. code-block:: C |.. code-block:: C | + | | :force: | + | | | + | #pragma parallel | #pragma parallel | + | for(int m = 0; i < M; m++) | for(int m = 0; m < M; m += MB) | + | #pragma parallel | #pragma parallel | + | for(int n = 0; j < N; n++){ | for(int n = 0; n < N; n += NB){ | + | float acc = 0; | float acc[MB, NB] = 0; | + | for(int k = 0; k < K;k ++) | for(int k = 0; k < K; k += KB) | + | acc += A[i, k]* B[k, j]; | acc += A[m:m+MB, k:k+KB] | + | | @ B[k:k+KB, n:n+NB]; | + | C[i, j] = acc; | C[m:m+MB, n:n+NB] = acc; | + | } | } | + | | | + +-----------------------------------------------------+-----------------------------------------------------+ + | |pic1| | |pic2| | + +-----------------------------------------------------+-----------------------------------------------------+ + + +.. |pic1| image:: cuda-parallel-matmul.png + +.. |pic2| image:: triton-parallel-matmul.png + +A key benefit of this approach is that it leads to block-structured iteration spaces that offer programmers more flexibility than existing DSLs when implementing sparse operations, all while allowing compilers to aggressively optimize programs for data locality and parallelism. + +-------------- +Challenges +-------------- + +The main challenge posed by our proposed paradigm is that of work scheduling, i.e., how the work done by each program instance should be partitioned for efficient execution on modern GPUs. To address this issue, the Triton compiler makes heavy use of *block-level data-flow analysis*, a technique for scheduling iteration blocks statically based on the control- and data-flow structure of the target program. The resulting system actually works surprisingly well: our compiler manages to apply a broad range of interesting optimization automatically (e.g., automatic coalescing, thread swizzling, pre-fetching, automatic vectorization, tensor core-aware instruction selection, shared memory allocation/synchronization, asynchronous copy scheduling). Of course doing all this is not trivial; one of the purposes of this guide is to give you a sense of how it works. + +-------------- +References +-------------- + +.. [1] Sutskever et al., "Sequence to Sequence Learning with Neural Networks", NIPS 2014 +.. [2] Redmon et al., "You Only Look Once: Unified, Real-Time Object Detection", CVPR 2016 +.. [3] Lee et al., "Superhuman Accuracy on the SNEMI3D Connectomics Challenge", ArXiV 2017 +.. [4] Baghdadi et al., "Tiramisu: A Polyhedral Compiler for Expressing Fast and Portable Code", CGO 2021 +.. [5] Vasilache et al., "Tensor Comprehensions: Framework-Agnostic High-Performance Machine Learning Abstractions", ArXiV 2018 +.. [6] Ragan-Kelley et al., "Halide: A Language and Compiler for Optimizing Parallelism, Locality, and Recomputation in Image Processing Pipelines", PLDI 2013 +.. [7] Chen et al., "TVM: An Automated End-to-End Optimizing Compiler for Deep Learning", OSDI 2018 +.. [8] Lam et al., "The Cache Performance and Optimizations of Blocked Algorithms", ASPLOS 1991 +.. [9] Auguin et al., "Opsila: an advanced SIMD for numerical analysis and signal processing", EUROMICRO 1983 \ No newline at end of file diff --git a/docs/programming-guide/polyhedral-iteration.png b/docs/programming-guide/polyhedral-iteration.png new file mode 100644 index 0000000000000000000000000000000000000000..02f9c2593b07d1ccc52308ef81ca22da000c59e1 GIT binary patch literal 60567 zcmc$GRajM9*zN)p5NRoq#z0DtZb=mp1d%RDrMpW)5Tq1AN|X=~NhPE^q`N~xy1UOe zasTJyT>Q6ZJ?=*~E9P8deDTINeDg&9J{}Gw4hn_Bd-y;~35CL7K%vlMurc8`QhbS8 z@Ye-9iHFMA@NmaA421u`YA>y3|J2IZ-s!om5z55U%EE}v&cN2l$kOhGmHiqwT zhk7U_uIwDYJm%tj#o-ideJ|y4PwC@FYu&ck3`JD7zJcCMML}5UbUsxCQX3j$+866J zRR^AllZoS#wq(%d7BjKX1ZtuCFzMZ9D_>)56Q~sH4t<;>*Temudu?EAU0BZeuq~~p zxwVXoYa)j(C<&Y4{6VgCTN_1$JX~loiOzpzEAap22i?@mbZ~v6k8Q?=hKBb@q*EW! zGZ0L#t%ZI3_|dqanodGWY9_V+ySi^q<*CNP-M}VEkKYz1Kx5k*8l&FoXl)qB-#1W!CrP-yJW(oEbE{Gi9G@`P8J*>hqk{pU~avctv9 zZu@9DRR44CI!Sm&YIX*wcRL+MBM3%aBDV zM*rXE|KAJ!-yhs%{NRdB_3YXueyy!?7cemy1O%==fBrn*ONB)G`1ELviHYf+mKN2! z_wTiJBrRUPd|7Nei&Ilm)A`HJWPg9Z=};iJp5cUjV`BsVjD04?S^E3~FSZ(Ujk~Rh z*e?$ftqd2`eirxb%v8N2B_pF>7f7_fn3+R);*#7IsH*Eh3U4ijug3YJlkTV40KSOp zcGY;f8yX7UCuwfZs&8Omvp41>RPZ|1V#2ON1mztWNn*d$kJsGXOu_wvu*7jCXmC)C zRkINH=KGB5LR?G?9oK~$b&ZY5kD}Qrgq?G@OP7a!+;3k@QSnxlg0CB>p(ZVT(X#Ty zae2JFv|aee&|9mD1I%QZkeVMIqQR+;qWjgq=I0Z?zxlYP!sBSMRmkIzyCLLSz}~nU zfx5bS=h61J*88p1T>hM=_sg~t@)7i{Vj}IKs?*-sfcV=q~#SKKUX}-q0#@l;v z&`9f5CXfB*?OUm{(-X)WG?ZGgH9an-WN_b;9$2><{^5ldBS}tc;|`XM!@18fLaBvR z=1gckwz~xBr84`HbSpi(*Ya(q8)PODHL%WeEs1RVWN%_b>uRTI!T#Y=j?JH%i^m&{ zwEI)RcL*scnA*IyyY5?{*Uz>@=jqqu#PPzXDH<9ZPp$u{ZTkJYW;#p^Ke!%Ukwi1) z^{~AlwU3WauE&vMN^0t#{f%jU&tv|d8d+_&XD4f37GHb1yK_zZnBxY>*tG&7xQ@4b z6q;IF>lnj?yC;v7mTLZmug|#GnMd5hT%s4vfUZ|Lw+F1!bn~sm$gg&Gc6nw4>@Njx z+_-4`Lp_$}t|7LL*NH=VpDD^)BdZ}Zry?kV@qVEi-j2)auQ?8vO~aQZ_rqwMCtqQi zNBwd?$ZxY>nK0`7Ebe@?QuL+k1q$Wq>51&rD{)^;p2P88L5@H|SlWWm$Q|?cOgDsN z{*YT4z^4&rIyr+4_!-0Jak%X|Ve+>%hD}A%><3m^%-X4ff z+6&_8J@S&3j0NoOY6(aZ`+t7W)oYDg|84oy+L{#?8@5Dw z5qF=8onGbSq+Xm0Ja$gyEG;dOx5~~Aj-3=>$UJ=bu)O8NhYte1qAq0ry_=YX1Tq@? zr5u4GNgiyc)cyM{mdQ`k-D!{>-DQ+dCcNZ$7eh#{v~o}`QEr0*GcbFg+CBL zQD3(> zLA+{rWuyx#?gEc*wRcX%A%Qyk2i3*5-SSxkNP{9$yd3=IO*LfG z`k+A={oT9y?e1P)Uahe_mm#5~VF4E{riV-t>rYRPIBlk+q@<;*CnjjMOYEz=Qyfr*j5A2O*(3j(dU5I01V$#2Vmy(mib=_MH`TY5_ae>=p6&Z0G}%xeNVpbJmhA)A^YqXy6bPO8F5qE9Bdk?-px1C5Fm9yeu3_l@u? zoacp1U^2UOwMN@%Y&L%{^ZWNY$d+~3wLde>@0;4&Uo!;}UO>G*z&xm&sVGd@f!_p^}&+at0)myK>WRZ$9|%1rG=>=p4|6HK5_W zL0W0k?7vL>-+vf1BZC+d2w@6(@c3~(6vT+c#ISeoh_QXVb8|T$bUvGOdWh^l^zvM#M+GgS5#*RVV@qy@jf5O z*6ub#LqkL2qAU460c0V)urL)_@I|P!`w+F(i@mqw1s!mVe_Vx!Gtz6^cFl2VMYXCY zQnJh4_NGGk#;c-rJr^QYij1|Kvarr`N?@jzK&;N|>_`{YuUK(El4A(%x;awnltUtMd~(8RH7-2rx|Fr^I(#@OIhoUGRZA}Bu5?NHU!?l|vYXTRY5c1_ zQEaX2-%VXUzbT%FIyy9>2Qybt-eF+`f7=s+SI0{EekEM9DkFOORSpeB5hC+lqX53^ zP(}f9#L?6_OzpiM$7l7r>%|3>+$Z6yfIbRpEyk>C@sRC=C5woOd2@2KXVChQvz?g~ z6&@QK1Py=x=!lV?zItH{3j&+*)-5d9Q2-wEonJ5@YI}w|nwzTuY0w8TynFX9R?ExS z_-1!^Hz5s8C_c5|y)2ah^{=kf(o#~N?dIB{B;F`MyXWaC3U!tq8$G*Zk=gI{Yu;aY zn{s6g2oBEw5EV5Ak$|CBUT+E@v(3_ObEbKHr@)i11ft8h2um#|w`{%U@&j2}iKE>W zPUm&K?S&pJPEOAG{;X>%-_;c77F(8bDnkKHnva!`qO2j=>KYmtczMZ|26Bqjd4+_o zJ&Ix#hd3NAvEM9g50iL@r9yk zFxQ^o5Y^Gz8aXWFbt>$6ym24*>fH-~eJS{@=}PPue4ufbZM6$^cPx*Ul0zSf?pFk? zA08k7mQAyeyTmh2Lv!V@JwYho{VfH3K=y$_K@ydoC!IxBy3o5Yp(EGV2b0GYtTu72 zBYcFY*>QQ0Q;n4j)gRE};qD$gW-KLj;c#~)nAYo)Iq@O=Y{Qdo*bk_kM&GkC-q!P5p)-H9lP&$J2vHRPK)cMo_*l{{+bl7fd z=pAzL{6t}5r~En|JI~@0GcB!Pv79%XZ$0Ox{GyccSt&#BZcO<5_rB7>Wa3#WwF3j> z@HwxTg4VaTe47n@02V+Paas&>FOL)v@Qphsx0N{jEF9ihnW&_N4SS`N6F`T(zPESZ z$mj;-SfSNKD6Qu{8f2ungoMQVergvo9U^+jt|9qP!k4OsAF1T%l*jj5c6<^^JxdgH z_@-`=nv(K>-tb^^7HYPHsVVc_yLYWOrlj9e2`Fh;PE>ewcbv@L@e+Ujoc`~vOY(J1NIw)E2M26IMYqJk0=y;eOhx%T{{&EZ=TK-h@QO(FEJlqUo=#LphFFi zu@s9hfA_gDkyFvEs{ENl#}IhK&WB@zs7dnfHUNKU5BMR8VkmT<h;X_h9Ez#t+Et;`B%3-IimxY z@UGSik6M88PfSc~xQG8NOEpJlA0SI;f!P4V_3J6^?(duZ81IMD@$itMVn?Yr_$J&j zs$3oZ#F}NlwEhKU?U5>a#mVl7;8srQYKf~CJ2SKFp_gP)f8TP!w-P6Alm31+4^+yh zqHl@nU;2XyGOvJ6Vy=-z2I%hb>DP%*u`4eNhK&n&3>t6|T)qzlG$Jl8ctFPuFQqcIJWHM-DWoGt!{f{1~4b{bjcg!i3H0sDKuL>$#4^Mo?;ke*czz8MA18W5EA1 zIZDlKR#r~V+NDnh5k1re? z9J$SFYp;IS)k;sPpH6 zB^!X>Q180Gsxse)+VxQIVHN1>X25&>iY@34J_1WU;^_4Il4bg zx3;!wfuwEJlk4iu{{FyQF50IIc=SY!oxR;y$Du=yhgL90Wo%2up!|83Zlw+mAE;IB z7)v1qUL7KP56--n^BO7L2R)$03|4xHfp$1Bxi)XU`b5)UDOC6$N#l74TSdmIMwdY1 zdAN`^2)p^-HSjJ4u3D)hTgBO_8>*k(0PiK{4gQVahWAW zRY3u?z?{W6T`ph&IFqTG(~$8r&3g7XDay-H$PI`SfamVNbK9;WNT9#$tTHXY+A;fn z;(2oTr(J06f*y6}8vOn9>SgJ)p69?f_o3)tL;-CM`04)0E?}5a(`m-#-@-M!?k>M^ zTPr8`p#z@83LEy>Vx+M6;BTCwippys^Swu(UE5xv=2~NXKZ&@>5}4f4E>T`IGJ`Bc z5SeB9jtUTQOaNGFWzL+CR_jfSVGk^ND?&2zcG2|uzCEK5b>~OXfwlxx3P}hAj29R4 zBSNCL1IZrJN?&!7MrOB-moRqx6RYsAinwxvoP`Yy2E`9J4w0kE0QLeFVAiPnC`tAHT#>5(GgiscHfkp^I93Xxr)zm0pxjheN z$=%OR_5cqA02_uj_Qs$wwC||GY0QC~#&w=P&e9{3_A4!i9%^@WEEp79RIbj|p+bvm zh=M|aiz&R_Dar4$c^&D9h(wmY6)$D5g3-Srg1n$?^EV4*p?|Ys4N#(WU^Pj_#T3B) zKJu86h0=%w$tMU3Sd=$^vA5Vo0Fk));sYUo)xFF}{_;aGaZ}>;XKOE63xNo%R%A)t z{z-)8$ON>BaC%7&k3)Mue}94F@m;u+cu^0^g@pxR8-BZF`asMi;FcNa#=-fj1$F0~_`i8(;#4<0@8)ycX4PXSmV8=GQQkBfJ> z&o&KeuJh@^92yGIRoIF~r>r&4Yn9G#b@7zpH&<@0?+oZp1LuV@zQcZnk}?SNWP{OS zn+Njpmb-4m5oQ8hfgGa5(2JgKuZ~@VeE@1x2Q@rAIyzu#N*{D>Iv{|Gii$5LDnyi- z$FIsuonK+b(6m6q^|eo6I{mS%55j)~wL;HH=>PshJBu%NfINaQ8I+<_9VgtT>MkM7 zf}LG{K?y#D#Ds%leZ7`|MH`dI1(n=2|Ff{{ZlF6Tp${HBsIn*;`)Yq1_9AFwr{EK2 zU=qR^{WIwq6p~dk<94=m$fZCB38fVa4JN-c1tx$$9ky&VygnsxW4bZ9tc>dB`-gCy zrdrsd=T+WlZKkzLK25-*^*AWPq1{e+9^oOJuKNbW=KHrOFdA<%0XmW5Eun1^=E|e6(6+vWD%|*U11H;

Rw)C0rw6%R?B)2ozjHl zVkqnR8>QO27JVN`OZ$IkcQtcbK{OyKDYmpetO%$v)p{I*+S$8$0q4K%wbk$-eEFMA z^?xqHqv1f<+}38%{VWuD@eA%kJz%3!_& z(0Za!Z?Ey;U%GUOcZ-FY89|Eql(2SX;u~v~28|(nNo$%`UydSg%SKB|S$=+o6#+E| z6o;WQV{Xb+y6#dZE1*-7Z@S@YmdSMGRO*l9KIecHpGXQGnWL_?FLh>>R-TQ=R8MW6DM1Qkw88z!2m@_d=Ch)L@z(XVU&SXEk+ zkZbQvzZu}np0w^&gc4FaQ|E6rEgS{Gai}RGH}Zpe+WU7L_A451_4vxAdQs=^Ofqgh zJ#};E)vrsWzbxR+SzP=52=7N3>=1uckttzUs^9<-!RM1iRIVCs-e3o?% z7?{Z2yWl{OesFPNDz(4#dcrFpryBJ>F|qrG3srI#_KzW4RObtB!VBVmf`r4W^}Or# zU(sR1Z)BsVBbYPgUEST?KdWStW;bjB0*8`_w7Ay*c4|mnXx=wpfIldNy~@sdglX!W zc6jAFO-Thl8&Oh8lcy-MVLa4zO8B0bBl^i&cfPna(bMO}U2$q+pY+u+kVvNi(U!;)zR1bJUkIgd8X11nf{K=sk>g=#d*!5En-a%EU}t9L!@>(4#`SJlqQO zhM!gwIt#YKp-^}=G$>JY+}tF2d3iXhGcL?5ED@hR;iEu|lL5{KxBCeS;>_F}1_~(q z4Gk?QGB`Qx(jEW+Fq64o5Q4e}afpSLH@HnS|9xJm? zp_kXF=YDSYtTU_~6Scxm4Ys7DB$DhVA(zL5P!*H@J{&2(=5|Zrl*`Y4_3foZcOSE( zZN=id44ePzuHitccIkTJ?Wf+~M!N5oI@D&tOIAf(&-aOWtu*fgWJ~N!cY6Xa{fump zWwArg&DVqsrvFR~(@y`kI19-Kd#fL#-ubM$--`{5eN$u{VgBXImu|_J@xe!LxxksM z(a*C!5deJ?^hgxSt<^R>F){HcJ_b5ENp=I*MJ%jGBFjH_SJrFN67|ofc-44*a(@aK4Sal`Z80XR@ z#gYqQH?h&dMU7;yO+Gk0w58^KFO$3xbR~K#j`vUILQg8;J75jQ=3_yb*(KkGEU+$K z<@O+Q26`U~b z{ZjT;g591fQAh+fEu24xU6wc!{y(z-C~Ma^5tzrhIm4GEpVo1!5&l# z)c0u?mrZ{BMje-n_fI#;QMs?fpVzKW-b$pSr~e{7Q3SG-OfnJR(9FKVdA9-;{XRQk z@D%^Z3cnvRm$9+oL?O!et5>fe2@-^z)*iB}s;J!bcu`wZGwAdS1E|b%4hqQVt5Q@T zploh#qSO>nSEXXxA?24i1O;Qo>(EF^Nx_rB(!;a+L@5T= z`na3UiY)^SnAzD`T!ywM)KYTK7{{OXfZ_2{7n1U-6iLm$@q$)`mguvq5*N^}0xdaU zKi^d0Ei)>l+BU&+)B6)=_>&!k`PS1i;K#8b)#}=LMZ04&pW{{EOs;$338{okSVL0C zH6l{dRHW92oNs+u6@h|?|I<5_xK)%A`3)hpwbyuobV~gB9oVFKZXy)ePKciJx0js? zqV6eYSkFdjuCJFkxIc*xB&kP8a<9f4hSMTz`u6fY8dO!161`c?1>rJBHs}bq&#_QC z53#g+xgMIQ?k{3*9zR2cSigXQ@8U=`sV5@(cMZ{`z?`|Z+O{@koH4gcGsco<~-#yOklbuM( zP^P-vLj(mZSB(`a@`94^I}n;crA1XWHs1eZDh#<5(f)4pXddvXo?n2T`eErx&o;bLx6_MlSC?88x=is>~i_<3i0XLcc1 zsrc_hUeK2Igyvl-z#&V0wC)U1BO5*01tq^2Kki|cO6hEK;tSqsF)m{|i=LaPXY@0n zLhDUOYo*Vy(Z0X>7@H;-xE?Iti0xD5@#UJ{dHz@@4ZS@Z_2BDPf=M4^U0ssLiJ^@u z-@~)V389UiZ!N*i*tS8tM(PI>C8YPBVaIS{gnd`5&dimX>30?h1570O#+b`dA?~I> z;d_mcDfJ&nzB8Un!4X*b6e0Win}&v~<9H~JgM%-If8~UaJG}1t5k}nqcLy9~zGyDc1rrMk1`04%@%BumBR1y??#_eMMe4)5Z+wNX;6e$t|VLYe0oSC zjq+|bR4OL!FyxJkh>pgp8fJ6f;eR=!fy;0`PCiW%N$U0M!{Vu_sT>rzPcr+o0`S3k z9LAJZG$gcv_rB}@B9r`_(B__zQR+{8$aF0-RI96qXz79RjK>ae<@q&&TAz!~^Rv++ z%pf0dsOso|fJz}nrA#f=YW$<-gY(TFL#!U5&zMPl10~M$YcxJ130t$!vXgDox++!5Efo`FmPU~oNv7NsT`xW8VzUpd=y6}BvS@IL~O`Ix<1{5-J@RC;xSL z+9!QLS1SJJ&p)OJ3d%Jabdz4xsJoX>v&h>62h#Cf*y1Xn&ZV;t-7;NZ5_8xy6>KH*QrXiy~D8XE&{vA4~< zeNSeUrjq%Ay*l}lPnE-B{bqOEPN(YoF|JIvE5`@Lp3oQ<~GJ3f$qIiYQ%J1=~! z-@-XcUV!NS=Z}19kuw3Pz-UOjuwU znwZqS;I08z1xo0Dc`74=`LDwcyq@U+F|@7iq8>E?BR{_y0xKUryaXB}3f0xs<=AM$ zbb8`sPJQ>n*Ka#IRHXhinJRz2How;_Q*rtLbQ)w?0EB*i7gAd`9)LuT(&MHi&938$ zk-KU5%!8AVnAis_V1JfPc_eMC_3puF7Rs8A8Ee5_5WwoL_9~qsNvcKoG$DV3Iq##T zaQZq*tjm>FABmCxLCpZ<8XI;{08$L4S0Y>>y}HVwE^gxd`E?x_8hCp_&+p0;;oybf<4;!)7C z&3(6Mi5c;I*O=G9Z8Ty#;1R+SX5d|R62h{_^U0bj^FSbb=!O%Is-O&0m?HuSA*B_6amaZoxp! zmoGE8)nY_!Y;3SEU3zKl30(vylcwyge`>6|y?HxwQo<&RR zqa3Q9W+)dge#@lP{tK_n7d?g<4VX~GYcEn`%`d`o!^QMnF8oJ)DC2W_7Q$mA|9>zBM8t^871DF*b6GT!QLjA0Qe5+c(66A$XhY2vc5$30f@zq0J(vgF>O8>ko}cT3d5WO;2Bo z&ly-saHPsiPY=hW`FRU`yFY*a;I-emeOuhK@1rOWn3;V7_w$^nSeTf?F?kEpz+4V_ zoi#eDHALSzZo|sObp?#Py^ox(AfbVB!K2#MmCSE*cA|vc4CAHKyjC)ao+syd&gE=G zC}1;nfz_Bx2BtFx2M1lF^PCn22L=XKiv;_YLk1V-=5Cd;xkb-F^v0{GK6&ya!OH$y z4oz=G-3zP8rOwVynaY>nnstH*2neEg+LGd}Fu+$a8e{Jo-Sv727Wk5fPo_By^7R zPvnH1pDfr0QcZn0AK|5n*w`7^*B|AFx@k1RYrizEjnqwHg8M4B7#QCD*&Hw5*I!n< z?Put;+6sbGFYft}h~fR=Pa=atFNvS@WopPrvllOIb=W*hlgCk&k(&8+KD(Swlu_#V zyrb?7C<*cacan?^kV0)*MVsgq>hbjJz4S#9mP|LNQBeG$9YkC?A0h3Vc9+Geh!-SH z$DxlAan)-uNgSeXeHR!w^M{M!x~7iKCvJTkC8c*`o_QJBZ{Omc%P@;F_l3V6s2AEHH`XwFL#*5FkI=TuiG)YKF zzvSi;q6lee@3@T;A3k+-K2UTs`|tyxYu=mf1-R3{n^qjP$XNwOW4)sLdwt3GW}wf$aSR z@JdquIWj66Zg!fe-`kqQc(yleLO|*FOURM#=1m!7%Qg5AthuQn2C^nTo5qDe1}T(! zZNRiR>>We&Ik(BNpIRxhkZcMh`dDdn8Ahc(u!jUTu^=}VF(9zH*}@MF(FGKPclbADeeds9jgL%o_M$SK>TI6U41VpjbHp6@X zDv6G6?rU7`{B#%*4b6Cp<4+BX_db{VNXg05>@U&e9L@}NbSNO&hDO%zu*Vl61ZyOz ztOdZRrs>dKU~li7TG*Une1wRYI0?i7sqyo5)+q2*5PtO|ryLU?|K=^Zswt56tgUkq zbm_u3PI6yLDyyFlh9^w06*I#59GGcCIKugfo*`M5)GtZ0G4adJ#%9M9p_uNC z)XvUrlmIwbkk|B`Pef+B?B7#YfGN>bP@yEY(988-^j!y8GJ1i%>CWR zmznkM?N$DNCC_Sb=Fw1~{ed0!teBzWsiNX57#c+N=e>#*eEoWaSo+wcpH)kv2p^@l zyCL5E_NrQ$ZEBC?1DJk+&X1&FOc=Spmr>a=|{&5+-;>!qYScD0^tYIuFiH34(RK5)xj( zD+teDK|z*Glo&!AHtKN%pCKmpk(|BGM^)Dd_IZ=z{IrO$PrY7MXq;8yCHg5z0#4+^ zna~7vBhj|KUj9HQj{BQUc><4*GNrGT@ z3#GTc7x1_F)<)Hj4hg~w(x#g`J0JSL$kwlIQB_aB*|NT7FW2TlA9m~apV(VN0OkP+ ze6hbu!-iOH#KN(3*f0jDRDv7!cWrHrbE#8V@f922sP{WsuL~%O*Gx4}=JYP||cMtn4E_L0}C0?Bd3M-Zn& zaCC?dBo-v4#S^%D<$1m8mg`Nl9*}?@1o^~GA3ya(fW=gyc^&D{iYMv%MHI3{=^huB7rnU z8bG;dC~6_cnSg#Y#56vaYQT3bN9+Qa&=orcPE80VE{tUYO%xVPz=xMXZVC7W1hA+) zGO}dn#iPQmtq<-WmXM6t{u}qv_3R|5@A`d3MIt0$Q2^ShK6aqN$Rp5Rm;{{f&3F$V zp0dbc&Ot!}T=Z&)Ov=}<=%|12Jd&;H(RntuEZEwvF96wAzSt7f`9y1HwUk2aA03a# zJntTl7V>|9^|d(CwAuN@UMCN|Xw;)9-yZ~?|2_ec3>He3^yD64pd!r*<~L7|D4dQ~ z5WFFk-GIoc?1~W|{1vcp!zCWXs03#t{3B$~3p?4fs3>DRdUbV_cXuH6wt5jhV`G@=`9~0+f5gZ9N73PTtv{1U z{AOV6ayoI<^4e(neGT;5(q>p-|$4=XS}(i{i;$*2=C}mwf0fe5u-o zIRU+9C3E$c79m`4QH3*vAxEgsq1BI8}YXn2NeE&en+f z8KJyu{4TsbTISkb9y!-{ni9_Asn#}>raqL2(R=8gXQV5-FES=29muDPK_o= zX{Q&s9|{QSc*gMU+00!1KoUZx)A8F^I);O7^jwmOf zXrk+_V0M&<4%bW}{lwV&$+7^vgpW2(hNt}Vo2-G1>opBNPmlMt38Kd=(AD~2-)N?5=I zg+En-pYR%MUkHL7iCCi8xVXy2lpWWDKzi5DV*(kDSeghD*1Ld37ew>_R36QL-1cAA z)5+dddn*&EU2^>~xpu`{XvF{dlhwcDTg*8Ik3WG2%eE;B(h-vO-XPz@Gp-$^{MoNF)kSkuw!`vpQUWPBPjDbM!c%GEcX9b$JL$7w92PI#v?ZG&q1 zQgE?kiRAn1cs7SU%jG!XtLY^dujf1>AN}EU=iR&ah|L}-eAO3Q@pAWzg88kY%7;Aw zAD|NAs2FB)^N>dGtx6BLY{mA;-;!YCzE|pfgavs^qjB{xLl?B)}pCp}+J zTyD$^n{8Bm5`obm#NlNyeGGcCMmRgmW!(w8Fm7rgPvMfX4VmD|9Pm&e&i7{%oWk?G zyvkel_m7fmSBjEHYW&+gE1@h}cy_S=FyRVxPX9sc;b*m4PA<_$_tM4VP!A@gkoB0c zhh58hV5CFisa;NKqVoNox`sx1i=U1IX^H4QonS}o#j~adku4rOzG;qDE zOM6}6qj&b!9HY(liz>-8PaisP*3eO(yIvp_5p3T6Qv8nZx?xr(ZRivd%Q)EB;r(B7 z?hqu0g4FcKq&VhVSZ%xLC+ExH6)f0M5fXm3<{1xmd-9HrpPvG3FWKlnU67caUmssf z`)R~UVQsTz9Pc?tJwqj^ymt51#>V_+6-KdfJJp2(pMbX)#jS@Aw^Vj4d@J4e=_}$y zG;!z;e6bA1+Y`FQW6dwZQD3l77w!scym*n`eZ#qIk*C>epd!g%-m z-RL{Ia8k)}1QdKYiC|`30R}BR5(ZeZwek|O)9=Bl#pH>*OFOQhT=Vf)-etKQ4PzU# z3Zk#*@8qna`*`n1Fm^edx=V%7gw(+Bf+lN|m7ZqK*FICAJKdAVFwj6kPCg)%^RwTi z+MJi~Q0L&_0HA@DeT*sSeozST+CkHMT>*0vZs+&ZR|aUmv;e~!e8r>$y@Ae(RKx^? zLd45IVR-kCY}QITJ=lODG-`;9`To7Z*0BH1(#8>22kKd(@A4}Q&e@Z_FG@V)rEco# zS8;G~4Esolh`Q#ZDS#j_-M$^srvmylqQJ%{f@lsWD`2@A8~I#`qd$DO;WnBG!|pP3 zaEzu_iFBKCdTA*bd^OBDO;)m$w6vCQ&EQ=_A;fFvgCl964)&>7TU#er|4d3kciCA9 zTSQpm3qm-SBCeu4vf5DVuh34F|G0r-K9JKGYcRH=MZ5ue_-Bt7wACn5-&0dcaRc zOS{JDlnv?{3_jg6>V z=|15xFX$y&Bj^N^OJ8jVN;AF%q`dyv_7`*Hn@7H}m4|?xki%}_jk%e7?OBBF-8m4dcORxLoI(JxR zhL5hHZx*q+9aVib*K8LKby+Ozi(IK5FHD8i@z)Y)79@7D$`uj81#w=2Vr(9{pS+E$O$=(J=+9 zvgcPMa2nrMC#{Tt(faSiqOcq^Rt^ynYRL%?@;?Jw23#Fl>Tf3ca~eZmVdQ05=^h_e zV{tUQTSseD(th%yX{2>q?jXGD$Vz58_IrFC=G2K0-%391y=(mXJE6n)cEh`0ii_18 ztqk!Lv9OYtd@g=;{EKTjUPRD*<>tL~2P^aOJcIqeV6Su1qlpP`Vh4F0WvQ<%;7ch+ z7K;B8VT|Y_Xo>pM3zZ|igxYLbt@UKz0H6qh=qv~15BJ8Bq;Z~QzAa0>zi~2EKiHo} z&<`eGpDB-MuW4Snmlu`mu?2yYEfbFEJQSf&)b4A14w7A-8f!Q2 z{CBMSjD@nz=MJ`V4CDI29uhc&^Lg6x zhmNG_uiaH);cuJf$NNsZa#76{5ro7w41|u@ z`L~(V)%Hk-zgdv(C0S{jtt^c!W_o1x@1|uHnidp){#-oe!?0S9gTr8E_*g$V0D4Vjnk>BM4;O5VfO zNt|#{4Y;1N>T|hx{S}s>>XORN(e4IR;)I*@2}5U7!R{PZG;LV;+jfRvET3K{B4TKz zU}U`Nbh4vWXm@MQbDw^QT*p~!EucKbFUM<7NT>DpRIsL1ACpqogy>0M&fr#n80z$T z%4t%t+U|Cbw&gHyWbw6YE_YH?m0waeJ=@It+~%{*0wLBO$jFo$*5klqiQ4F<7eu-h0BtHrLMrY%-b)D214i=e143(E)`!-3AsW4N;;>!;gLbZq@}ib z(s3J~O@A*Tr}?0F9jhPVADP$2mpJ_>vQ*?X3g14eJc-&8J@AVUz9TbGtKB92;%M@3 zT;KJ&v;l*&uCV=s%G1;#>+eZL<5PVZPt}F2Xs}dr8s;3$h#T|pzw3kyEGX+H+PCox z%7iX&S~2u=H_Y->g3Y$|ZHPMxMA??%fbAL9@z}q-$+ID#L zEGo#J&i-aOI1oE~(`?wy&(Abb9I+X4?Tws*$>`1i*XrTY8@We4te?6G#Fp0u_kvi^ z4@>tYuxy^wdv$@$C;u>*ESuvKv1-(6b&kJyW%5d9Sd`bjgN~ zM{&R5+Tzm3W%KU8ZI?~c+l-9Ne;rv?9MsY_{hA@ZIcj_PrQICnjtF|*WQNRxau*#hn;{M@+h>?mkZ|W4Zwm1`F^hf}IApG}WKHMvl}|>-wwq_bdSgJ> zuIq2x+k<9>(4`aW{@(SK#M8RPQLS;YGk4kk9D*nR;|W@l zNTDaihjRfi)S(gZMW7xqTxzsZmtbP88Fvzx?0mD;l>akmM$UI#~8M_~raH(Ol)O_<4)*9o~Q3CTK~W2c*&3htt{TkZAOqt+dD zWw~Q$xV{!=^%c(MdWJ1c#_|Y9N7hw-SJ6onc)#Lx5FG0_kBhk(b!Tq5XhMb-Ju?D! zQxA5NFsY*q|9-##2G1Yobj30qoytjc=7H>sRR(vTANS>`?WT!kDBQ=okr4XU|FZh_ zV&-vfdfA=yTM1HiYOfWlJJHvD$wtd_5y!In};*X!X z;nF!3vfF_~7&N|u>J{%@v!EcVd38yb2^!(>2sz@K>p?#-hJGNit$t8>HulFsPCis7 z!tYjMOMRwE;^W6lH{EG|Ty3FtvTpO(^M6L%-C$6(HCB0A-+y+Z8yggad3>T!va1nA z#a~sxPCihgrB&g^#|32hk$ah|Tw5r0nSc{}ABX_X?r*O5v9+Ig-)VbENpg>9{|`@J z0Tt!Gy-g^JbV!FZ7D#tUiXy3WgEUBYC=MYaAqdi?0un=a3eqXvBGN70-yYBXe`npb z&N_E7@4WMi9nXIDvoE)|gwzcEr?&9Oa42!e54>q^mzcYH&Dx zJwfu@-u6#3%acl%Q#Ytgp3}vF9i&DX5HUB09;)&wXXUJhClU*!D@54D@Kd{R$bW7h zFQHZ5Bm2EspGbeXC_p7oCMq5goFOo4HadIR8xNX3ND_PtsSFX>M;3jTbyK@Q*JU_emh!MdW`SX+M|5WX$M!0_;ZcqMRRLZ{Y% z6YI_EBe!Nzti;gEv#h~Xd1h9pPKnF$ms`2)H(>Lxo_QA|b|*aZpiFpD`sXC8|v?W)wO^T>HE=3!AhTMX0RjUMuxWmG7jFa zu|e$hz1#Wk`zwpzJ#4bCP@fgnGkGA8QTEKxFrsS#zC~s^6}rx&A(z}e39zc ziSqp0?*+E2rrXYEjEW*_%&SfaUI}WPCIZfEkPR@qUbH5%^5zu~5Na#c)zQjM;x|xb> zws|vtPa%X*YA=A95$*{F>iJ7OnZQ0kONxpzJ3H>v^J&JTk{=PcY3S#vM+~m3L%%wv zcA-lF^lTbU|Bh|cmKnW15x&DyBL6nzygwZu$uv0l=4JxQm7!J!(W?sgW<YN3Lty6w3{BCPE(gLy4KO%eFb`@cJVNV)wDa_ z@mTz})6dTtH#hxT(&sc0NRF?C3#7(;>5_3#qb=%hxMVoH7dr$_994*y7u(mup?elP z;yn$27#u)K=V@1?A~dz&x{on2GCn@FBgVMac85gjHC0NC4uHYbkx^ZstO8R#-H5@30Q zrNEmwRS=Al0rz>L?d&O(Yu?+UtA-(|?ju@a==FB|9hntPyr71emvGgSSWDUL>mf$f5^`EB?gA4(YAn>k)(#AO1Z}!BjO$0y`(~O4@CP*Ph5)sSB$t zcordC2r<-Y&YXFf5j*5Kgp>D9z`l$ihbHAFfn*OHr? zoB8F-RTK?Do4b(w+qZAq9-=@^j$azDDbzupP)M1hm8y_DZ+L$drG*0n`TW(ufTW1s%5p8b;Gn@+U={};T9)bXg_}EA z`~S1&Th5%NrKLs{lhT{sq7YOHjCZI;@ApL z=4%gUE!ICP{BH$0D0P|OFH}lw$dRQ_q*i%>p!gFk=%*U1z9f5Ca=K@Z_Uz#ov?L zS!pGc=-FM}e=^_!dq+F}dw}=~#6%$m*;B<|q>0red@50wia~||6=X{7sr=A1+-VKC zx}1=Rp552^`zIcCJEfS;d^>7?o20!^XW&|=fZb{&V(2|h$IVn9MwE5GJ|O+?fW3Mw zejX#BEVh5j%q$ScLd_rfF*?%`QgcWSnEoA(U)yY{+r8Qr={D1deET7G1u4{%#UoM2 zui6o~HJ*p-CBw|0-G?Z+fF+{(*eqYmhsPl_7k=uaL$W@KPMM|;zh=E(a-9V8G! zGurLH*L{YIZ?*Bdf9=|$(Bqw8`r^$b=l&`^w4reNzF~Na*c9ZkLeDT zK$??35igQ9-uo_ybjYff;+v}2)LET_5PhZzd^c_j{_ZwqoKod^3mt1>5(@|Eq^+$(7 z^siB_JN%m}z>USBW*X>cImp~^-vXYO!lkUld|CG+2=e2l9W)glV+@JYZD}<%b-r*# z@M_lyev-2w2;0Uk7ZYitBWpzqj+dGKTdi9k>QxyKmKIQ=+#Ohm%T;(Z{aHZ|xM$4F ztk(Z-E0IA!YzQ+es}rc(eM7P-ej?1v2qZL-B+b)q&UzA;vFnANoh+WbCVuql=*Z(e z6AA3lcjbQKDEKcB=qoU~$%Y`rv_96W^La6D+m$`AcFjN7`Fep3h9|% zi_{G`42w%oh`}RC?eWAjF=A+_t4ph^lc0gPOR>)jPJnUe+-jg+h)d0{xaNRGE31mq z5<~-0HE4L-uj;0~tFfJGgM1zxc@nL>_;r5`#06HI$J5*A9DtXNedJx-yzhLF1uiOx*pIJk$lwub zuj#YnslCk_Xcs%IT31=vJsuL?EAI^?5}V)g7!tO%!3|BZs!469qwr;0GTJI_AcIys zuqD;-oj`MP2DQL%NMz;j8z{TSu$IV{VzdkGD>l>9Rhu#Dl`F~a--Z8OU8%R-a4m6# zbT3Xc>#gGuL!4TTJ;pO6s3k>Ep*}6$sR>z7(L(uT8dz4*d0czXL;Zhd4Ru;5bj*sY z2^l{RkBk(%u4#azhP=K$?epi)fvOJlVCctwu{x9kIwag_MT>?PE6TPF^Y|*Hq-i_r zB~=Uiqfl{Jz9h&P&gk@|-y8fhg?6DO zhpt>?kR5Rb`wdWJTl}bLkPb$8&=mTpjzLpYtICN*NJvP}ZTtmj!M%9Jgr{#`k+{D$ zx(ixFxw&SvHlS5?1`Ue_~OguA|e5dS}o@5>`}9UE4KAOGr!{hXy*341flnAdu&c$mZP-(;owCA_B=pgR-+!QUS9<~fJ{sj0k+Gms1sJ>MHdiJpNzWzzck zI=+9!PKx)K=i&4$8-v6VdPVKu$78Gvu=yy1WeRF)ZRfovMZm&(YHG^F5%=W$7(@&_ zwwtbXbal-yCcb3l=Xbj$jV}c2FM2R$N&{NLm9tP`12F+`m}+rK&q)J8*W)j2xI-(Y zg~RZ3O=DjC4c&EldfyhH?MLX^sb&r4bjbbf@KIQUcmdeH<_=57q!l|H?58 z$OJ^Go&$Q9p`VcLp&uwR{{q>nuj%RN+X5$F7305ySO%qt+dVKst3^)%s+g%u1kJcY z=OraH5GU#YJ;)!NemgjzLc)8Dt8==X8G#Zafk@<)f6g+97w}m9WR`nRHO$bIT~J^S z;%lI9^A=Sll{E%kMR;-NNxKTX4z^SH@k=S*;ULi^c@Cs~)0tXN;8bv-bgH~ANuSHA zLMhX@Eu7YNsxqp&S{R}h5bn7LTAIH>F^cj-S2C>Y4(MEzfSA#~1&Ba_!00<@3xIIQ zV|Sy%Q79QM_J`xHxDwMg{Q9HcIPe5lS69*PMof@Zj&bf4l5bXKlo>^#Aqd3RudJI5 zmwug{ooK)pHtP7m0zy&ggY!M+JRoliFIMESV}jC@1BohKcoItW3##`+NbTmC)2+kH z*GlqTU*ep0aX1ln&0n*#L$1;LsQ6-tf#MVhEJ;DlIjc<`tU+q3GEhIEz>V5uP^|>D zlunRQ?gQ=FC)U=hl73fud_j){b(mKj8nsKz9p?hcQyebj)zpYVOY;}F-4~#LTD$l2 z2`@i?F{q_o>}5&p)XbjSx8p^5!bbs2HhNG9Z`&(4AEp%TpRCjd%EFW&P-Bte$qIUX zTqq@Qx`<#{PTW_Z^<^wMKbkPPv6voAXRN?bbmAOIS3TEl6(#)?dZ z*78a`Ks9cx(vb=F?&Doe6aGZ)N{2Z~e_RF(#$0o32Hq_88w3((g$W4M(i>i$jIt?8 zg@63?slE6`mmuht4gWRJE?0aiMVcJG*u{af#{4-w{k6pTin28^H>hBO(9Q@6D??jn zZ|`DU!x;-OYp$NXos%XIygFA%~&HyLxLlLw>2wY6G6DgQBo`tns-6?&k=scH|Yv-pjCk|(T3l9 zp&^vaoi*_XFxH@1*9)>;m^zV6V1?)}wu8?==2>H@OAG{fXuUKL*I7`x)|anexj_+F z`(q|SaY+d!k9l__?O!KGP|~XM%2a&%bOAi;Git`yKUZ8ph3FG}E0@oqy`1ehKwX4w zH#Hr0J3ksHBGB@*OM(&-sL)V%FT!sxldew{v~JE|Qx`r*D`A{Pn}M^S-qVtf%3uez zE2-pN$EK^~g+@&Qba5o~^nIXN0~K@(G!P{-0hPQaTs|{CUS5<~cH=BbvF$kbzTWCv z(BgGE-yhAL9p$qbQ2`$~q$NEJQb$K;K;FG{DA%>G3jFiu4_zD&>;lMz-Z%>G6oVt^ zQ8$y;Dm(_)v+U_$qA*$0zg$Kg+yFcW?V`2IbyR=Um`+;xi|vHWX2mSVe>I~@mhR|C zNu&IG|HlOY-M4L!;YPb)umf!vS4m}M5z_}0N8^2dUpI4$LD!3~3EP5N!2S=2fT5u- zo8>@;K^!z`b?|F3Unaop5Z_|VIam7?|+E*ZqyHCX+!;W&A={cYokaL%$C!Qxu`BeMt z^kDL~?{93#yhyuK-uoS_7j8A3{)tu|%1Q<+WuMCSTV7Vy@nGBn^OuY8Fk9)|dS-+; zh9q*-c|ewYzp55q!}Sw1eBB1ws}<+?^lpfwoP~e?{tcIq=R|Muc2-h`c+LgLVhZQQ zkcWed74q1=g+{`vxU9}Nx)^}Np+f)NcC0YHZvmUg7L_*e0%l;Cdnx9Bab^&=d(7#t zFg~b^`D<`Z2;9xHFp4J7?^Leze^_amE1f&s2?A9CYKtJHGmO<8J7R1<_bomBA#JpT zdH1I@yG@VPoUCt`w41J#V8~s?2VgU3VxF&r$uWT*>ea^ioozcCWsAxiBV~nkAcyE$ zpbvys@Rbj5Ylay8w_fFJKY;v-uAH|isd@y=*xQCh7~58(Bmg z*4EajhR`$V(177n(;_VjP`N?_eZcE$3Txm>SNn57R0%783XUW(EiGOn#4G_C_UzWP z^WZr?r+u8P&vQUFmwP&ym&AkK{MG!agQZeM9rASZM;x;vYN`~QyQ;fQ*Vdt?uM`35 z#`@v|s-!Hxs7nSc*PVQ-N7ptAi?lQqS=Hl6RK(~agvX0T%uA}e_gq6t@W-w=;AKyEo<_ogHfOegD#r*i&c zJDPK_r!2Q)8ed|o@>M%cJu)DXr)ur!_xjZ_WGw zF4IDte0?(V<*MQ7W3+73g;O03_a%+z&p)0BMXRZK83gM;l5j$N8!2`dJJbvI)$?+) z$NN+EmF-2DzRlxuJN32xYHtA+wlUc{0Vf?2*)JIZ`h8-f8#|bu`PhrTAQFOAhVN_W zE89_ShyHOA@`p$FZ#OONs$LGruM02kA_jfCORPvb@bH+KnN==&MW*Y6k^84hqF$w4 zTXkQi`F^+SzH}@sE*_i@@D#UnrC;0h>HqegG|vr>u=G^x_n(f>PgABAxF95=nehxypTL1rWl6x+!f~#+wXi!a4$Dliv(bM-&tHhmaA+dGWfGV%5y{ zu?m6!XYGH<)X(MFi`YRhsU#N$s-zH)gIt#}=^P*OW{7=h$cifNv88kJs9NC!;hv?dRoIf#NS|D>oe<)4=zV%?>5>%BMM^yma*!BX4MOOvk=CEJMa%f?vjF0_%2A6L-|by@n1PaL8caW#4@wY-!-d%7W2Co?KF#n){VJ9V_{nx?_qpB&g$^>Z4FQ$9^a&rBn+~B?@2~(esBHZv;^u z26ioE3=reIy3Z9=IU+{;Jt{vtq!?tH8y$Ztw0!-F^<`>x)p|BHVG=8A^RNM9U5l8cqxM$-30;NjhR(O*Vl)cZA2-9+zoH92tJLE_ z=}274eJ;+K+wrO{a(LMe6b8iC3I3_S9-_9h_2-DT+XkF#{(DTTv)%+~$bm<@Pb;`G zqnTR(*Gk-$m6L;<3gd$9yIo=mA3r~qs6xHSM%QChqkDY%9e@fp(2)fN_}QBG2shj@ z2@ZvAXG3rniHkPY9!yb(ZuCVN4X0x_eZ){EGfm~8#UoQGcA)SVMs-l_oB#c)rikV+j1 zWXzrREjv{SLP(n0wd+;4+7PIeg>xz#78dp_HHA*h>ly9epX6|_o|PADn?e0|+}4+a zCx(TMEw_1g!m#UBGZa-Zq^*^rS&Wr&{wm z(F7`4b+zn<$q=9fri%_Wsl1wH#stvgM~fH_J0-vD8N2&(t&}FN>j(JRVFPHLPD)$f zrdfq5IE2Qkh7c)J*G$jk7E@7E-)3i5F^r%UDev3C!-A~9tSmg!y=HoSs;XpE`Eyri zC!{jyXYaeF9;A%+8$5m-T0N==**#p@I;zEySVU=$Lg~0IurIIiji?M?Bs@1TC=h?P zavNqJktmC%kIYe8BoPe--h(wq;gpsfLPGH%=$<~fqCy&Y6Hub2YS&ulrmJI@=Pv;V zIbP6LtY2MJ>aw)8HG`ONwKfPSpYs}H*cL$Bw3eQj-zkJV5mJwdOl}bA0^P^N6J`+s zfr@6WxdGDU9~VbhGBDHGZ$RetFFlZWn8a)KGxAZ;BuH(mZk*?Q z`(^@Z8alH4+J&<_1gik?U|tNLu@as&Te}MI<(O;my16?_|)TJe@UzE=W^~i zi4aUyf;OMqckimd909@tpdnRV3&hO>kfMO4PyhDqMnqdP?3n_fqHJtgUCEA#2??O0 z94SnLV>DK1EWN1U2NZ(7nA_^8*+GSq8vuHsR34+FqM9I6QvRgE9x7#LcOQOmMox~x z`7u0Mxl+cbGBhN_2=F2wh@?G4>8x#RP$&sdWKH0yYSYAT0)MeZ&w2<}5&NEA54AXkSPsu7G?Gpz(SJvi_VR3~6C0A=KYwDbUI z2t%d=29*{H3t%Bog4<@?QIV07W@eu!oVC@}n?c9*JDeKfA=F49?Rp>2G3CGHZxW$s z8WK4Nn+Xbzxe|=5ex2Rn43aCVZpf-~uJq*SO1ia8bAJs0uh7!B0 z;4Mj@WJ6ZMV3F)P66te`35y(6Jb_pxvVm*wV~3fPDrruel1DN8}WphkFTuDm|j{H@Bn~=fhMg2W)yIZtS3y=@8y8k12`~s zvGTS^=HLpPO;8Yc1Mnr>@|^tq4VOGW(8z&T1@&t}YUDi=EGg;d6FypUU0R7&LY$6q0a^|UlM5T1a_5% zF2+4CPDVx$cfJ02U$_>89iy0KaCQD=ck@M)o^zj=nNOP9C@PMiii`@7W&ER2xhXc3 z+3)V|?vyGOyU58B?Khf+>G~A?SCcfkg#tq<9H4*>mCBd&XyMuHwgh5L3Kl8Ie>Eip z1v}~suwp<`Xd;E>t;tAsFMeL&qI^!e-@{888u_XKC11ZvC9eg zSdm3Xva?)7=gu*ypx@nV1DJN+#_h#*b!@I<0vm^>QNpA;R+G945b=3t6QRyM9LHh( zK$ywJLb(zM;GiDIXryjKZB<49xp}vw)ihTf3xDh%`P3JGk_Nq>DNSyUa6}aGMMb}7 z@a~-7$*#%Q&DVf^ZY9DKw>uR?1NgkXtdJb=W5=3lG!Q_wq5%d;Y`0Ld5~^|qIRF4| zD2bxzq@+AMN$D8WkM|qaGuTv?di>-ZX=nwC^2WuLIGwl% zI$c5YmcpP^&SSKhu>EhN5F;qqaPQq$CV3AiYO>$O)QOH{(~RcZH~ZgCvZIQ3L@s!; zvUv$YB>K;9_9q}j_qnyBcmJzo5vEI_I9@7zq+}=2k*n5FIF4@D-np|v^7X%J6@%1U z)N7{YNO(jJA0ID=(_L*$iIF)79aW&0(Ee|ah^&J~dQ_qhdZzBDbgkzmM)4 zMs5-H!V|&g=mc|R16~D{NL7#x8DYTnvQtPxab2MTX36^@K0XI^xlt=imwR@g>rec1 z=!)d|_FS0%94#?VaTy+}a76efXm1Id!Rxc_uDL-XPnJIM2ici#hyN9nWg`BSBgUeh zCTe+4SS=?)QJ9`9;BD?Ko49Y~|88j^3N#^#{J>dnv|*K%t{xp}%Ua--T3HGOaMVq2 zG5UALbXlkc{vLb}T-Mb=%kJuV@$(5iVVtny|8AfbiqEpF1_1rvUP_B>g-09N1B$*@ zVnrQuB^aMjClxCtFM$PP(XD33s}2p^t@dRdIUINKR08Ow`e8C4oMe9!*xp;o&iGz& z{#8@eT&D2{ziV1r*lft6+va6=Zr@rmM6!nl^ethYtu1#QRxUxoKQsfy?-;Jf$HRI9 zMMSm7YnNpsnadD%I|wJE8J9JT8o_Y7fAvh?wasxz37vMA0|Dd%EB+u}5s*YnTBSJr z+Cw^eJ%=VPg?8bL`DosEKn#Fj+0i>JKhdM(Pxa1 zF)5P!-!=apzXzOZgx%?A!x%uAP^~I4=UJ+%VENy7fJX>nu~bLEW-GMoE#`TUL~HX`{5xumkwT1&M&Pv0U&8G* zW`3gyI_SXZ7)<`u0IG{)US0RCiPfAtF&4#(%vG2fpvm^#kFaU^wcgc?EC}FW){dmx zl`R(F7mTFq;dr-!PU-a3w>}ZxZAK3ujKGnUL4;lD4`qgL0Rlnd(^#jCcYq>3oxSXm zxQqut9X}L9a>L5v!4^HIF>gHI`&>4NORTmlfnm@OBL`&UI%(4C;p9MFHouu-=w58U z|2EO;1hG)2YJyW8sHMVesA1tS24%Y2H@IYMeYTsz71}A^eL6hqdUNN$bvlB*2w^aq zr@w43m_5THy?QlU==56m>8UlYoaeW@uo>6w;NHr7Iz1!sH(E+|{|49Tw{!euyYqio zlaxBFB2lY2j5%Hhl8* z_oJ#a(9l8=O%Mm*6Bu!QJ=$n#JhmUA8X5E)Ztv2aWc~Mm>y)Uvfe;No>Rf}RD#r5R z1kw!LHwYvl;q~*3z=PUDQ!pKd{>2<2z66|B*3QMqkg!$n-f=WR}Z^fQG=^EV~TXM7`qo=U@=T|Y1v zrL&SejD=A#RaKPRFV*e{%%(X1OCTB>6msB1^scyk!!BZGtjjDlXSCaA_iCNrQPI%A zl7u_}gv+Spd3nK<$$auSY^A>KdFF^(oAYh!#hKklzz+HUSCt5Brs~4$ajFk70LusT z=LjKpiGMuF>kLw{V6t(YUUKTTMrEbI0<9HYno*YcpQioq^H!)k5K`yb!xwYDW{7sQ z=S4F*(~=}|p*)|B+I)s;HgM{1qB6E8etvn@uty9iVAAOH_l}C$EJvV4!*zWPHanzA zPSs;0^lr962^QpA1%|zOhUduEe{376D@x$9u}Ea+hPycCmvfP*wg+r1|{#~v%!B+};U4$=IHf1e^} zAI!IkoUk0Is6Kx#3;m~1k%nY=e`%-L30<(V=DMyTFQiVWd3IDoZQI8_G(t`YXhDTc zQ-yS)(iEf=Ark|r14ikv;;ouiPTPb0Bb@#;5YZVlh1oX$WhakJ3rPTqq7nD`1gej) zHo{`9)@9H!Ap^t~W|rja{QNJar7@7z!VyNhEzaqcR$H5ds$)Sg|5Z7)H$(Pa;kfMt z);*b5s5tgrcsL}N=OFhI(fvW>!8|B#K-3_A!n*anA5>j? zLA_3CY};mj_ZYtEE2Q3Ut`VS8tZ`!se@;DEZ~zuoBd@r(Zn&vyXvo1D!it2^ip0Xq zX!KBC7uZ`MbSMLuQ8D@*+#n34gswVlP%MCilBui$p-HAk_9(`)v8C0WJXk z5pH5nP~TE!QBf4UY_UZj9h@zw@&OB>2xjv7?YuLL1SNOiR1d!+A3SQh_DTYK;QZuJ zv2Ic0sralUI}lD_T;5v0Ly=BCFsxt&x#AT|jeacczGjN8*13%Ki|djF1$kIK#ROgj zT7I4(fa;-l8IZA+ZRf-uIj|Ivv#l21gEI`^3t;)L8U|2o8o)|=P!nNnw?euWegm!L zOL@7*dAw2 zurH{||CPAhl`wl@Uc8!DVtKMHjV8)U5;Qy0_9xe#<`co^*FgMG`_~p15y1wKSM%Tt z+1Hu(FS;~pZ~D*?0J`zDv~(yn>6dZ>Ocn~B8k9`!8#QK6gC&HXBqAm`m*W;>RaJaO z1|uplOGTBoJ^%jkix3(Du#NxaOdfw4;5|UGfbq?4@lzwCqs}l23ctBRE`sie}+M zC5R;|K&XFGa$W-BBS@B&KhZ)P)9wH^p@w)tMfCOJqB$BAQY)bj`#DY>ajzM!pKry; zLtMmazV2fc{FkKQ3d0tmmC$wUK zfq-xCfzbvz1qE;kOMEB!@6da^id7Z|-xsn|aB2GYqX2G8+kgyQJ3zxQ&+y;?%JTr& ztQ2ffS{fcxl+bAT0Mzo7ot(JK%F4h1!WjbK8SENCPSyz^SsKyU(H$}d1_l*)%jKrg z(a~^b(xF1}&nLGn?c(Lg!5Az6aqmM2(3sk<6we(9h>3l0CdlMO#>NIU8KpsyVEZr} zl3U!Ils{ky%5BFp3~-`9eE1~l$ql8iuB{B%LePr`sxI7Dw;HGb0yqPMr>+8#TZR6i zGh8Xr4M)-kGF#@<3Mlv|VKrRRm90h4$w9-3J zH6s`|Fd>>SOKTwX{o_{S`6o(#2pa(w1|NnXt_-xSfZ8ZD^qB)~Az)^SY2m$qgWrV^ z8Z-~hZ*5xqj6iP19ZA2Qw%1I(fIRRP3)bEL;{w2}yhecjyq?K{r~_z-GI4VgZwp`m zKuiHFhv#4pLG|I4MA0Yi7dIgK7rTR^uysVUTIXx;HQQRlx78 zBU;1PT!e!N=z9UG9ZubV5vW;%l<>4+C@~J(7y~Lg-xjeoP^N+$<*}a<1Sf|Ej3pnK zS6UOM`MAFwq(h-|Fc~ji^IV2=iW)eCuHNa9Rt+>V$?H20)}hh}ieK zSq{8!*MTd@(cFtkL=8k3uCK8Zds+d5^O|OgPfX0kmj);MyFH5O;X9rfTF|)Mc-q9JL(hTPVuQ{uvo#3@TEghjR;4m~f8T9vZ^LsJi-{6FrU*eHS9fd}n4r zn&n?sCp*UJ)8;uy!=Th?;UxAucVHH4xA;d3@Q}P1??OXQXiT$3oc@Fe7l;Ee*+=3- zLP9b;Yz8(rCp&xDIV^n>7#-H!Nl>d?x|ul9*Cpii=Q@6HW*=8j^?_JqrhLTa;A*H{ zGtXo}7OH&X?33QR;pYA*=v26Y9Zoh-7YVl|@;vfg3GVy{i0n1NjM?Fv#V~Dla;<{`}3E78eF%LCGLH%x&*=sP6@s74Yz%rPY`LixVWTt!4UdY(cbc>gw5Je zxUc>t2Xls04rCMf?GmcJ0P~(4GIo|i0F_myaotS>91z|Rz{>Y}A2GzDO$D;>!WgztzG{*)FIWPEw-;U4?*h zM&14UwMCXhGb1HPgARu1imS!m0ZCbN6~3;H&Jc5YR`g)a zphimDJoZ{j)xrBnqfK3?2oK#X`F>1;2zRCE-YqIAro*G7*Uqkp!HB)CX@6*p5OE2tP!^xq+DE%cf1s}~2sJkhzuyo#Ouu-u z2SpupS5B}H;A_SgxL~jp?y>@M+}8YPd#>%g)~>k(3jcHxvis0gRc{F+OwoRo>%8^* z#pOhnGHQ&O3xP2Sz5)z{&>yLp+`zA#$=9%ndtxS>-21AnN?uORttwa^f)9>3oYcN7 zrEc5}F@UzJPDe!AXs$|?}>Tl6TSu&1rGDr65Aw(iT@qyg1X4+4W7RxIi8+EAT z4(VVu4UpA?(@sB<^=&2!kWB{PDt%4H{GxkJJw;PqEe!$D)l~>FFz1k+d*sT-&##dj zJrdvA+WN|y6dX`>ZSUyNP_}FT#f+h*ru)?49Wk-GE?Ef(A8Km(D+GLwwmB~KpFDYj zS>FKj%!|96L4r9XFE8(5iOYGUB5L4)m$cpNFB@b{Ye)t=+u3Aq_=Zwp=ig%=^thYt z-pSc{DfRNotGObt{aPhiAD{TtzoJTE6aKn^H9AO*Y_nf_ABZzovxVj$Cue-RJ8zci z#oy26h8*?&?ti%1drejqgHDa4yBNch6IN+z=#;ytjAx^37asqEZoQ8pZ@yai>$!V+ z!&)Jm5_2GR$?8-X_v-=P zugwp|l3RnA<8o9X#lok@lI0Zv+8TwOl=_IN@(UQ^7s2dl$8X^5gYboWbYObmz&e))T=JluYyHf zH4VM$#Ft-C;$i`G6Pj_wm-t1R>zIMA=rs19Pm*hQ`YV@J2A z84(;_erd}QcQ>1wuzVP1bmA_*mc8}9|Fv+%#2QCId27Mw+K;t;EW1x%GT5yN;>K;o z3#N<=Flz>*ZH(N7YeoRPrNNi)Z)n@_j;f~O&l+68s8=R1qwX+zu=j+zV`(?X)$RdB z_^BfBsl=>CUEHU+6MG&$ae6bIJ2@L<#t34DTk3VrY|v0*5&S;O3))U|?q=*gKV6VJ zy~E1-waflI1&Wtj4{-A`GNkI#UYMEfEeFtacf7N^`|66cMRUGahMiN{w0%}VLBV9v zK*5REjNrM8OE5nAudr~|i@7XH@ufZdwlaF;M4Z2RbO&jvF|m1xkfk)%M8s>5Lrweg zMRr<2NK09La@1H)Pfrr87nWsVeeg8iu+`;D;&sQ~jV_UcMY3+^E|a1w8;j$KJ0OQt zF}b0S@4ta9t2&e(rIE|WzJvf$#YEMM2NVGs8mGZtZZ&nIcdygsR#j=+rNOP*7o0Kb ztys3U)KPYI%ZGFcvT|5CmFTL2U8(AcOjxih`S(qVwBX}V-Rm#tfu{S_{`+xVU+{S< z#%;yyvmn7b@aoGoEz8Fz$F)m*jZem{tsf%(FX5S;ta!QgS4&f$7RjmA&w#CPv=%!5 zfpED}O`}lb!;R;5seLOhSgx+Q<-)uaxDS(G(O5$v#VfiXCkJ~%iWq3Yz_o5nZNKWw z@LC1^RWI5h8TM>j= z1UgkxSKa_xv)2S%a$qCE>{-9>a%m;yv7_YoBxi+*`i)DXjt-YPL_M1v+p6^n@V%)J z&P-Ixg^|*Q8ZJ4xx$3Elpt2X}%0yL{K^qM3o|CvdCIBwvhC^89*B}995j@|uJ2v3FzkWoxla^7ruz-p=;Jb(=8~uLRJo?Y&(4uE04r7kOKkW>V;%^UY5 z@Ig&DyGBjq+cy9J*P>PBZSSo_G(1}UbEeb6ZPb^?Y2ftxwRwd|qa zfxij8E!N1TOd3H4NqsdgK!4VQonQH559a0FD98hbs`X!M|jQIReQ~>DcO2y<;XuE-b>ijSa4Godu zvmi?bV`#Cse)jXhQ2NWh8Q->w=d-8t{ z@#rYF%|G0Q{Jh;QTtZk}cxm@U5#IkATK~jEC=9jktwOBp&3G=+8n-(Dr~~UrT;#+m z+`ACKVtLzC9^>V`+RM$FxwpU0z9;Cu)K|$w$LnD4{)rEPOF*EUQ~T`sa~2;QBS|;6aus99%q;Eg4YskoUZ+GhPVAc$ zQRHgmhk9t6K8GlWBGjkPb zW?C^hFf_!R*9uu_G^sJgw(nwqw5E-Xjn=L#;BTnUCY{3J`+U9m6*5k&|J~Y-2Xxuc zj4+aJ!L&vgPj|IZV9x=zpU9BxQ{*ma%MPXZXiW3h%>h=HWZ2rwaSs)F zahrbyOPn#Zm~b7`p7VbL`XIOe)&7JO{0;gPDY+N<+q)2}!L9&}V35@Kug19)|Re`KM>^atemEqdB&!N1c`w~3is}# zAQ^|1df)A!qRN(|flLn&Dr!8v%xC7ixzzCYh71laBQAZ%eR)%@HPeW>%(VhcG@6;> ztEQfwm1_kxpjkn(@F7ZuduBM$Vl@9vix~!*R_V%vnZG!XN__5n-RlZiI;PYd{m@|s zXv*X2l*njkNG>fe&uD1yZ{2$L!z6pJeol&{S!~%@WYpCMV;qQX5x<51nTrO)dVmhuU5&huBmDUt zb4$zXu)FHtWhITS9mEF_UB&R^~1v<+$IeGakT!l|L zf3u$5fnhbVq`4x2QmoO&(9#Ac6};9bZqawJW__S7FAIS3cjPAN9T-YIs3d}iiHW(q zx|+Z(+O@W?luN;l`v%+sPz2eABO9x(Ec&^+yJsSiFCX8E&oQ}A_AV+lH3CRXp9#LB zM!2*5KS-!{Zyh!fSc5`APDza($cA9*2{&atBsf#`~a7{AbND zm>nOvPtb6Aew*R*;)rhZ4FCR*qPVu~*E{Y&?=!e#RBt^kEy#>s0cZ_i-ak%4TCuWno|Jo(lnS$$+=-37 z4Zv)JD+CA)Jj0=8I`R>7B$$Qp?;YHf9t;#g524cwA`EC4IMOtBye?up*v5yZ8DE-3~(~1>v;8avrmiO}V`gzFMBa^O@R$Wa6BrsTO zsFBw1E=g#DK=$m}qJXNd?mM8*2nRbFEe|Sl?X6^~EdELRd$OkLdA_e=ZXsWz#b5=D z!v6EWUEkOC5~yrz4`&J^RyR`0h2ZZ-xEZH{P^{2 z>fHB~GTo0lSrt>vhU>P2gz4U#DedF}~joZEnA>;Dbm8mEWmujL-R+mhWy)ySNo+ zv6jioTJw_o6hkf9+aE_i)s-KTE;^l}_KfUD|42~7%c<)l5(A^L(Qk%DW)Q3mg4I`{ z(_iY5);(Wek4uqRA_L?^y%^&|vuRF$Y)?6?31}6r3?RF{Q$4*rRfL%=;wA~$$*{m| z*x$Pauf`gfz4MOtz9#-E*Fuc)Rm_Hmot((h@*KzR-eQ&tN=(>3++Ty1@SYx7mOxa| z5Ya7JVse%La*m+V+6YaT&Ks9@J$=1zM4G5;sDdNa|`yeo~ViAu4P7rhjQ_w#p z>oqTFnm=glVmaia;0T{Lf>eHq3sFbg?qXk|D;V72$B7#^xI)^?fURsqL~ueA96T?W zu4H;3(G~YjE2rh##-!uQ#rasdJLjGH!#b-vdZkRF;LREWkExqybE@$2bU@Zx8dI8P+GSdII9*=}$1?A^PLM-l!A8z(32E|2w{ zHm;e8JGH%9HDn8#dPGsC>Vynt=t?xAf-DFv7(P|ED`xrElJNnGT=AW0&{`F0E=L+R zcR9RH?PB!2R!V!&+ahS28|OfJ|dyorKT6g=kHJbO>v?o0JZf*jSt#IPHmG)oY@MNSDLE5Hy?H? zBzjlhC!_IQW~lW(UU*`VWU4H7^~p=*whjz*yTMuez@#nfmeix`UJ~b|HGRT3jF%qf zPZY4NQ~jxPD0hlB1yJw(`yICc`I_JH6TO+~_s(vcGS;dCf5~1g4V+0_@+>?lbDyCc zFnri`NX#9v86p0idMTA)q@kqvODSoY$c$Vl2Tv!-F!^XW9 zJ22s2(OinLYi%dZVw8i%bD06R{%|HtBD1?kc9_6Fq!ZKvq>zTi#ez>+_iF!!ZtE~d z8OipuMnp+HL_|{3cyI94HrP?iTzp}<|M6q?><-zVoCNl`Fx2Y5Z2^m+L+?v^KSC5^ zt+6poBD&0z=NmL##Q4lp4{8kR+-NiOK%YrlD8u2bwp0ipfM2WRv_7o4bMYk(D=_lT zki;R3FGanhO!r#?iIyl(Fmq`*igEwySGBO&Aw>$ z7u%4F*RBImNNaV(n1 z*$kL$pdarj?i0iumN+2_t+3OLozpC#Jo|;|@UW(5$$t9A+si5tuzgm&hF)eS6C9a}$~ zbKGELUZFi{xNNqVJ?D@~LOR*#XZ+hVLJYK=z`$%cf_Y$Q+g+YgkP0Ey6D04DrC{xlT86#w# zQpuDIr3^_(Lgx8BcF*(t-tS%C`_H%5*Lv2oihJ*U@9Vnm>pIWlJkI0Dp0@X`8|h!^ z;}Wo5qqcMZu|MQd!G9|#z>@pB&zO$sS_WVG;$;g8cK?bF#m1Vpo*ksK=FOKp%eOv! zRID+DTT^|udR1*)7YOLlJGlLc;zK_I~DFn=)LSEXL`hdR{q1OlQAba|DPhFzZd+T=gNBO)Bv z%ID1ze6{iVmABH=T;x&4YY^SB^roC8x@? z#*v3-eQwZyp^iTtvY+|k**&5WJ>n7NcoBrO#yo6M6AI+BS)+xVQJWOJ>z_r}_V(wP=1;+rL|vukPV z_9xY72ePUSSm^j#XB|M`%F`{#2B&G=u23awCmR3I=4pAYV^-E-Im{&>Llz5KWi$3DXR6H+*ngCgdBGD z%m7(QmCu&0LsR0-68xkYeQu|wcFer#Lq~>jO#R{OM}-9{*1{|X7VUi~s0qx~UZ08$ zHqs*fEE6wqg_G`KkKJgj{^N3!xI5~Z!l98jX|?6;zc|u(uCG*8(F*jx;&G|Eo=BQ` z7UJVQd?$| zgRZaGuD&vRQWt-i@63-6!d&-?bl7TtC)slnX*-+X2y*U-Geg0I^vva8SMl@bIPVzGs$3GQ3}%5cjS@GXN?fA8<(XVBw_-ik39P8 z^&GO=w5(%vEaT+Ehw7>U0kouQc=R{TL)$|{#aq$i8 zq1=9YLl4?qO5Cq8aRs;Vg;M1kKhZY9IxzE`ewKCCd;TPWAjPKYGp?Wa=dp!)sqx=2 z7dFNf-h&>iYXwcxmCj;>>fLOr9xEwNO9uNh%%uO$XaNhZSslPN+xn{)b)?9q0$G|{ zH@1uPPTqE^`Df`7v4>6H7|B1^E35k(s0tr(eL~s5sd>AhB=_GLprgUFwBpX&>NC7M zuX1@8p}MI+t8C@Oj>?~h|K}PmT+@qd%)R$Khus=s>Q;l+#&E8IdqeYo*PLX)K5LDP zpJbXFm5@;Uy{T`SC%7ubjrZ4NC4Ds)aki?0tF*u5sQNgKf^uV%&s%t^a+tu+~ z%qP2`UXee!E0UytF*Q1ZScScTS?ke<9SJ+u!P^B*giyIAeSxkm@I~(gQYXR%-$xG9 zip&IuvWpgv98hRxU}7SlI^%yTJ1s4_`9Cgz>^qSodKjlSpil5HH5RN&rVdU0Djlg^ z@AD!M^u#sieb;#pB$$_pSNUc-@X+LT?^aZKQ#+-|92rc}y7wAWUHy!rd<6S?l6Z3W zI0u4b=9IcO;6k@c63za2RuEG((+$*1_q`3fNgH}Vv2k$hMa#{Zj7Y+w+M9}v@MN8w zP^Y(l=R)hD0gJ$%z!CXX2nCfWt#cCwqP3#jHu$1l2ICARxBQ)sP+<(Ct^3!jt8=p* zgMDROxaf%CY_(cj%1ZOU$4-FyR?G*x{1bNl6xXdK6f~sIO5+Xs{mWjx)cT)Cb|*f| zu<<~O`=uFM2O4Jbslcdj8`mkv&Y!^<{+CJ`McV(>;~EF6k0r_0^!Csa9GZrAY&5aA zCHpwu(b4>S&+%;dr2V|X6#A*2yB5_*PvV<*VyT)xU(aOx6RNNjeB2Qd_0-{Xw!2du z_5DW;3#Zbn-!-O@F8Mc_VJYCjB~g1A*rT@xoZ+NTHLK!%pMQ?vE%5M9=J|9p-}wiT z+sCighKKnWKuzftDCcOs$sY@w7kHa6{&njbRwiSgl0cCTZ{yKY>xU#QU2 zR{iRHP{6$6z(0{Otw2-wP1?JiI1!~x}C5utf#n66@YbUc!ZbwAa!p(Znp?dgF(f1~6nH_OQAcz7M zl+pN49%M`b*Ym8X!&UTd}XDHC0o%(yZ`XM@W{xFii+)`u@_%YmYGSS z00`8Yf&g*?iTV4;s5H6Z_C*Nz10M2gwoRHp6fsoo$RipQEZx)39B*R?Cl^2$TEa0q z?#Rvq`vYow&Bdi=!c~8zvT+F1p+?M~r|29Vr*6R-WUKS<5vNjz)kVkIxw&R{ zscU6lA+UgU@zGzu0^p|N;c)=Nkzw$;$2#A%t-bvQPMa-NlTM}3>ynn04eIKGkBs%% zhFcuQcbTlDx!d~sBEU}GX>sBv4TZ&<%ZtDEdPCvk>`6}0VH<$b!6@Wb1a1efGz-tV1WoAmWPqS1yAebSxH zP7Kj`RQrm{RkhG+?N3Ff&G$La_75L!!sKPA#d&UOnY#=|nVS_`>&C8ast_VLe51#(w|&Nd%4x@=sKBbpDry zh@C)?F97nb8%+G;0;%z2Ww1t`bnHxqNM4`(~ASvt9JKpfJ9QZ;hC0OUkLn@7sD10S9?p@9U`^ZLOuK<1A7dLIe0eL2pe3H`g zsLZBxz-;Ci#9_3g+_(!Ls-Q7fWD1BqXh4)8dRZ(x!s^px*TbU81c~6)3dU+ro@WJ@3Ag>P;w-V zUg3=#0Jx2jd|c)_kBmUI14e#yya-hW&`|5@>PTStK$fmAq?=63T(1~WD?|*D_N?}H z!@6*m@2~hCqeGg{sQ7V6qU|7wY^3*= z6c%nSh8liqe<9tsbOD0jUJ1r0m8)AzZvK4a#cqfNpYOZjlg?ow7=3p5U0mDPH2_Mf zBaT?=Af(V{-vnxGTH!et6Mvu!*@ia;gQ#x^-kwSJsr}i4Ts-Q9oBV9J5l38ebJHIS zJyqf9qBWDi7K$PRsh%^wL;AhQo#QUp9x=%Qg8|}AR?z6smYYn^%A%iGr`k^-VeRO+ z7x+pKO)n72AvQ`zKn@MrA>CKU!_l=Y%y7`=Bq#8AKl9wQVwv|dir6aK`}!nj-<`+4 zL8_4)kDhl|B+V=unF%VA3Fkm0?=UGF?=dF3S{rctn0`Z7w>ibYHpxX0w7+$pu;AD# zgpDMqNwUn&qn}bh;7FzD37KYarO}b{$S4$qbt)`>MvvAgbYrZpjwl}nOPX+ha@273 z%2x$}0C`|JOKJAPcC=*Pmf|eLJ4Zx7(^P|na&FQgIXNzf<#7ZRF)}dR?^aHh*n%?$ zD*@S$kkC2RI)5Cj@0yzOQ2~l7v7UX?U)FtLnHVr$WT*R-e%;%ST z{NSQ}3ZzeN<0sddq}4_~}Mu@YMrK3v1Z#6(N= zrX@Exkk8MQnIsnS;&V7rW7PH&$l|-V!IkUO0U5;@Hi|jP?s4?aJ}1?b7(_HK}U3kDS1BN+^DHhB~;Sbqh$5nX$h|}$|T+M;Ek{C?a}+SglUVK=p6dW$Y6 zNK?<@5NW_6T&ooS#!9szD_5{V`Ec#C&TsFFmHZBht)6@BU(K_n`h*9aSODdPeOsZP zN;c)vN)vfZ-xy6@Pk-G~7ewV);*yUp>JQim$QDJfFfV!iU^_XrV!SI3>P*Zlrl>uE zM`LRAid55zYhpWUVUaeaMA`=9;yC8>QR+B;yz`D`V72sJb+_yz)<}?w1~mM{C#t;>Ju31D`*<#d) zkB2AO@l%_+jDo^0ajB$)1g_{q$4N7FVYKR*infbK+5$)CeEQ>o9^eoV9!l+;S^qHvFaFWTWZd_ zpR4Tr?%B@J9quO^R31U^DxM?gHMS}4wYo2C$EOb5Rv)G!AeIf#?n0#W+391br~{J- zo3=U%)LFDtL5}Bl!NEbtCjw zsNK#~F#v&=yPlH`oQ_%k{%DvkR2H`=UP3SbrL*q#_U-ON$BMmLf$ttZ?_%DCin=FUO3~xW?*z0Moo83v zV5L1O&9%BE?0ZZDP9Wl>t;|}7#l*;{(JmJhAVq>FwTP!c< zJtSGNfB)H&u3cTWq3SZ{i+40S3Y@rVE%Paveg4bUVrfG+r6Z4=TDla1>sbG||E>;@ zClKjZiQJz*PjxMgsC}#QP$($K^XMMWyS#~iToz$B?Y3v7Uw>cR=HD!u*WPJoCV4_@ z=m13$ZA-)0wV-{o+QMwjCCdi)$#+C+3BKyOK4nfNn{?&#>uY3I`R* zJp^ajnlg#B^2P=$sx+L~2Zm3DeJ>iGb|*3^q??~kr|mPA;*aSTsFTJTt9>1uZpJ=y zhPnFBnbS3<@hQ%g#ItX$_zOm)r{unQ@Vb^=|LkSaa(Va1^UQu8ll%8>-#yiLQ*QU| zfPs+}Gs~9oxu7poB5_AXk}1AVyFZ?wRNhu<# zNQPH8)3{}ex*~7u^No4yt>LyU{NJ2PiApA#ny-`Pi^-FHgTcOaY;(n`Ai) zn=ki^;qL>pXY8vVzoTkh;bme9R(omwSs`j&DeihH*^ z**I3VM)ppaw>jJ8NJ9YP@z=}bBhnwzh6XD9eq2%k1zez;4_x;V2EnVv2fv@YSoHaT ze@NXR?UJcr^o16$7f56rD@`0!=#Zf~uS>kQb$(+Bv22%?@oNaqKToEGn`EWTe3sc7 z-gaT1<2wJ#M~7B`5Wx9VtKZ<Vw z(`+6m1_!n|KV@Dt5E(jP_HJA{TDhra@=9ioT9D6^CB zOckqXU2up$CThJ?!17| zSq7P%KcmVT{2Ql(KF7&j8)fO+cz1$--YxbsV!qeP{Q<}CiUdRr{9adS3LvJ_dXSHL zGmmRADW81*C&q7*q}7_9bjV2JMva6!7Uqk2(`L7vAKR?B#qV5J`uJ|tyDe5 zVS{*Dt`;#IcWBBymCZD!sQyP_dnkuJp}&+g+asPkTL<4}-_G~EBJtgvUEXBM>-00{ z9SKcB1v-c7q*=>en#MCOna+H@e6zK*W9)WMfG>*Yvwp4nj4Uh1RYhY1zcg{Yy?nWH zW`9S8rf96hQyMYj8np=WnM&K*0=xR3`(^HmTg~0W=@1pUzyG?hd%&&UiPumcBrN{! z7E4Q7%GriZ?+?q^cRx)>`QG#+$WnQs;E3I39wGbGG zRio8O+3zM9)jk}4_FO-g;-4$F_k=K4E+0qjU}nbg$8R~y_;J=R1lc`%1ibH)bF68( zb{;M`ZkOM9%zoI20q?D)S#X-?cC4B&z?h=k^wlJ3@u@J*>t3g@~yi52S^L(pkRueBc|=d8x>N4k|DL`mSmEQ*-5s zjBG=ywm-PMG&P(`M+VM`EW2>8t_!*_9pYssdEzYo4g^lYdf=)cZGZ@93T<_6=eAQt z3yT6Yu&`jRF0NdJX4v`F()H^rQ|;ZA)cL+f&npTypWayF6n!+-L4?T+*DD1Z98zDa zLsj5H9rp#OLC7zaL$b1>#-e-CoU~58rF;aErTXfwB@+`991-l9bBhQc$DL1{=zb8{ zIJ0_ZNVx6*L-}D$Tl7(B+!|mDvc*%^{CEEh)k4;`utM6B#fuIeIIvS!+$7VOPPs}I z6_SM&KmZ|j1B=s*eDuCy$Mg<}432yf4=@$i1fvjbo$Qr4Y)v~tgiCd#DvV3^RIL7K z>fq(ANb|X+FIh@y0r99A*BY00%y?Ehl*qD@<$Ray&>Wx{BJ0SYNC3F=V_6#bR zA=9kAB#pjP)@@GQO$d|`b=libKJbipPd)7ARVF=Ff-bOhu@xH1{@l;KyoNvfZN~$rz!h?(bTj3Tpttq_;gg!_Un-(#)3d?+O3n3WnWPQHBk zGEc6SXWKURTK+oeV-ZPj9vWpD(9L>QK!M-kjZSas^CwRZbDt8zGJ?=jzSC=&Y?Iv# zQjkO=HK3W+eL)co-J-h`!>%cusa@DXg*M<#6~DmXNl8huJfsiW$MlHu#>vEtYts!} zVjBMBp~{=pX>%Y#4xS)6p$&|={*Cno`Kd_Xffx2Z?9Vq;?I5UX(YMB=tNXK%u zWb9zo5i04-z(>+<-^ONe;)Iy4?6xgiRKX;>`KlB5#Kfz(!_F?#BrEnYIql-E8z2SO z?(g{YDRh2*o@!1KT>zwfFgp(er_CNjW+5Uybzt7J;sR`{1_sW)iQ`TkY47e12I{hD z<1qwsNyN!<4m#t9DSBdf?~?lhnmmo^MMWG)ZV2Az+7Ru>h%!*9xVRKa2V4Sa30;o@ zlK|9Pyw07=kZ#mTH_-jGMX^0pU67Oxme>M9aX?!gmBg4AfVpw9WjNW5` zc`1PlxzFoAZFtjbD%l*NMXx9~H@Aw8PMEYDQJnMK-$dPm%zN^5Rwp1C^&a70=(q(c zgHY{-i8?8ND=Ta7>Jl}+A9q9q@_z}N)ehX;8F%Cj1TBYlL1Jof&@!aKYRBeVJ9j7d zthHJPCI&ayX`VjaduK@meD9}_g?lE=uvm+ZWVWE=Tf#K~^uDn!IO%p7V+)!VG_CH+ zsJ?voQ264hKmInhwpQNWW<3rh{0J4-O@UTjofkmU1+NWbV>tLFOkEI3xXP0biSyGL zl6mg6cwiKHBlFas0SPtAOm=BW>KIeEx0fQ(Wrr_6GTKSH>$b3?@6~t35wlIs0eAv_ zX#2tvBtnKdkO4L=5Qm0*$tSO=+y$X-cW(W_06+BGL($N@slo%|ZY7Y)AG#_c5~Iki z^x+_L!_)s8{3CK)r^y@R#alih`%r0`($x2n`@=kzuFF?dw7-r;9l3t6fjXWOZpz0_ zojzt{g!P#)#lC3-+j?71kNg$yQlPQ`aqw{Ewch$Ly2N2G?2-qsy0(fP-TiBN;$BJ~5FZt8O69DI_6r)Kma{ zyAc0BOF~4awnS?M06hU7rb zQ3WN#Br7^5hJ5n)2*nR=cO4za*#}{dCzO0XUwkqMe~_@z4|xIVr~Y)gzS);zU$I{z zfWfYe=fJ07?ydK(|JmJC0613@Ev-h3s)?aSnf2(%Z3ip+_wTFe2fy0M(%0qK3MMo` zLooGYTor_tf}4}%GG4r32ZeIL!@Q?M&>*w@$MXC;hq6lA-It2C;^f=Mw3WpKMG$+g zU1GeE=TB)2TLVx9gr)WKHRV8TPkAL;%Z)D-vBBt!e3L1Jbg#iLy#@ zmfV(BR<5qDAC#~8Q<*41I1?OU7O3MxlJIe}8vu1cgCGq+x$ekv@Kw+qgD|r|q0V7p zgKJq!q^)@-@_?z*lgjK+2R zvmVbB@I2BbSAKKVNh3ffU4%2lsCXdlwk7xG&6`QeD2qI;h5BJy!bDHcS5u1GKX?aY zk!Ye72LPg$dn)Qk`bjHs_wJE;0Ia^gaxpEwdgg=Nrl4I861UVJJp8Q@DY{(9zOQnM zT&e2MJ1`!C7<=YFdyUG?XwCGGBN5K4xvMUCL+?q(trKC zzOgYSxED^PZLbRk2FJ-NW}**2I8$FTgr9u<%2+K|6_HandY?b$`_@!n-ZTpr4pZOA z;PE@h@+Lan+U(`F96Lw=Wp{t(&4gcoWgbQn$tW&BxoWE(!)DyKqW=T9f^UF2+*u93fY9 zB%eE>S+0-vH6OA3d|J4UYh{pmY|?)p0**jTeWULpMMR&^!Nh8F&XJVh+{&p%YTNo9wG3*_Tg$ee*xB*ScfCqU|vG6<#gSx0hb8ukoJCDj6+J z*!j0W91!*N0V5-OPw&DP{~U|8BeVh!bR=s6^?%bI49FALIS5@Vb-{E-My?x{+**&= zLdqGq6@o1-xiuGgI$XF@2fPHXJm10_Z8Rvx^j5)~iiUrs#f@|OzXKL3e5;-A zM^V}a&-H~qdaz|wroT>plMZdt{*McgtgLk8ZAuIMt#77S_zKUZrOp#W z$uzv_Bu)(m$cl+7nlgXpyRKG@f`o@{6>`1wO+%a zws~W^SA<>w4G`tN2hN@PaYoaLhRWrqd3oWLB(}&aMYfMWp0V#D; zZ8O5;t1CCA3lBPye%~>cB^BCWb&C4HkCg6AH!Z z|JIr(`tP@27#-XO&iF373r|h5GR`EI)XgYDp!ExHzVBY8&rO!;@)7bjnaM7p$lTs)y{XtZ}(1UB8DKSWOWMe-3uBi ziF@&_En;_0^>^o156 z9Nt~o(EM+zI5r77e*e@Y=|9)NMMf}>rVXrG=IHA47W?;MVsqOOQcscE*k3unzsGfw zdVpB~RIIMPMfQK66Jbe1mQWYP$R|>-Xd}8UXa?v7G^*AleqJp}`ETu<-mGr5rFAHm z%ba#(n>_(^iI|OLspm^I68}xPSq48_mOHP0im#HHeFYVX1M_sTj?ciO)8)TQzTgu7 za-QHPEeiChkxG9mIK%S)MEcf0+x-&NjK5!(?#4~6+tZ$k?oQ(-4lt!94Afg0D=qSy z_^kX(i4qXDG++r8Fps3sPxcTk-&p1%SkPvAjtUri&-_fJ6<`akuS(iamjIqp>rCP9 z>xqD&P_1p&18M1oGp(O1C37qkYgz=(1P?}Q313^2xHtJZV}=_KD@3uG2nIVg8%iGi z^MIdCsGOHMwWolpuT_bH;9#`#5}c7mES79_16ZmT2GD8%CWKk+TWgRhz8u@}{p{DA z3tU21^Tl;I5;WMt8d$={rc1d&o%qX>V$kijawoMfXlfRQMbD1@3@5`^h`idzVB4>R zC!}Eht(f7pb~!7OA0cTl*t9pRKX^anFfe^Isd}I6GD?@G3$q zK~NcblDg?fn?V2N%+2Fz9@JDLXNdo<4Pn6NhTk0Jr+RiiQTh0VfiPe#lf3-J`OSfG z!T%Yk#Yv;E(YDn-qoL>Ed%Uq>vY~;VqTKmS$H!{x9fOcB*4C(z$V!2#A z=titsYWi<#KiuapxXlwA>uyE50%g6cf#j!QU}>HFYQH3rw$Tw4XM#ipw`6gz#4jPD zjzDlDXd6u)7dL{WK~h3>yt??8*aTytL(KvaTU@slH=b>zcm&VXfbfMiivh)$K(LBX z)g^d$^GoFNVH=GPy>3<8&HkaCj4-t$u4lQ`msqxby9Br%96|=2mgep62iv><58sk1 zS{k&atWN64;9YHl-{hH~l;>y9F6Fr>ncO=OR5wT0+IYER0D!R1wo7HRUR${gO zez3cuyyG{@|MRhpztL=68#~^u=CmqLs7|?e>xG3Exc)xh$VPqB@91Vxw3(Z1UbSFIopWhsDfL$Yo{GBDWsDjatFx0TYK^b1J4(%st| zh63y?pHlhLp=*<${JUF}wy}X1W|xl6C<&7SPYShm^!5(Ud4kNtANv+D z>BSf`TKR}Ngzc!fqH2$QQdiWUn684jHhU{OIVJ(1Ab5rHDU$9sTl2)}=}j+ue0G6L zN)pDU*h0Y)fOedgFdHPB!RkV;PWl~47`2y*YVmXrO@pNZS*=rVgec8(-QLYmR^M^f zYK>j;P~;u-_%>XQC)ozQZ73#Q)X-#eW0MI%5?}u3-R>UvOrZu&ARI_ZG4=S+dI;7~ zB=aiAu9N4RmsrLYXOES8^m#;?W)lb}3|Vvfs`xlV6)e@>eE%+to-JvRu)q--YL#a2 zhcFYe3b+JGIn$l%M*>2kqU_s>fjGaq1!ZM)d82%*n$a$f-zRmCYk*Uw$X_R5(q6Hku-gtd^a~lh={-(NcRRfr>72=oCTJ!req$TyTtFNsilGuP11>nmqDv)G+p z;KM_~ngVZ3YP3JI^Abrb%w}sPOZSHlGTDC=`6|pKk7t<}+H9?!nvy1wcZz$@uln$Z z=rSx+H!a;PK#~Tw1Q-snl`_pUiX>F0vvdDtwOU=|RRGHkOifDy9)&TZsTQrnz=*1e zys=0Q;cWOQPgzyw#V0$E0XVi&sy*tw1UT_3_lNSP~Iw(xE?X7YF%_z za4$RK`7{adGs0|xvmN@4$4{II1Y|QTe${MHUe8?*rMew)??8t=Afn5HkuKR-|C?5O zqiGJNFI$n46E>zBM_0a15(xLoeiIKL*@SQOm@=MTvV?HL%rVd)^y0>-kOl2@v9Ll@ zlVfvVMWVG4n@j+7fbQq#8;;7*5CVy0DS9+Trx=0n1X+TXWC@);v=+i;dG$F413U+# z;`(HHM{+0{-2FfSJF{C@V$c->uyS+61591g19DbcH`5MWB-ly{pyPCFL!<&wK#<{p zL=0YLsNHC?P7mp0MHDNaJ zCt*8ZcUBtwEvqAEcS=ct0&;QREk1T|w?NC;CqnxK-{47y!2bQm-Ba-;@c6@!XlL9N z&vYIPI6t~~p`8(=PR(K!?Og&lL(szsi8)4pUpMEtBZ0uln9nCW2IY+zckgXI zyCG2wX0q4($?xV4WUm_=rV<^~ z|F^sK->{&j(H7jtE2XV;En z0|O2~e00x?$-m|-S{V}z#w@w1=jP_L4=AQXja5u9nZ^7(b^wSX%iPsn+}cV(T{ijh z(QyJpD+x0}<1mpQ1bKOEVzH99!Lx)|;3h^&@g_k~e!z!v4%!4R3fK*^3k&pdsrW#6 zbt;X$PKLxf$uOZLo&*tug+<53?H`HSUG`OyVQ>=$PsPS&I40ph7xu6LXnfTckq~qv zBl2vMgCkX_C{#}mHSQU+V4)^}1{0l>RQ7NAcie7vg#a8VB<+d|texX?xSTHYv>9-T-Mi_roQKv$!l8YL#snjmtjO?i7>?eg z1%3v_ibSIuso_SMfSF!G#8}=OOemwHeiOfbk$_r)1r=z9ofxP@N1DrJRnf-<$V~sH z;b_nykw!blC>tHI(Y$GHK6>xgKG2fv>^3U}gOCOYAHY=d0#Ba!?%6~fN+7-?w+pU| zHkKe&a#j{5xjX|qK724ZeVRU8bMIt1Gx$|#x3JJDYNt&d*>m6kU8eCpV>J_qL4vr* zzwZ)SsbP$=>md~@+L&Qi7^q=H=&z8*Q11Le{5Z~)arFq@BLD@Iv zC!@?Ka>AT0$PzsR!Pv=3@G>ZiAo$~M>wh#S#s1_863mNY7$~z4J(eO8U%$rg3C~Yr zmMW0jt7Q`}shBTkK7U>hmqK!2dB(AHIbM@%f&tG@1x~Cih=JAS2W4sT(c`O)C`d3_NWd|x^ zX-Yb`QIB~}?<2f5xb6n2l#&a{8MtLs+&_ns!s-BxvI;oZ1dE{oNJg->JH?FZ_>mnY zd{Yd$jTHrrQ_2A^Ke(bPZ|1#EABPV38Ix?{1ZZ!+>=~WAKMs7RpgB9M#V#9whmE%G z*~d%!O`e;w&gVlqAu276x%$KbWuH}d^>^IkAf^R$UV|kG36Po3$I%$r)q!Y&N@^=h zE~z-U>ZYO5P+s@RR_Gss-=I<(hGyiFVW{BZ3?c>+XRVU!owdw!^NLeX6<-z?i)n~B zI5`<{b3QfLVQ~C7$NbL|uycN@HKQ0A5fKp*w>KW_L)vs51QM6z4W=&L21{gA;wXtS zTsx9oQX>CLgPA+123$yqeOD*?t9;*GkFQtuukkzhyM*GP+wa@suZU(e(clKEU2v&_ zer-T}n?oUp47xz6gx5C>Ux2uZ;@Un2NypkC7Q(UD_y6i*aO*fa=C|GK>FzGR&*Alr zwb8Mb!bqGGVUo3BBf1MPN44YDw9e2^BXV~E>wWj`)W>PkCPlXCId9x3R(+s!GFo9q z1*k6-omxEDGco^gl!LBP4|>G5JZvQ_HNL@U+a~=?Q6Jd>;t9@*+m5I$PU-&q{aY`Q zbKnzpBFn^5V!G+9urrtzq$;g@1&I~8J+eL(!7;=)*^B#q4*EcnddQzyDWvRmaS4+k z^erd`PRETRkYHYs4|bcVP0z}u9-pM!vQxaiy*bG-?>c|FS_s;`Uk_d?`kZ$<-JeRd zk-SoJE>N>`hP^p`Af=7kEhA0yK9Ogtl*Af2jg zNM#{}0x|2|ev^|YcY4Q1!p(K6HP*UeMiIo>sW(EA13}xaR$pKE8hprW{{1U^eeuv< z{tu{Wg0xIf0WGqpNmsUFmW^p+fadJlq20etQnjU|C z=_LijySk$33$g&QthTW|e)5FA@yM*#pp&9L+nRZaacgX~8EbLxAsa)Em_&3I;Sq=F zFY~}-bc1*3D{t?}*!|7*{MWd_w4G$1IHz4MsDj)M&7m&;0X0z3D?!FzFV!vW)~ZKR-Q?` zi6{Wp1kbb0n+EOAZQ%>vdn4hGi>Ww%1pv0ZH{9T}y?aad)PRdqwK!kvfV2F?eZ=_t zURI{rCe=e^6e`)qn!mONY`AxY(zzdCiq$+wuzrTq zOy!16w46m7i#g+2;p0Lfda*2_`L7#aXTDb7+a9X@vnTO=;oF(nKv#v#SltQJj;MOm z=dT}!OQ?@?$37F-#G1kC$|@YC$|Cf%L|kgq^Rh1^k#Wbmww^8Rwwe61_Fn$Q`HF?9 zb8er$_Lq(%r&&d&Q>#Q(DJv)(bgZcOn$=IATU4|*GNbZeB`agz7X!?_mDFw?=WudPGDvkAEcJHyQrGy(V4NgtoqHPZ@jfX%IoEtP@ z8Ul$kj1ILeQ;X{^>I4uGZFtzL-Hqqi zW6x^~!(=_h1s+19R$N-=t<-9jYvJ3+wTQxWQNN-juX{bI6%`bjvcD4&^2R1cm}EhQ zf<3g18eju{c;6PoD}o=Yh~oyFdEqHCRO^O@iL=v*FHlqjtU~(An=^vnz@8BU)A|jr|wy;VFyOCMDFmu)2)a% z;9g?0Eg?dpWMz_D6hcl}J#-fli`MnEbai~P!x|Ps6>XZkvDAe0q-KCnb-DP+X63!!al` zp?L&po8{-ii?`;yCO+>s3f42T`^~J^6J$n)ldC3wQgVe73Zc`f`8uEt%5dv!~ z9I0x%UU@&slw}>OipoMB;QZtLmRX)`z2E~&OBB39+H6d#e-hBwfa3XLm0bUw?O9Kj z7pgYGSCb+e-=o(aYk*^{8`fnDt-eg!G1BlUR_wPwl3WZq-8!7z1cHvUGso$NC(X?x z-d*V>5cHLnggGV6o@!O4jy_5-C&FOt=FOW^-zR;Z^xS)n9(iM41xf;&?Fo9dRTv@D_96j03OY^CX^4?e%IlIydbtcF|hPcNlq&AN-%Xc-wB z;E8;T6edE}*|hLNbfbuX-z|1lK0dxLh~x=8$E5Yrl(My|p#RNT9QTSikO> z2hez_S=c)PfnoB2RUVHl04IB>2GKLXMI}32;yL%s{Sf?@-%4H@9)r5cWN0rET*HU@ zNa)O4VZCi&cQ540PcH`hFC_vwZEszm4QKT}ZLD>85|X^7^UxXk z9Q2|5&qi*yF0r##JKlY1-|`1?|JIR|U7ekSS-F+u1X%smj33S;9MKhb->2F(F=6_~6`JHocfWbR~hfW>panIL5Seu4k z`Z4dE$e`BPb}h)c2?&spA{ldYUUTb3Kkxj{H|}3r(fzub07LT6sau9 z>5XWR4=Pi2zsHe5l3KV2Ox;&;Qtx2P5bL*<57M$*qGckrfr-_*2izOxie=-} zq@sK`R{D`hN&l+E^W4x$a>&lD+9mfHp4-&;&eb5}Aj1u#;dNJuD@rYdbYZE`_5dD` z{z$-mjzv{OQ-ajcw}vnrS2uDe+qwtk&}^yiyUY$g4;cZnliT;(d!io}QlBtHK=0#W zSRfGiB_&%PY1E-j4Cft}g-@#I&z-qxt-X!?{d_ewHS*V&_X%s?Y`>rmA$2IB2Y(3` z)?P$RnQxDxPQ+JB*{ffkUI?_(=iIU-H0wkzbn$q>KEm7K#w5=ezCe!}I!+S1I5|0k z(h&SS^FGHLfq)$;ClHpxqN2eh`$|>VmFko|zz-lpKGjYndB-BhlnPur!k9R?1xT)2kJZutX%$pu7SSGJ95dwYV4kqC411tE*MYL4q__ zR>Nc3labUCkrHM_zo~a_Gx<;jmB!Q9|hZEk6Kjph0~ zqV3S@Gj3l4vBZ#H)?O{nc>bILfJR7Y=$6^Avp8R%j7T!GbL@vjFYGdWp483nfCM=5 z$o<&B+wdStRjsli#l#cDJ3Bc&o?G?WYqCP@u?r;f%a?y<-MVTsK8CXfO*@7d8GI5B zcJH~)6SPcBn%Az)`ccqVR8)YV(*RxjhR%mzUy@#yAsS5qF5|3-5PplqTpd{&)&tGg zcNMUdP4oUkx4zt6^#F}>s^F3y4(byBRg89VIM8Fd#ASkF73mGE_ww$Vs&0r&g zDuF=XsjaSREP`Y!`BK#T#61+s>(h~k8}}tE9R%+IU)PIV!@pBa?qZ^Bl`zcRReZ` zClUR{^&|Rc&+>QWT1Y+SVe=5o)mA4Qb}WT;!qE4(v63keea>^Awh}*CU~L?3_Ek?! zjiT@R!kOfQZY_^yh-zx2K19O{=l30S`MH_&IKI5W`)ud2#{-|vr_C*EWcx=8(*fH$ zY-yRkqvU`wc*p36m4KGO_FH<}f$irzkLW@0)>ybyExq#+;FWM#AG@#B4%C>HI#Eql zuH@_xS1>-;7|}plvuJYdPk-|<1K9gy#R9LSdHLuE55!ZX4m{1~*}fYnybK~GcktjU zz<#c{^6Q19Hqi-=I5Ra&7E`8}F0$3(_t>jm6DMSa5h{ zD6#&|&CX`T4$}bn&^-f^2G|pD{4>XkpS+2~TTXLyNz$yu4`(`pO~#U}`&CuSwk^q_ zk8~0+GatfPJ(Xba1glw87CfsT>Y)o6Xr=OE4YM@dE%5SBC_96md<#Kv{n)Or6S~Rr ztU%b@r)p>pcg~L1%uWzpLCPhQ`-2ri@&v}&bolE>b0eu-UG-%IMHy%g6jvyBo!Ckw z5?KI|;aEWEbo<>-^&AM{KM?hYCJ0z2K*>x^xnp8uD=J?HvQ@6lpGI2p4d*p3nwp-z z0nFOn9X+fcG$&>g(tn;VHU?&i{RNyvN=ix;#?F1K4~4ng_M(oR!S7HaK<)%aF-~MBrQ(tY*^q(7jOos+u4cJuu(e>zvCeB^k9Sq z%eolhnzPdU;^Mlxau03*|0Asg{1^%-K0ZEp9V`i|Ay8e)pJLpY#m*{7`twJ(oJU2S z^yg}$dH}8n3HcHcN(*BmR@Tz(|6b&DP*m{mHQ!xz)&E}eX5zP&;0F{?NYyZUF3I`; zI`)CdNoJfdBs|@`Vk;tE*UZmSf8WT1t6@CEdl9;9gMnPs&)4Y)3;n*zu9JP?P^1>| zCD|acvS#`{2z`T1Wcp8&6!X&J{rmh_uL6e-oylbR`;l5yn|0J-6BXGI#+Bjg1|lYD zVMk$`K_0_%(3OECVSwK??1>dUS_}IWRt}Cw>fC?t`yn~3`oL4i#2>c>7xN=U?QWBTXg z{P1xF1x8=y=jO2eeu-*5s-@*{>BRBlZyAIC4zIC_iJ+i18+=SO4k{>Yt*oq6Q}V@9 z)tf-D_LlD7i-;zIA0P&e4)PFJcvkC-1P2EnEP^%0_W-rOBfU#tH!HLh^d`WyPiSG@ zw8`mG@ynO*NUwh*fen?kD@;$uoiN`r78Md=e13H1>D7!UPZ~3~v#<<&Sc1_!gpgQ! z=leeM;X8>zmpJ>KkKr>T10Sb`0LEC zy}jd3d~2=kmx`Qf1^$`EsQvpd;{?>Zaw7MFRt#lQv*>4q6g@q8yz(k5FKWa{-?woJL)>Hs1mGc-hSC0$$6f%OPyUP&>)*>T z#Q*!#;zDrd3GUlxT$q%UbVgyLYX~gf;r|()%a=Gwe!YV@KL~yZl1X%D#An++lop#9 z>1%04k!GxYdpG#nzQ9l)Y)3!(%4C%9^_&$ZcK?trcNKQZl7odmvsFp*{KW-m` z4a@PXq;KSAhNzSjHn*FF47uK7A%&b7#X)`nNtCz8cY;6Ab+`9@E zoUI{$;wgywn3_T^_7gk@>=Q_lBf~mFNee`i4$B)yJ1?kT?dZ>}fj<<@A^Vvh&3bX4 z(tMULmZtx+4v%0R5(WT3I9*a+`W_3u`*O>W^7`Vr^7ZBM2isX$Z=k}xll9-nz)vt# z3*Z2!xGDcY^BqO zW{LMna?GT#6U2L*Po(mqS-E@rSov^M!Z|z+M*>yIKKt}3L$I-&PI;e@E_`Hz6S(b%eBI_I*&r2xY25MA`fL^})$WIll|Njge gX6pa)M!z@62BLSz@10}1O~4;*4L$W7;@O-33%&D&0RR91 literal 0 HcmV?d00001 diff --git a/docs/programming-guide/related-work.rst b/docs/programming-guide/related-work.rst new file mode 100644 index 000000000..2222f70ae --- /dev/null +++ b/docs/programming-guide/related-work.rst @@ -0,0 +1,209 @@ +============== +Related Work +============== + +At first sight, Triton may seem like just yet another DSL for DNNs. The purpose of this section is to contextualize Triton and highlights its differences with the two leading approaches in this domain: polyhedral compilation and scheduling languages. + +----------------------- +Polyhedral Compilation +----------------------- + +Traditional compilers typically rely on intermediate representations, such as LLVM-IR [1]_, that encode control flow information using (un)conditional branches. This relatively low-level format makes it difficult to statically analyze the runtime behavior (e.g., cache misses) of input programs, and to automatically optimize loops accordingly through the use of tiling [2]_, fusion [3]_ and interchange [4]_. To solve this issue, polyhedral compilers [5]_ rely on program representations that have statically predictable control flow, thereby enabling aggressive compile-time program transformations for data locality and parallelism. Though this strategy has been adopted by many languages and compilers for DNNs such as Tiramisu [6]_, Tensor Comprehensions [7]_, Diesel [8]_ and the Affine dialect in MLIR [9]_, it also comes with a number of limitations that will be described later. + ++++++++++++++++++++++++ +Program Representation ++++++++++++++++++++++++ + +Polyhedral compilation is a vast area of research. In this section we only outline the most basic aspects of this topic, but readers interested in the solid mathematical foundations underneath may refer to the ample litterature on linear and integer programming. + +.. table:: + :widths: 50 50 + + +-----------------------------------------------------+-----------------------------------------------------+ + | | | + |.. code-block:: C | |pic1| | + | | | + | for(int i = 0; i < 3; i++) | | + | for(int j = i; j < 5; j++) | | + | A[i][j] = 0; | | + +-----------------------------------------------------+-----------------------------------------------------+ + +.. |pic1| image:: polyhedral-iteration.png + :width: 300 + +Polyhedral compilers focus on a class of programs commonly known as **Static Control Parts** (SCoP), *i.e.*, maximal sets of consecutive statements in which conditionals and loop bounds are affine functions of surrounding loop indices and global invariant parameters. As shown above, programs in this format always lead to iteration domains that are bounded by affine inequalities, i.e., polyhedral. These polyhedra can also be defined algebraically; for the above example: + +.. math:: + + \mathcal{P} = \{ i, j \in \mathbb{Z}^2 + ~|~ + \begin{pmatrix} + 1 & 0 \\ + -1 & 0 \\ + -1 & 1 \\ + 0 & -1 \\ + \end{pmatrix} + \begin{pmatrix} + i \\ + j + \end{pmatrix} + + + \begin{pmatrix} + 0 \\ + 2 \\ + 0 \\ + 4 + \end{pmatrix} + \geq + 0 + \} + + +Each point :math:`(i, j)` in :math:`\mathcal{P}` represents a *polyhedral statement*, that is a program statement which (1) does not induce control-flow side effects (e.g., :code:`for`, :code:`if`, :code:`break`) and (2) contains only affine functions of loop indices and global parameters in array accesses. To facilitate alias analysis, array accesses are also mathematically abstracted, using so-called *access function*. In other words, :code:`A[i][j]` is simply :code:`A[f(i,j)]` where the access function :math:`f` is defined by: + +.. math:: + + f(i, j) = \begin{pmatrix} + 1 & 0\\ + 0 & 1\\ + \end{pmatrix} + \begin{pmatrix} + i\\ + j + \end{pmatrix} + = + (i, j) + + +Note that the iteration domains of an SCoP does not specify the order in which its statements shall execute. In fact, this iteration domain may be traversed in many different possible legal orders, i.e. *schedules*. Formally, a schedule is defined as a p-dimensional affine transformation :math:`\Theta` of loop indices :math:`\mathbf{x}` and global invariant parameters :math:`\mathbf{g}`: + +.. math:: + \Theta_S(\mathbf{x}) = T_S \begin{pmatrix} + \vec{x}\\ + \vec{g}\\ + 1 + \end{pmatrix} + \qquad + T_S \in \mathbb{Z} ^{p \times (\text{dim}(\mathbf{x}) + \text{dim}(\mathbf{g}) + 1)} + + +Where :math:`\Theta_S(\mathbf{x})` is a p-dimensional vector representing the slowest to fastest growing indices (from left to right) when traversing the loop nest surrounding :math:`S`. For the code shown above, the original schedule defined by the loop nest in C can be retrieved by using: + +.. math:: + \Theta_S(\mathbf{x}) = \begin{pmatrix} + 1 & 0 \\ + 0 & 1 \\ + \end{pmatrix} + \begin{pmatrix} + i & j + \end{pmatrix}^T + = + \begin{pmatrix} + i & j + \end{pmatrix}^T + + +where :math:`i` and :math:`j` are respectively the slowest and fastest growing loop indices in the nest. If :math:`T_S` is a vector (resp. tensor), then :math:`\Theta_S` is a said to be one-dimensional (resp. multi-dimensional). + ++++++++++++ +Advantages ++++++++++++ + +Programs amenable to polyhedral compilation can be aggressively transformed and optimized. Most of these transformations actually boil down to the production of schedules and iteration domains that enable loop transformations promoting parallelism and spatial/temporal data locality (e.g., fusion, interchange, tiling, parallelization). + +Polyhedral compilers can also automatically go through complex verification processes to ensure that the semantics of their input program is preserved throughout this optimization phase. Note that polyhedral optimizers are not incompatible with more standard optimization techniques. In fact, it is not uncommon for these systems to be implemented as a set of LLVM passes that can be run ahead of more traditional compilation techniques [10]_. + +All in all, polyhedral machinery is extremely powerful, when applicable. It has been shown to support most common loop transformations, and has indeed achieved performance comparable to state-of-the-art GPU libraries for dense matrix multiplication [8]_. Additionally, it is also fully automatic and doesn't require any hint from programmers apart from source-code in a C-like format. + +++++++++++++ +Limitations +++++++++++++ + +Unfortunately, polyhedral compilers suffer from two major limitations that have prevented its adoption as a universal method for code generation in neural networks. + +First, the set of possible program transformations $\Omega = \{ \Theta_S ~|~ S \in \text{program} \}$ is large, and grows with the number of statements in the program as well as with the size of their iteration domain. Verifying the legality of each transformation can also require the resolution of complex integer linear programs, making polyhedral compilation very computationally expensive. To make matters worse, hardware properties (e.g., cache size, number of SMs) and contextual characteristics (e.g., input tensor shapes) also have to be taken into account by this framework, leading to expensive auto-tuning procedures [11]_. + +Second, the polyhedral framework is not very generally applicable; SCoPs are relatively common [12]_ but require loop bounds and array subscripts to be affine functions of loop indices, which typically only occurs in regular, dense computations. For this reason, this framework still has to be successfully applied to sparse -- or even structured-sparse -- neural networks, whose importance has been rapidly rising over the past few years. + +On the other hand, blocked program representations advocated by this dissertation are less restricted in scope and can achieve close to peak performance using standard dataflow analysis. + +----------------------- +Scheduling Languages +----------------------- + +Separation of concerns \cite{dijkstra82} is a well-known design principle in computer science: programs should be decomposed into modular layers of abstraction that separate the semantics of their algorithms from the details of their implementation. Systems like Halide and TVM push this philosophy one step further, and enforce this separation at the grammatical level through the use of a **scheduling language**. The benefits of this methodology are particularly visible in the case of matrix multiplication, where, as one can see below, the definition of the algorithm (Line 1-7) is completely disjoint from its implementation (Line 8-16), meaning that both can be maintained, optimized and distributed independently. + +.. code-block:: python + :linenos: + + // algorithm + Var x("x"), y("y"); + Func matmul("matmul"); + RDom k(0, matrix_size); + RVar ki; + matmul(x, y) = 0.0f; + matmul(x, y) += A(k, y) * B(x, k); + // schedule + Var xi("xi"), xo("xo"), yo("yo"), yi("yo"), yii("yii"), xii("xii"); + matmul.vectorize(x, 8); + matmul.update(0) + .split(x, x, xi, block_size).split(xi, xi, xii, 8) + .split(y, y, yi, block_size).split(yi, yi, yii, 4) + .split(k, k, ki, block_size) + .reorder(xii, yii, xi, ki, yi, k, x, y) + .parallel(y).vectorize(xii).unroll(xi).unroll(yii); + + +The resulting code may however not be completely portable, as schedules can sometimes rely on execution models (e.g., SPMD) or hardware intrinsics (e.g., matrix-multiply-accumulate) that are not widely available. This issue can be mitigated by auto-scheduling mechanisms [13]_. + ++++++++++++ +Advantages ++++++++++++ + +The main advantage of this approach is that it allows programmers to write an algorithm *only once*, and focus on performance optimization separately. It makes it possible to manually specify optimizations that a polyhedral compiler wouldn't be able to figure out automatically using static data-flow analysis. + +Scheduling languages are, without a doubt, one of the most popular approaches for neural network code generation. The most popular system for this purpose is probably TVM, which provides good performance across a wide range of platforms as well as built-in automatic scheduling mechanisms. + +++++++++++++ +Limitations +++++++++++++ + +This ease-of-development comes at a cost. First of all, existing systems that follow this paradigm tend to be noticeably slower than Triton on modern hardware when applicable (e.g., V100/A100 tensor cores w/ equal tile sizes). I do believe that this is not a fundamental issue of scheduling languages -- in the sense that it could probably be solved with more efforts -- but it could mean that these systems are harder to engineer. More importantly, existing scheduling languages generate loops whose bounds and increments cannot depend on surrounding loop indice without at least imposing severe constraints on possible schedules -- if not breaking the system entirely. This is problematic for sparse com-putations, whose iteration spaces may be irregular. + +.. table:: + :widths: 50 50 + + +-----------------------------------------------------+-----------------------------------------------------+ + | | | + |.. code-block:: C | |pic2| | + | | | + | for(int i = 0; i < 4; i++) | | + | for(int j = 0; j < 4; j++) | | + | float acc = 0; | | + | for(int k = 0; k < K[i]; k++) | | + | acc += A[i][col[i,k]]*B[k][j] | | + | C[i][j] = acc; | | + +-----------------------------------------------------+-----------------------------------------------------+ + +.. |pic2| image:: halide-iteration.png + :width: 300 + +On the other hand, the block-based program representation that we advocate for through this work allows for block-structured iteration spaces and allows programmers to manually handle load-balancing as they wish. + +-------------- +References +-------------- + +.. [1] Lattner et al., "LLVM: a compilation framework for lifelong program analysis transformation" +.. [2] Wolfe, "More Iteration Space Tiling", SC 1989 +.. [3] Darte, "On the Complexity of Loop Fusion", PACT 1999 +.. [4] Allen et al., "Automatic Loop Interchange", SIGPLAN Notices 1984 +.. [5] Ancourt et al., "Scanning Polyhedra with DO Loops", PPoPP 1991 +.. [6] Baghdadi et al., "Tiramisu: A Polyhedral Compiler for Expressing Fast and Portable Code", CGO 2021 +.. [7] Vasilache et al., "Tensor Comprehensions: Framework-Agnostic High-Performance Machine Learning Abstractions", ArXiV 2018 +.. [8] Elango et al. "Diesel: DSL for Linear Algebra and Neural Net Computations on GPUs", MAPL 2018 +.. [9] Lattner et al., "MLIR Primer: A Compiler Infrastructure for the End of Moore’s Law", Arxiv 2019 +.. [10] Grosser et al., "Polly - Performing Polyhedral Optimizations on a Low-Level Intermediate Representation", Parallel Processing Letters 2012 +.. [11] Sato et al., "An Autotuning Framework for Scalable Execution of Tiled Code via Iterative Polyhedral Compilation", TACO 2019 +.. [12] Girbal et al., "Semi-Automatic Composition of Loop Transformations for Deep Parallelism and Memory Hierarchies", International Journal of Parallel Programming 2006 +.. [13] Mullapudi et al., "Automatically scheduling halide image processing pipelines", TOG 2016 \ No newline at end of file diff --git a/docs/programming-guide/triton-c.rst b/docs/programming-guide/triton-c.rst new file mode 100644 index 000000000..789bdb268 --- /dev/null +++ b/docs/programming-guide/triton-c.rst @@ -0,0 +1,83 @@ +======================= +The Triton-C Language +======================= + +In the introduction, we stressed the importance of blocked algorithms and described their core principles in pseudo-code. To facilitate their implementation on modern GPU hardware, we present Triton-C, a single-threaded imperative kernel language in which block variables are first-class citizen. This language may be used either directly by developers familiar with C, or as an intermediate language for existing (and future) transcompilers. In this chapter, we describe its differences with C, its Numpy-like semantics and its "Single-Program, Multiple-Data" (SPMD) programming model. + +------------------- +Differences with C +------------------- + +The syntax of Triton-C is based on that of ANSI C, but was modified and extended to accomodate the semantics and programming model described in the next two subsections. These changes fall into the following categories: + ++++++++++++ +Extensions ++++++++++++ + +**Variable declarations**: Triton adds special-purpose syntax for multi-dimensional array declarations (e.g., :code:`int block[16, 16]`), which purposely differs from that of nested arrays (i.e., arrays of pointers) found in ANSI C (e.g., :code:`int block[16][16]`). Block dimensions must be constant but can also be made parametric with the use of pre-processor macros. One-dimensional blocks of integers may be initialized using ellipses (e.g., :code:`int range[16] = 0 ... 16`). + +**Primitive types**: Triton-C supports the following primitive data-types: :code:`bool`, :code:`uint8`, :code:`uint16`, :code:`uint32`, :code:`uint64`, :code:`int8`, :code:`int16`, :code:`int32`, :code:`int64`, :code:`half`, :code:`float`, :code:`double`. + +**Operators and built-in function**: The usual C operators were extended to support element-wise array operations (:code:`+`, :code:`-`, :code:`&&`, :code:`*`, etc.) and complex array operations(:code:`@` for matrix multiplication). Additionally, some built-in functions were added for concurrency (:code:`get_program_id`, :code:`atomic_add`). + +**Slicing and broadcasting**: Multi-dimensional blocks can be broadcast along any particular dimension using numpy-like slicing syntax (e.g., :code:`int array[8, 8] = range[:, newaxis]` for stacking columns). Note that, as of now, slicing blocks to retrieve sub-blocks (or scalars) is forbidden as it is incompatible with the automatic parallelization methods used by our JIT. Reductions can be achieved using a syntax similar to slicing (e.g., :code:`array[+]` for summing an array, or :code:`array[:, max]` for row-wise maximum). Currently supported reduction operators are :code:`+`, :code:`min`, :code:`max`. + +**Masked pointer dereferencement**: Block-level operations in Triton-C are "atomic", in the sense that they execute either completely or not at all. Basic element-wise control-flow for block-level operations can nonetheless be achieved using ternary operators and the *masked pointer dereferencement* operator exemplified below: + +.. code-block:: C + + // create mask + bool mask[16, 16] = ...; + // conditional addition + float x[16, 16] = mask ? a + b : 0; + // conditional load + float y[16] 16] = mask ? *ptr : 0; + // conditional store + *?(mask)ptr = y; + \end{lstlisting} + + ++++++++++++++ +Restrictions ++++++++++++++ + +The Triton project is still in its infancy. As such, there are quite a few features of ANSI C that are not supported: + +**Non-kernel functions**: Right now, all function definitions must be kernels, i.e. be preceded with the :code:`__global__` attribute. We are aware that this is a severe limitations, and the reason why it exists is because our automatic parallelization engine would not be capable of handling array parameter arguments. + +**Non-primitive types**: Non-primitive types defined with :code:`struct` and :code:`union` are currently not supported, again because it is unclear at this point how these constructs would hook into our block-level data-flow analysis passes. + +**While loops**: We just haven't had time to implement those yet. + +---------------- +Semantics +---------------- + +The existence of built-in **blocked** types, variable and operations in Triton-C offers two main benefits. First, it simplifies the structure of blocked programs by hiding important details pertaining to concurrent programming such as memory coalescing, cache management and specialized tensor instrinsics. Second, it opens the door for compilers to perform these optimizations automatically. However, it also means that programs have some kind of *block-level semantics* that does not exist in C. Though some aspects of it (e.g., the :code:`@` operator) are pretty intuitive, one in particular might be puzzling to some GPU programmers: broadcasting semantics. + ++++++++++++++++++++++++ +Broadcasting Semantics ++++++++++++++++++++++++ + + +Block variables in Triton are strongly typed, meaning that certain instructions statically require their operands to satisfy strict shape constraints. For example, a scalar may not be added to an array unless it is first appropriately broadcast. *Broadcasting semantics* (first introduced in `Numpy `_) provides two formal rules for performing these conversions automatically in the case of binary operators: (1) the shape of the lowest-dimension operand is left-padded with ones until both operands have the same dimensionality; and (2) the content of both operands is replicated as many times as needed until their shape is identical. An error is emitted if this cannot be done. + +.. code-block:: C + + int a[16], b[32, 16], c[16, 1]; + // a is first reshaped to [1, 16] + // and then broadcast to [32, 16] + int x_1[32, 16] = a[newaxis, :] + b; + // Same as above but implicitly + int x_2[32, 16] = a + b; + // a is first reshaped to [1, 16] + // a is broadcast to [16, 16] + // c is broadcast to [16, 16] + int y[16, 16] = a + c; + +------------------ +Programming Model +------------------ + +As discussed in the `CUDA documentation `_, The execution of CUDA code on GPUs is supported by an `SPMD `_ programming model in which each kernel instance is associated with an identifiable *thread-block*, itself decomposed into *warps* of 32 *threads*. The Triton programming model is similar, but each kernel is *single-threaded* -- though automatically parallelized -- and associated with a global :code:`program id` which varies from instance to instance. This approach leads to simpler kernels in which CUDA-like concurrency primitives (shared memory synchronization, inter-thread communication, etc.) do not exist. The global program ids associated with each kernel instance can be queried using the :code:`get_program_id(axis)` built-in function where :code:`0 <= axis <= 2`. This is, for example, useful to create e.g., blocks of pointers as shown in the tutorials. + diff --git a/docs/programming-guide/triton-parallel-matmul.png b/docs/programming-guide/triton-parallel-matmul.png new file mode 100644 index 0000000000000000000000000000000000000000..7b11ba2afd3980c73ab1e38fda341701af918a65 GIT binary patch literal 3115 zcmcs4i+8eVIyg$iohlyaj|Ypq4J8>02vh@>fmhbZz06O!z1$YXW9qM-6}Nr|Ht zC}3%+Hdvz+P$)Wu8yrie-Q-?v%5BpUgQY}yquGi~D zQB)uhNTpJVM3Rw_0YQ*RBoYdRGMNlP5EzCLF*P8uSo}%PM-U9jce z?$GQfpIZE^{>Z9z0N{TxKAOWTxIWr1+DGr8aV76P1HQN`bkET*G?@u`nbsoGt9-dd zPD^|xZ&v@2LlfWs`{k-bf(vUgPLIMFOr`v-hw8Fw$ zmHL5iu8cO|0av*M^R#LfCOBKKIrx!5 zT8i7aAaUBSf7_P)yVX;WqUBhYvTO^Of=qjndy0B1RmIH%g`;hjFlo zU9Tb*d}q{8C1`ZYJs;cE$@-STB65&NKLx7fwmUT-cJYNM9|(d;(cicJH_X!%2rhVj z-Y|vHWLWj^VTccQzgl0C-ICZDq5C{Ncef4H7lquMzXogS3_i1>?QQp!rvB=V)LrV4 znp??aSNX~EV9{xKWa|j}_Zc~NU(zBsetHGc<>?N4t25W!)VKA*p3-9>6PebE zm1;I?8{5yQg4Yqu>oY=kUsBy9vb*fiN-H*E{5CwOFx8Q-pK@0Jy#Mz0?=KF-ywb7Y zeir_J`ycuwJkVY)^+qVt(#SSW+QGAJ@4{}fRj94ZTCVa&YR`53iAwmO?OCZ;>3f8UW)Asqq6C{+Q_BDFe$>!^JeRH(NR2Q;1Joi{P)^_4y^C zF8kQwLi!DQ;G5bJ>hdn*3{Sf}-Qzx)>$)*%0<4wDpqu&R(VIOUg~!tYxo^GyQz-b)OXlgan>_ppG+={x}S#!F7qsLTXi2OzvDZNa-NwH3D%*U%`iep)-(ul2%d7+bzkNrd(( z75g$_v1oAW7R@pY588AC-4^ZUO{ zQhc0b>!7r77_EEwPVhT@glZJT#@KGZ<4RuRZUk}E=;wRRna3`BBr)JHSus#THU{CQB3 zRpirTzP^a`%zJ=