Ë
    g^(h‚E ã                  ó2  — d dl mZ d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl	Z	d dl
Z
d dlmZ d dlmZmZmZmZmZmZmZ d dlmZ d dlZd dlZd dlZd dlmZ d dlmZ d dlmZmZm Z  d d	l!m"Z"m#Z#m$Z$m%Z% d
dl&m'Z' ddl(m)Z)m*Z*m+Z+ ddl,m-Z- ddl.m/Z/ ddl0m1Z1m2Z2m3Z3 ddl*m4Z4m5Z5 ddl6m7Z7 ddl8m9Z9m:Z: ddl+m;Z;m<Z<m=Z= ddl>m?Z?m@Z@mAZAmBZBmCZCmDZDmEZEmFZFmGZGmHZHmIZI ddlJmKZKmLZLmMZM ddlNmOZO ddlPmQZQmRZRmSZSmTZT ddlUmVZV ddlWmXZXmYZYmZZZm[Z[ er
d dl\m]Z]m^Z^m_Z_  ejÀ                  ea«      ZbejÆ                  jÉ                  ead«      ZeejÆ                  jÉ                  ead«      ZfejÆ                  jÉ                  ead«      Zg eT«       jÐ                  Zi eg d ¢«      ZjejÖ                   G d!„ d"«      «       Zl G d#„ d$el«      Zm G d%„ d&el«      Znd4d'„Zo ed(eQeQ¬)«      Zp G d*„ d+eSep   eep   «      Zq G d,„ d-e<«      Zr ejÖ                  d.¬/«       G d0„ d1«      «       Zs G d2„ d3et«      Zuy)5é    )ÚannotationsN)ÚCounter)ÚAnyÚCallableÚGenericÚno_type_checkÚOptionalÚTYPE_CHECKINGÚUnion)ÚTypeVar)Úimmutable_dict)Ú
OrderedSet)ÚFloorDivÚIdentityÚModularIndexing)Úfree_symbol_is_typeÚ
prefix_strÚsymbol_is_typeÚSymTé   )Úcountersé   )ÚconfigÚirÚ	scheduler)Úprologue_preserves_zero_mask)Ú	code_hash)Ú	MemoryDepÚStarDepÚWeakDep)ÚIRNodeÚTritonTemplateBuffer)Ú!indexing_dtype_strength_reduction)Ú
green_textÚyellow_text)ÚBaseSchedulerNodeÚBaseSchedulingÚ	WhyNoFuse)Úcache_on_selfÚexpr_fits_within_32bitÚget_dtype_sizeÚIndentedBufferÚPlaceholderÚprefix_is_reductionÚ'set_kernel_post_grad_provenance_tracingÚsympy_index_symbolÚsympy_productÚ
sympy_subsÚunique)ÚopsÚ
OpsWrapperÚVé   )ÚBlockPatternMatcher)ÚCSEVariableÚindex_prevent_reorderingÚKernelÚPythonPrinter)ÚMultiKernel)ÚDisableReductionÚEnableReductionÚNodeScheduleMarkerÚSIMDKernelFeatures)ÚIterableÚIteratorÚSequenceÚ
perf_hintsÚscheduleÚfusion)ÚzÚyÚxÚr0_Úr1_c                  óî   ‡ — e Zd ZdZej
                  j                  ej
                  j                  dœ	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dˆ fd„Zee	e
dd„«       «       «       Zd	d„Zee	e
d
d„«       «       «       Zˆ xZS )ÚIterationRangesaÀ  
    Each range tree represents multiple sets of iteration indexing
    in a single tiled dimension in the output kernel.

    If you have two loops ranges one (4, 3, 2) and another (4, 6),
    then the range tree will be:
            4 (i0)
        3 (i1)  6 (i3)
        2 (i2)
    Where i0 is shared between both loops, but then the split into
    different indexing vars.  All loop ranges must iterate over
    the same number of elements.
    )ÚdivisorÚlengthc               ó    •— t         ‰
| �  «        || _        || _        || _        || _        || _        || _        || _        || _	        |	| _
        y ©N)ÚsuperÚ__init__ÚnameÚvar_listÚ
var_rangesÚnumelÚprefixrO   rP   ÚkernelÚroot)ÚselfrU   rV   rW   rX   rY   rZ   rO   rP   r[   Ú	__class__s             €úZ/var/www/skyplay_api_hub/venv/lib/python3.12/site-packages/torch/_inductor/codegen/simd.pyrT   zIterationRanges.__init__`   sO   ø€ ô 	‰ÑÔØˆŒ	Ø ˆŒØ$ˆŒØˆŒ
ØˆŒØˆŒØˆŒØˆŒØˆ�	ó    c                ó,   — t        | j                  «      S rR   )r.   rY   ©r\   s    r^   Úis_reductionzIterationRanges.is_reductionx   s   € ô # 4§;¡;Ó/Ð/r_   c                ó,   — t        | j                  «      S rR   )r0   rU   ra   s    r^   ÚsymbolzIterationRanges.symbol~   s   € Ü! $§)¡)Ó,Ð,r_   c                óz   — t        j                  «       D ��ci c]  \  }}||“Œ
 }}}|| j                     S c c}}w rR   )r   ÚitemsrY   )r\   ÚsymtrY   Úprefix_to_symts       r^   rg   zIterationRanges.symt�   s>   € ô <F×;KÑ;KÓ;M×N©<¨4°˜& $™,ÐNˆÑNØ˜dŸk™kÑ*Ð*ùó Os   ˜7)rU   ÚstrrV   úlist[sympy.Symbol]rW   údict[sympy.Symbol, sympy.Expr]rX   ú
sympy.ExprrY   ri   rZ   Ú
SIMDKernelr[   ÚIterationRangesRootÚreturnÚNone©ro   Úbool©ro   zsympy.Symbol)ro   r   )Ú__name__Ú
__module__Ú__qualname__Ú__doc__ÚsympyÚSÚOnerT   Úpropertyr)   r   rb   rd   rg   Ú__classcell__©r]   s   @r^   rN   rN   P   sÀ   ø„ ñð. —‘—‘Ø�w‰w�{‰{ñàðð %ðð 3ð	ð
 ðð ðð ðð "ðð 
õð0 ØØò0ó ó ó ð0ó-ð ØØò+ó ó ó ô+r_   rN   c                  ó–   ‡ — e Zd Z	 d		 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d
ˆ fd„Zdd„Zdd„Zdd„Zdd„Z	 	 	 	 dd„Zdd„Z		 	 	 	 dd„Z
ˆ xZS )rn   c          	     óÀ   •— |€i }t         ‰| �  |g i |||| ¬«       || _        i | _        || _        |r| j
                  r|	�J ‚|| _        || _        |	| _        |
| _	        y )N)rU   rV   rW   rX   rY   rZ   r[   )
rS   rT   ÚindexÚnodesÚ	pid_cacherb   Úis_loopÚ
tensor_dimÚgrid_dimÚhas_zdim)r\   rU   rX   rY   r€   rZ   r‚   rƒ   r„   r…   r†   r]   s              €r^   rT   zIterationRangesRoot.__init__Š   s„   ø€ ð ÐØˆIÜ‰ÑØØØØØØØð 	ô 	
ð ˆŒ
à=?ˆŒ
ð *3ˆŒñ ˜t×0Ò0°XÐ5EÐFÐFØˆŒà$ˆŒà ˆŒØ ˆ�r_   c                ó<   — d| j                   ›d| j                  › d�S )NzIterationRangesRoot(ú, z, ...))rU   rX   ra   s    r^   Ú__repr__zIterationRangesRoot.__repr__´   s   € Ø% d§i¡i ]°"°T·Z±Z°LÀÐGÐGr_   c                ób   — | j                   j                  «       D ]  }|j                  «        Œ y rR   )r�   ÚvaluesÚcache_clear)r\   Únodes     r^   rŒ   zIterationRangesRoot.cache_clear·   s*   € Ø—J‘J×%Ñ%Ó'ò 	ˆDØ×ÑÕñ	r_   c                ó2   — t        | j                  › d�«      S )Nr€   )r0   rY   ra   s    r^   Ú	index_symzIterationRangesRoot.index_sym»   s   € Ü! T§[¡[ M°Ð"7Ó8Ð8r_   c                ó’  — t         j                  j                  j                  ||z  | j                  «      rt        | j                  «       |«      }nt        | j                  «       ||«      }|| j                  vr¾t        | j                  › t        t         j                  j                  «      › �|||| «      }|t         j                  j                  |j                  «       <   | j                   j#                  |j                  «       «       || j$                  |j                  «       <   || j                  |<   | j                  |   S )zF
        Lookup a given RangeTreeEntry, creating it if needed
        )r6   ÚgraphÚsizevarsÚstatically_known_equalsrX   r   r�   r   r�   ÚIterationRangesEntryrY   ÚnextrZ   Úiter_vars_countÚrange_tree_nodesrd   rV   ÚappendrW   )r\   rO   rP   Úexprr�   s        r^   ÚlookupzIterationRangesRoot.lookup¾   sö   € ô �7‰7×Ñ×3Ñ3°G¸fÑ4DÀdÇjÁjÔQÜ˜DŸN™NÓ,¨gÓ6‰Dä" 4§>¡>Ó#3°W¸fÓEˆDà�t—z‘zÑ!Ü'Ø—;‘;�-¤¤Q§X¡X×%=Ñ%=Ó >Ð?Ð@ØØØØóˆDð 8<ŒA�H‰H×%Ñ% d§k¡k£mÑ4Ø�M‰M× Ñ  §¡£Ô/Ø-3ˆD�O‰O˜DŸK™K›MÑ*Ø#ˆD�J‰J�tÑØ�z‰z˜$ÑÐr_   c                óÀ   — t         j                  j                  }g }t        |«      D ](  }|j	                  | j                  ||«      «       ||z  }Œ* g t        |«      ¢S rR   )rx   ry   rz   Úreversedr˜   rš   )r\   ÚlengthsrO   ÚitervarsrP   s        r^   Úconstruct_entriesz%IterationRangesRoot.construct_entriesÕ   s]   € ô —'‘'—+‘+ˆØˆÜ˜wÓ'ò 	'ˆFØ�O‰O˜DŸK™K¨°Ó8Ô9Ø Ñ&‰Gð	'ð %”˜(Ó#Ð$Ð$r_   c                óf   — | j                  |«      D �cg c]  }|j                  «       ‘Œ c}S c c}w rR   )rŸ   rd   )r\   r�   Úes      r^   Ú	constructzIterationRangesRoot.constructß   s'   € Ø$(×$:Ñ$:¸7Ó$CÖD˜q�—‘•
ÒDÐDùÒDs   ”.c           
     óN  ‡‡‡	— |j                   D �cg c]+  }t        j                  j                  j	                  |«      ‘Œ- }}|D �cg c]!  }|sŒ|j
                  | j
                  k(  sŒ |‘Œ# }}|j                  d„ ¬«       t        j                  j                  Šg Šg Š	ˆˆˆ	fd„}|D ]v  }t        j                  j                  j                  |j                  ‰«      s8 || j                  ‰t        |j                  ‰«      «      «       |j                  Š ||«       Œx t        j                  j                  j                  | j                   ‰«      s, || j                  ‰t        | j                   ‰«      «      «       g t#        ‰«      ¢g t#        ‰	«      ¢fS c c}w c c}w )z,Figure out vars from this tree used in indexc                óˆ   — t         j                  j                  j                  | j                  t
        j                  ¬«      S )N)Úfallback)r6   r‘   r’   Ú	size_hintrO   r   Úunbacked_symint_fallback©rJ   s    r^   ú<lambda>z4IterationRangesRoot.vars_and_sizes.<locals>.<lambda>é   s/   € œ!Ÿ'™'×*Ñ*×4Ñ4Ø—	‘	¤F×$CÑ$Cð 5ó € r_   ©Úkeyc                ó˜   •— ‰j                  | j                  «       «       ‰j                  | j                  «       ‰| j                  z  Šy rR   )r˜   rd   rP   )r�   rO   Ú
index_varsÚsizess    €€€r^   Úaddz/IterationRangesRoot.vars_and_sizes.<locals>.addñ   s5   ø€ à×Ñ˜dŸk™k›mÔ,Ø�L‰L˜Ÿ™Ô%Ø §¡Ñ+‰Gr_   )Úfree_symbolsr6   rZ   r—   ÚgetrY   Úsortrx   ry   rz   r‘   r’   r“   rO   rš   r   rX   rœ   )
r\   r€   Úsr�   Únr¯   r�   rO   r­   r®   s
          @@@r^   Úvars_and_sizesz"IterationRangesRoot.vars_and_sizesâ   sO  ú€ ð <A×;MÑ;MÖN°a”—‘×*Ñ*×.Ñ.¨qÕ1ÐNˆÐNØ!ÖC�q¢Q¨1¯8©8°t·{±{Ó+B’ÐCˆÐCØ�
‰
ñð 	ô 	
ô
 —'‘'—+‘+ˆØˆ
Øˆö	,ð ò 	ˆDÜ—7‘7×#Ñ#×;Ñ;¸D¿L¹LÈ'ÔRá�D—K‘K ¬°$·,±,ÀÓ)HÓIÔJØŸ,™,�Ù��Ið	ô �w‰w×Ñ×7Ñ7¸¿
¹
ÀGÔLá�—‘˜G¤X¨d¯j©j¸'Ó%BÓCÔDà&”˜*Ó%Ð&Ð(:¬(°5«/Ð(:Ð:Ð:ùò7 OùÚCs   ’0FÁF"ÁF"Á*F"rR   )rU   ri   rX   rl   rY   ri   r€   ÚintrZ   rm   r‚   úOptional[dict[str, str]]rƒ   rr   r„   úOptional[int]r…   r¸   r†   rr   ro   rp   ©ro   ri   ©ro   rp   rs   )rO   rl   rP   rl   ro   r”   )r�   úlist[sympy.Expr]ro   zlist[IterationRangesEntry])r�   r»   ro   rj   )r€   rl   ro   z+tuple[list[sympy.Symbol], list[sympy.Expr]])rt   ru   rv   rT   r‰   rŒ   r�   rš   rŸ   r¢   rµ   r|   r}   s   @r^   rn   rn   ‰   s¿   ø„ ð /3ð(!àð(!ð ð(!ð ð	(!ð
 ð(!ð ð(!ð ,ð(!ð ð(!ð "ð(!ð  ð(!ð ð(!ð 
õ(!óTHóó9ó ð.%Ø'ð%à	#ó%óEð;Øð;à	4÷;r_   rn   c                  óp   ‡ — e Zd Z	 	 	 	 	 	 	 	 	 	 	 	 d	ˆ fd„Zd
d„Zdd„Zdd„Zd
d„Zdd„Zdd„Z	dd„Z
ˆ xZS )r”   c                ó$  •— t         ‰| �  ||j                  |z  |j                  |j                  |j
                  |||j                  |j                  ¬«	       || _         t        j                  d «      | j                  «      | _        || _        y )N)	rU   rX   rV   rW   rY   rO   rP   rZ   r[   )rS   rT   rX   rV   rW   rY   rZ   r[   ÚparentÚ	functoolsÚ	lru_cacheÚ_codegenÚcodegenr™   )r\   rU   rO   rP   r™   r¾   r]   s         €r^   rT   zIterationRangesEntry.__init__  s~   ø€ ô 	‰ÑØØ—,‘, Ñ'Ø—_‘_Ø×(Ñ(Ø—=‘=ØØØ—=‘=Ø—‘ð 	ô 
	
ð ˆŒØ0”y×*Ñ*¨4Ó0°·±Ó?ˆŒØˆ�	r_   c                óŠ   — d| j                   › d| j                  › d| j                  › d| j                  › d| j                  › d�S )NzIterationRangesEntry(rˆ   ú))rU   rO   rP   r™   rW   ra   s    r^   r‰   zIterationRangesEntry.__repr__  sH   € Ø& t§y¡y k°°D·L±L°>ÀÀDÇKÁKÀ=ÐPRÐSW×S\ÑS\ÐR]Ð]_Ð`d×`oÑ`oÐ_pÐpqÐrÐrr_   c                óL   ‡— ˆfd„| _         d„ | j                   _        ‰| _        y )Nc                 ó   •— ‰ S rR   © )rU   s   €r^   r©   z/IterationRangesEntry.set_name.<locals>.<lambda>   s   ø€ ˜t€ r_   c                  ó   — y rR   rÇ   rÇ   r_   r^   r©   z/IterationRangesEntry.set_name.<locals>.<lambda>!  ó   � r_   )rÂ   rŒ   rU   )r\   rU   s    `r^   Úset_namezIterationRangesEntry.set_name  s   ø€ Û#ˆŒÙ#/ˆ�‰Ô Øˆ�	r_   c                ó8   — | j                   j                  «        y rR   )rÂ   rŒ   ra   s    r^   rŒ   z IterationRangesEntry.cache_clear$  s   € Ø�‰× Ñ Õ"r_   c                óX   — t         j                  j                  | «       | j                  S rR   )r6   rZ   Úcodegen_iteration_ranges_entryrU   ra   s    r^   rÁ   zIterationRangesEntry._codegen'  s   € Ü	�‰×/Ñ/°Ô5Ø�y‰yÐr_   c                óÚ  — g }t        | j                  t        j                  «      r|S t        | j                  t        t
        f«      sJ t        | j                  «      «       ‚| j                  j                  dd  D ]l  }t        |t        j                  t        j                  f«      rŒ.|j                  }t        |«      dkD  sŒIt        d„ |D «       «      sŒ\|j                  |«       Œn |S )Nr7   r   c              3  óP   K  — | ]  }t        |t        j                  «      –— Œ  y ­wrR   )r   r   ÚSIZE©Ú.0r³   s     r^   ú	<genexpr>z8IterationRangesEntry.precomputed_args.<locals>.<genexpr>4  s    è ø€ ò ,Ø56”N 1¤d§i¡i×0ñ,ùó   ‚$&)Ú
isinstancer™   rx   ÚSymbolr   r   ÚtypeÚargsÚIntegerr°   ÚlenÚallr˜   )r\   Úprecomputed_argsÚargÚsymbolss       r^   rÜ   z%IterationRangesEntry.precomputed_args+  s¾   € à-/ÐÜ�d—i‘i¤§¡Ô.Ø#Ð#Ü˜$Ÿ)™)¤h´Ð%@ÔAÐRÄ4ÈÏ	É	Ã?ÓRÐAØ—9‘9—>‘> ! "Ð%ò 	1ˆCÜ˜c¤E§M¡M´5·<±<Ð#@ÕAØ×*Ñ*�Ü�w“< !Ó#¬ñ ,Ø:Aô,õ )ð %×+Ñ+¨CÕ0ð	1ð  Ðr_   c                ó,   — t        | j                  «      S rR   )ÚhashrU   ra   s    r^   Ú__hash__zIterationRangesEntry.__hash__:  s   € Ü�D—I‘I‹Ðr_   c                óX   — t        |t        «      sJ ‚| j                  |j                  k(  S rR   )rÕ   r”   rU   )r\   Úothers     r^   Ú__eq__zIterationRangesEntry.__eq__=  s&   € Ü˜%Ô!5Ô6Ð6Ð6Ø�y‰y˜EŸJ™JÑ&Ð&r_   )rU   ri   rO   rl   rP   rl   r™   rl   r¾   rN   ro   rp   r¹   )rU   ri   ro   rp   rº   )ro   r»   ©ro   r¶   )rã   Úobjectro   rr   )rt   ru   rv   rT   r‰   rÊ   rŒ   rÁ   rÜ   rá   rä   r|   r}   s   @r^   r”   r”     sf   ø„ ðàðð ðð ð	ð
 ðð  ðð 
õó.sóó
#óó ó÷'r_   r”   c                ó€   — | t        d«      k(  ry| t        d«      k(  ryt        j                  | «      ryt        | «      S )NÚinfzfloat("inf")z-infzfloat("-inf")zfloat("nan"))ÚfloatÚmathÚisnanÚrepr)Úvalues    r^   Úconstant_reprrî   B  s9   € Ø”�e“ÒØØ	”%˜“-Ò	ØÜ	�‰�EÔ	ØÜ�‹;Ðr_   ÚCSEVariableType)ÚboundÚdefaultc                  óÆ  ‡ — e Zd ZU dZeZded<   ded<   dZded<   ded	<   	 	 	 d5	 	 	 	 	 	 	 	 	 	 	 d6ˆ fd
„Ze	e
ed7d„«       «       «       Zd8d„Ze	d9d„«       Zd:d„Z	 	 	 	 	 	 	 	 	 	 	 	 d;d„Zd<d„Zd=d„Zd>d„Zd:d„Zd:d„Zd?d„Zd7d„Zd@d„ZdAd„Zd9d„ZdBd„Z	 	 	 	 	 	 dCd„Z	 	 	 	 	 	 dCd„ZdDd„ZdEd„Ze 	 	 	 	 	 	 dFd„«       Z!e"e#jH                  jJ                  f	 	 	 	 	 	 	 dGd „«       Z&	 	 	 	 dHd!„Z'e"	 	 	 	 	 	 dId"„«       Z(dJd#„Z)dJd$„Z*dKd%„Z+	 	 	 	 dBd&„Z,dLdMd'„Z-dNd(„Z.dOd)„Z/dPdQd*„Z0e1jd                  	 	 	 	 	 	 dRd+„«       Z3dSd,„Z4e d-„ «       Z5d.„ Z6d/„ Z7d0„ Z8d1„ Z9d2„ Z:d3„ Z;dTd4„Z<ˆ xZ=S )Urm   zo
    Common base class for Triton/Halide codegen which both use flattened indexing rather than loop nests.
    zCallable[[sympy.Expr], str]ÚsexprÚkexprFrr   Úallow_block_ptrri   Úkernel_namec                óÌ  •‡ — |€i }t         ‰	‰ �  «        |‰ _        |j                  «       ‰ _        t        «       ‰ _        t        «       ‰ _        |j                  «       D ��ci c]/  \  }}|t        j                  j                  j                  |«      “Œ1 c}}‰ _        g ‰ _        i ‰ _        t!        j"                  «       ‰ _        |j'                  «       ‰ _        |�|n‰ j+                  «       ‰ _        |�|n‰ j/                  «       ‰ _        ‰ j3                  «       ‰ _        d ‰ _        t9        j:                  d «      dˆ fd„«       }|‰ _        ‰ j?                  |«       y c c}}w )Nc                óÜ   •— t         j                  j                  j                  | ‰j	                  «       «      } ‰j
                  D ]  }‰j                  | |«      } Œ ‰j                  | «      S rR   )r6   r‘   r’   Úsimplify_with_rangesrW   Úrange_treesÚcombine_contiguous_dimsÚcombine_modular_indexing_pairs)r€   Útreer\   s     €r^   Úsimplify_indexingz.SIMDKernel.__init__.<locals>.simplify_indexing}  sb   ø€ ä—G‘G×$Ñ$×9Ñ9¸%ÀÇÁÓARÓSˆEØ×(Ñ(ò B�Ø×4Ñ4°U¸DÓA‘ðBð ×6Ñ6°uÓ=Ð=r_   )r€   rl   ) rS   rT   ÚfeaturesÚget_mutationsÚ	mutationsr,   ÚbodyÚindexing_coderf   r6   r‘   r’   ÚsimplifyÚnumelsrú   r—   Ú	itertoolsÚcountr–   rb   Úinside_reductionÚ should_use_cooperative_reductionÚcooperative_reductionÚshould_use_persistent_reductionÚpersistent_reductionÚwant_no_x_dimÚno_x_dimr   r¿   rÀ   rþ   Úinitialize_range_tree)
r\   Útilingrÿ   r‚   Úoverride_persistent_reductionÚoverride_cooperative_reductionrY   Úvalrþ   r]   s
   `        €r^   rT   zSIMDKernel.__init__Y  sK  ù€ ð ÐØˆIÜ‰ÑÔØ ˆŒØ!×/Ñ/Ó1ˆŒÜ"Ó$ˆŒ	Ü+Ó-ˆÔàFLÇlÁlÃn÷
Ù7B°v¸sˆF”A—G‘G×$Ñ$×-Ñ-¨cÓ2Ñ2ó
ˆŒð 79ˆÔØJLˆÔÜ(Ÿ™Ó0ˆÔØ (× 5Ñ 5Ó 7ˆÔð .Ð9ñ +à×6Ñ6Ó8ð 	Ô"ð -Ð8ñ *à×5Ñ5Ó7ð 	Ô!ð
 ×*Ñ*Ó,ˆŒØ(,ˆŒô 
×	Ñ	˜TÓ	"ô	>ó 
#ð	>ð "3ˆÔØ×"Ñ" 9Õ-ùó=
s   Á"4E c                ó:   — t        d„ | j                  D «       «      S )Nc              3  ó2   K  — | ]  }t        |«      –— Œ y ­wrR   )r.   )rÒ   rY   s     r^   rÓ   z0SIMDKernel.num_reduction_dims.<locals>.<genexpr>Œ  s   è ø€ ÒI°6Ô& v×.ÑIùó   ‚)Úsumr  ra   s    r^   Únum_reduction_dimszSIMDKernel.num_reduction_dimsˆ  s   € ô ÑI¸T¿[¹[ÔIÓIÐIr_   c                ó   — t         ‚rR   ©ÚNotImplementedError)r\   Údtypes     r^   Údtype_to_strzSIMDKernel.dtype_to_strŽ  ó   € Ü!Ð!r_   c                óT   — | j                  | j                  j                  «       «      S rR   )r  rÿ   Úselect_index_dtypera   s    r^   Úindex_dtypezSIMDKernel.index_dtype‘  s    € à× Ñ  §¡×!AÑ!AÓ!CÓDÐDr_   c                 ó   — y©NFrÇ   ra   s    r^   r  zSIMDKernel.want_no_x_dim•  ó   € Ør_   c                ó°  ‡— t        ˆfd„t        D «       «      }| xs | }d	d„}g d¢}	ddg}
|r|
}n
|r|	}n|	|
z   } |||«      } ||	t        «      }g }t        |«      D ]s  \  }}t        |«      }|j	                  |«      }|j	                  |«      }|€|n|}|j                  t        |› d�‰|   ||| ||xr | j                   ||d‰v ¬«
      «       Œu |S )
Nc              3  ó,   •K  — | ]  }|‰v sŒ|–— Œ y ­wrR   rÇ   )rÒ   rY   r  s     €r^   rÓ   z3SIMDKernel.construct_range_trees.<locals>.<genexpr>   s   øè ø€ ò %
Ø°¸6Ò1AŒFñ%
ùó   ƒ	�c                ó`   ‡— t        ˆfd„| D «       «      D ��ci c]  \  }}||“Œ
 c}}S c c}}w )Nc              3  ó,   •K  — | ]  }|‰v sŒ|–— Œ y ­wrR   rÇ   )rÒ   r  Úmasks     €r^   rÓ   zOSIMDKernel.construct_range_trees.<locals>.filtered_index_map.<locals>.<genexpr>§  s   øè ø€ Ò2U¸3ÈÐPTÊ´3Ñ2Uùr'  )Ú	enumerate)Úseqr*  Úidxr  s    `  r^   Úfiltered_index_mapz<SIMDKernel.construct_range_trees.<locals>.filtered_index_map¥  s3   ø€ ä)2Ó2UÀ#Ô2UÓ)U÷Ù%˜S #��S‘óð ùó s   ™*)rJ   rI   rH   rK   rL   r€   rH   )r‚   rƒ   r„   r…   r†   )ro   zdict[Any, int])r   Úall_prefixesr+  r.   r±   r˜   rn   r  )r\   r‚   r  rb   r  r  Úactive_prefixesÚno_r_dimr.  Ú	grid_dimsÚreduction_dimsÚtensor_dimsÚtensor_dim_mapÚgrid_dim_maprú   ÚirY   r„   r…   r€   s       `               r^   Úconstruct_range_treesz SIMDKernel.construct_range_trees˜  s#  ø€ ô %ó %
Ü!-ô%
ó 
ˆð (Ð'Ò;¨|Ð+;ˆó	ò
 $ˆ	Ø ˜ˆÙØ(‰KÙØ#‰Kà# nÑ4ˆKñ ,¨K¸ÓIˆÙ)¨)´\ÓBˆàˆÜ" ?Ó3ò 	‰IˆAˆvÜ.¨vÓ6ˆLØ'×+Ñ+¨FÓ3ˆJØ#×'Ñ'¨Ó/ˆHØ!Ð)‘A¨xˆEØ×ÑÜ#Ø�h˜eÐ$Ø˜6‘NØØØØ'Ø(ÒJ°×1JÑ1JÐ-JØ)Ø%Ø  F˜]ôõð	ð& Ðr_   c                óÐ   — | j                  || j                  | j                  j                  «       | j                  | j
                  «      }| j                  j                  |«       y rR   )r8  r  rÿ   rb   r  r  rú   Úextend)r\   r‚   rú   s      r^   r  z SIMDKernel.initialize_range_treeÎ  sR   € Ø×0Ñ0ØØ×!Ñ!Ø�M‰M×&Ñ&Ó(Ø�K‰KØ�M‰Mó
ˆð 	×Ñ×Ñ Õ,r_   c                 ó   — y)zr
        Hook called right before codegen with every index that will be
        used in the fused kernel.
        NrÇ   )r\   Úindicess     r^   Úfinalize_indexingzSIMDKernel.finalize_indexingØ  rÉ   r_   c                óv   — | j                   }d| _         	 | j                  |||«      || _         S # || _         w xY wr#  )r  Ústore)r\   rU   r€   rí   Úpriors        r^   Ústore_reductionzSIMDKernel.store_reductionÞ  s;   € Ø×%Ñ%ˆØ %ˆÔð	*Ø—:‘:˜d E¨5Ó1à$)ˆDÕ!ø EˆDÕ!ús   •/ ¯	8c                 ó   — yr#  rÇ   ra   s    r^   r	  z+SIMDKernel.should_use_cooperative_reductionæ  r$  r_   c                 ó   — yr#  rÇ   ra   s    r^   r  z*SIMDKernel.should_use_persistent_reductioné  r$  r_   c                ót   — t        t        j                  j                  d„ | j                  D «       «      «      S )Nc              3  óP   K  — | ]  }|j                   j                  «       –— Œ  y ­wrR   )rW   rf   ©rÒ   rý   s     r^   rÓ   z(SIMDKernel.var_ranges.<locals>.<genexpr>î  s"   è ø€ ò *Ø,0�—‘×%Ñ%×'ñ*ùrÔ   )Údictr  ÚchainÚfrom_iterablerú   ra   s    r^   rW   zSIMDKernel.var_rangesì  s4   € ÜÜ�O‰O×)Ñ)ñ *Ø48×4DÑ4Dô*ó ó
ð 	
r_   c                ó:   — t        d„ | j                  D «       «      S )Nc              3  óJ   K  — | ]  }t        |j                  d u«      –— Œ y ­wrR   )r¶   r„   rF  s     r^   rÓ   z0SIMDKernel.triton_tensor_ndim.<locals>.<genexpr>ô  s   è ø€ ÒQ¸”3�t—‘¨dÐ2×3ÑQùs   ‚!#)r  rú   ra   s    r^   Útriton_tensor_ndimzSIMDKernel.triton_tensor_ndimó  s   € ÜÑQÀ×@PÑ@PÔQÓQÐQr_   c                ó^   — dg| j                  «       z  }d||<   ddj                  |«      › d�S )Nrp   ú:ú[rˆ   ú])rL  Újoin)r\   r7  r®   s      r^   Úindexing_size_strzSIMDKernel.indexing_size_strö  s9   € Ø�˜4×2Ñ2Ó4Ñ4ˆØˆˆa‰Ø�4—9‘9˜UÓ#Ð$ AÐ&Ð&r_   c                óð   — dg| j                  «       z  }| j                  D ]R  }|j                  €Œ|j                  r| j                  sŒ)|j
                  j                  «       › d�||j                  <   ŒT |S )NÚ1ÚBLOCK)rL  rú   r„   rb   r  rY   Úupper)r\   r®   rý   s      r^   Údense_size_listzSIMDKernel.dense_size_listû  sv   € Ø�˜×/Ñ/Ó1Ñ1ˆØ×$Ñ$ò 	GˆDØ�‰Ð&Øà×$Ò$¨×(=Ó(=Ø,0¯K©K×,=Ñ,=Ó,?Ð+@ÀÐ)F��d—o‘oÒ&ð	Gð ˆr_   c                óL   — | j                  «       }ddj                  |«      › d�S )NrO  rˆ   rP  )rW  rQ  ©r\   r®   s     r^   Údense_size_strzSIMDKernel.dense_size_str  s)   € Ø×$Ñ$Ó&ˆØ�4—9‘9˜UÓ#Ð$ AÐ&Ð&r_   c                óø  — t        |t        «      s|S |j                  d   }| j                  j	                  |«      x}€|S t        |||j                  i«      }t        j                  j                  j                  |«      }t        ||j                  j                  «       |j                  j                  t        j                  j                   |j                  j"                  «      j%                  «       i«      S )Nr   )rÕ   r   rØ   r—   r±   r2   r™   r6   r‘   r’   rü   r[   r�   rš   rx   ry   rz   rX   rd   )r\   r€   rJ   Ú	tree_nodeÚ	new_indexs        r^   rü   z)SIMDKernel.combine_modular_indexing_pairs	  sÂ   € Ü˜%¤Ô1ØˆLØ�J‰J�q‰MˆØ×.Ñ.×2Ñ2°1Ó5Ð5ˆIÐ>ØˆLÜ˜u q¨)¯.©.Ð&9Ó:ˆ	Ü—G‘G×$Ñ$×CÑCÀIÓNˆ	äØà—‘×(Ñ(Ó*¨I¯N©N×,AÑ,AÜ—G‘G—K‘K §¡×!5Ñ!5ó-ç‘&“(ðó
ð 	
r_   c                ó¾   — t         j                  j                  j                  |«      x}r!|\  }}t	        | j                  ||«      |«      S | j                  ||«      S rR   )r6   r‘   r’   Úexpand_floor_divr   Ú_combine_contiguous_dims)r\   r€   rý   Ú
expand_resr]  Údenominators         r^   rû   z"SIMDKernel.combine_contiguous_dims  s[   € ô Ÿ™×)Ñ)×:Ñ:¸5ÓAÐAˆ:ÐAØ%/Ñ"ˆI�{Ü˜D×9Ñ9¸)ÀTÓJÈKÓXÐXà×0Ñ0°¸Ó=Ð=r_   c                ó–  — t        |t        j                  t        j                  f«      r|S |j	                  |«      \  }}t        |«      dk  r|S t        j                  j                  j                  ||t        |g||«      «      \  }}}||k(  r|S |j                  |«      }t        |t        t        | ||«      «      «      «      }	|	S )zI
        More aggressive simplification to merge contiguous dims
        r7   )rÕ   rx   rÙ   rÖ   rµ   rÚ   r6   r‘   r’   Ú_simplify_loopsr:   r¢   r2   rG  Úzip)
r\   r€   rý   r­   r®   Ú	new_sizesÚreindexÚ_pruneÚnew_index_varsr]  s
             r^   r`  z#SIMDKernel._combine_contiguous_dims$  s¹   € ô �eœeŸm™m¬U¯\©\Ð:Ô;ØˆLØ ×/Ñ/°Ó6Ñˆ
�EÜˆu‹:˜Š?ØˆLÜ%&§W¡W×%5Ñ%5×%EÑ%EØ˜Ô7¸¸ÀÈUÓSó&
Ñ"ˆ	�7˜Fð ˜ÒØˆLØŸ™¨	Ó2ˆÜ˜u¤d¬3¨z¹7À>Ó;RÓ+SÓ&TÓUˆ	ØÐr_   c                ó”   ‡ ‡— ‰ j                   d   j                  xs ‰ j                  Št        j                  ˆ ˆfd„«       } |«       S )Néÿÿÿÿc               3  óî   •K  — ‰ j                   j                  «       s‰ j                  rJ ‚d –— y ‰r‰ j                  «        d‰ _        	 d –— ‰r‰ j                  «        d‰ _        y # d‰ _        w xY w­w)NFT)rÿ   rb   r  Úcodegen_body)r\   Úshould_flushs   €€r^   Úctxz)SIMDKernel.disable_reduction.<locals>.ctx;  sl   øè ø€ à—=‘=×-Ñ-Ô/Ø×0Ò0Ð0Ð0ÛØÙð ×!Ñ!Ô#Ø$)ˆDÔ!ð-ÛÙà×%Ñ%Ô'à(,�Õ%ø¨�Õ%üs   ƒAA5ÁA) Á!A5Á)	A2Á2A5)rú   rƒ   r
  Ú
contextlibÚcontextmanager)r\   ro  rn  s   ` @r^   Údisable_reductionzSIMDKernel.disable_reduction8  sE   ù€ Ø×'Ñ'¨Ñ+×3Ñ3ÒQ°t×7QÑ7Qˆä	×	"Ñ	"ô	-ó 
#ð	-ñ$ ‹uˆr_   c                óÄ   — t        |«      t        | j                  «      k(  sJ ‚t        || j                  «      D ��cg c]  \  }}|j                  |«      ‘Œ c}}S c c}}w rR   )rÚ   rú   re  r¢   )r\   r�   rP   Úrangess       r^   Ú
set_rangeszSIMDKernel.set_rangesP  s]   € Ü�7‹|œs 4×#3Ñ#3Ó4Ò4Ð4Ð4ô #& g¨t×/?Ñ/?Ó"@÷
á�˜ð ×Ñ˜VÕ$ó
ð 	
ùó 
s   ½Ac                ó$  ‡‡‡‡— t        d„ |D «       «      r| D �cg c]  }g ‘Œ c}g fS t        j                  j                  Š| D �cg c]  }g ‘Œ c}Š| D �cg c]  }‰j	                  |«      ‘Œ c}Št        j                  «       Šd
ˆˆˆˆfd„}	 	 	 	 	 	 	 	 dd„}g }d}|D �];  }	g }
|	D �]  }‰j                  |d«      r|
j                  d„ «       Œ)|t        ‰«      k  r>‰j                  ‰|   d«      r)|dz  }|t        ‰«      k  r‰j                  ‰|   d«      rŒ)|dz   t        ‰«      k  ro‰j                  |‰|   «      rZ‰j                  |‰|   «      st        ‚‰|   }t        |‰|   «      }|
j                   || |||«       ||dz   |«      «      «       Œõ|
j                  t        j                   |||«      «      «       �Œ" |j                  |
«       �Œ> t        d„ ‰D «       «      sJ d‰› d	|› �«       ‚‰|fS c c}w c c}w c c}w )Nc              3  ó8   K  — | ]  }t        |«      d k(  –— Œ y­w©r   N)rÚ   )rÒ   rP   s     r^   rÓ   z5SIMDKernel._split_iteration_ranges.<locals>.<genexpr>^  s   è ø€ Ò6 FŒs�6‹{˜aÕÑ6ùó   ‚c                ó¾   •— ‰j                  |«      }‰j                  ‰|    |«      st        ‚t        ‰|    |«      ‰| <   ‰|    j	                  |«       t        ‰«      S rR   )r  Ústatically_known_multiple_ofÚ	CantSplitr   r˜   r•   )r7  r™   Ú
new_rangesÚ	remainingÚsvÚ	var_counts     €€€€r^   Ú	add_rangez5SIMDKernel._split_iteration_ranges.<locals>.add_rangef  sZ   ø€ Ø—;‘;˜tÓ$ˆDØ×2Ñ2°9¸Q±<ÀÔFÜ�ä# I¨a¡L°$Ó7ˆI�a‰LØ�q‰M× Ñ  Ô&Ü˜	“?Ð"r_   c                ó   ‡ ‡‡— dˆˆˆ fd„}|S )Nc                ó    •— ‰| ‰   z  | ‰   z   S rR   rÇ   )Ú	flat_varsÚidx1Úidx2Úsizes    €€€r^   ÚgetterzISIMDKernel._split_iteration_ranges.<locals>.make_combined.<locals>.getterr  s   ø€ Ø˜i¨™oÑ-°	¸$±Ñ?Ð?r_   )r„  r»   ro   rl   rÇ   )r‡  r…  r†  rˆ  s   ``` r^   Úmake_combinedz9SIMDKernel._split_iteration_ranges.<locals>.make_combinedo  s   ú€ ÷@ð ˆMr_   r   r7   c                ó6   — t         j                  j                  S rR   )rx   ry   ÚZero)Ú_s    r^   r©   z4SIMDKernel._split_iteration_ranges.<locals>.<lambda>}  s   € ´E·G±G·L±L€ r_   c              3  ót   K  — | ]0  }t         j                  j                  j                  |«      d k(  –— Œ2 y­w)r7   N©r6   r‘   r’   r¦   rÑ   s     r^   rÓ   z5SIMDKernel._split_iteration_ranges.<locals>.<genexpr>ž  s*   è ø€ ÒI¸!”1—7‘7×#Ñ#×-Ñ-¨aÓ0°AÕ5ÑIùs   ‚68zfailed to set ranges ú )r7  r¶   r™   rl   ro   r¶   )r‡  rl   r…  r¶   r†  r¶   ro   z(Callable[[list[sympy.Expr]], sympy.Expr])rÛ   r6   r‘   r’   r  r  r  r“   r˜   rÚ   Ústatically_known_gtr{  r|  r   ÚoperatorÚ
itemgetter)Úgroupsr�   ÚgrouprŒ  Úgr�  r‰  Úreturn_getters_groupsÚcurrent_groupÚlength_groupÚreturn_gettersr‡  Úsize1Úsize2r}  r~  r  r€  s                 @@@@r^   Ú_split_iteration_rangesz"SIMDKernel._split_iteration_rangesW  s]  û€ ô Ñ6¨gÔ6Ô6Ø$*Ö+˜5’BÒ+¨RÐ/Ð/ä�W‰W×ÑˆØ:@Ö-A°QªbÒ-Aˆ
Ø-3Ö4¨�R—[‘[ •^Ò4ˆ	Ü—O‘OÓ%ˆ	÷	#ð 	#ð	Øð	Ø$'ð	Ø/2ð	à5ó	ð !#ÐØˆØ#ó #	9ˆLØˆNØ$ó  �Ø×-Ñ-¨d°AÔ6Ø"×)Ñ)Ñ*@ÔAØà#¤c¨)£nÒ4¸×9SÑ9SØ˜mÑ,Øô:ð
 " QÑ&�Mð $¤c¨)£nÒ4¸×9SÑ9SØ˜mÑ,Øõ:ð ! 1Ñ$¤s¨9£~Ò5¸"×:PÑ:PØ˜) MÑ2ô;ð ×:Ñ:Ø˜i¨Ñ6ôô (˜Ø% mÑ4�EÜ$ T¨9°]Ñ+CÓD�EØ"×)Ñ)Ù%Ø!Ù% m°UÓ;Ù% m°aÑ&7¸Ó?óõð #×)Ñ)Ü ×+Ñ+©I°mÀTÓ,JÓKöð= ðB "×(Ñ(¨Ö8ðG#	9ôJ ÑI¸yÔIÔIð 	
Ø# I ;¨a°¨yÐ9ó	
ÐIð Ð0Ð0Ð0ùòG ,ùò .BùÚ4s   ›	HÁ	HÁHc                ó  — t         j                  j                  }t        |d   «      dk(  r2|j	                  t        |«      t        |d   «      |z  «      r|d   |gf}	 | j                  ||«       y# t        $ r Y yw xY w)Nr7   r   TF)r6   r‘   r’   rÚ   r“   r1   rœ  r|  )Úclsr“  r�   Úreduction_numelr’   s        r^   Úis_compatiblezSIMDKernel.is_compatible¤  s‡   € ô —7‘7×#Ñ#ˆÜˆw�q‰z‹?˜aÒØ×,Ñ,Ü˜fÓ%Ü˜g a™jÓ)¨OÑ;ôð
 ˜q‘z OÐ#4Ð5ˆGð	Ø×'Ñ'¨°Ô8ØøÜò 	Ùð	ús   ÁA2 Á2	A>Á=A>c                ó>  — | j                   D �ci c]  }|j                  |j                  “Œ }}| j                  s0|D ]+  }t	        |«      sŒt
        j                  j                  ||<   Œ- g |j                  «       ¢}| j                  ||| j                  «      S c c}w rR   )rú   rY   rX   r  r.   rx   ry   rz   r‹   Úmap_kernel_groups_to_node_sizesru  )r\   r�   Úrtr  rY   r“  s         r^   Úsplit_and_set_rangeszSIMDKernel.split_and_set_ranges»  sŒ   € ð 15×0@Ñ0@ÖA¨"�"—)‘)˜RŸX™XÑ%ÐAˆÐAØ×$Ò$Ø ò 1�Ü& vÕ.Ü%*§W¡W§[¡[�F˜6’Nð1ð $�6—=‘=“?Ð#ˆØ×3Ñ3°F¸GÀTÇ_Á_ÓUÐUùò Bs   �Bc           
     óF  — t        |«      t        |«      k(  r!t        d„ t        ||«      D «       «      r ||Ž S | j                  ||«      \  }}g t        j
                  j                   ||Ž «      ¢}|D ��cg c]  }|D �cg c]
  } ||«      ‘Œ c}‘Œ c}}S c c}w c c}}w )a¡  
        We may want to fuse `for i0 in s0*s1` into a tiled kernel with groups (s0, s1).

        To do this we need to split up the iteration space of i0 into something like:
            for i1 in s0:
              for i2 in s1:
                i0 = i1*s1 + i2
                ....

        This function matches and resplits lengths to the groups of
        this kernel to enable tiled + non-tiled fusions.
        c              3  ó’   K  — | ]?  \  }}t         j                  j                  j                  t	        |«      |z
  «      d k(  –— ŒA y­wrx  ©r6   r‘   r’   r  r1   )rÒ   rJ   r•  s      r^   rÓ   z=SIMDKernel.map_kernel_groups_to_node_sizes.<locals>.<genexpr>Ú  s@   è ø€ ò /
á��1ô �G‰G×Ñ×%Ñ%¤m°AÓ&6¸Ñ&:Ó;¸qÕ@ñ/
ùs   ‚AA)rÚ   rÛ   re  rœ  r  rH  rI  )	rž  r“  r�   ru  r}  r–  rž   ÚfnsÚfns	            r^   r¢  z*SIMDKernel.map_kernel_groups_to_node_sizesÇ  s™   € ô& ˆw‹<œ3˜v›;Ò&¬3ñ /
ä˜G VÓ,ô/
ô ,
ñ ˜wÐ'Ð'à,/×,GÑ,GÈÐPWÓ,XÑ)ˆ
Ð)ØL”Y—_‘_×2Ñ2±:¸zÐ3JÓKÐLˆØ8M×N°¨Ö, "‘�H•Ô,ÓNÐNùÒ,ùÓNs   Á7	BÂ BÂBÂBc                ó6   — t        |t        j                  «      S rR   )r   r   ÚTMP©r\   r€   s     r^   Úis_indirect_indexingzSIMDKernel.is_indirect_indexingä  s   € ä" 5¬$¯(©(Ó3Ð3r_   c                óú  ‡— | j                  |«      rydgt        | j                  «      z  }|j                  D ]g  }|| j                  vrŒ| j                  |   }t        |j                  t        «      sJ ‚||j                  j                  xx   |j                  z  cc<   Œi t        j                  j                  j                  Št        ˆfd„t        || j                  j!                  «       «      D «       «      S )NFr7   c              3  óF   •K  — | ]  \  }} ‰|«       ‰|«      k7  –— Œ y ­wrR   rÇ   )rÒ   Ú	idx_rangeÚ
iter_ranger  s      €r^   rÓ   z,SIMDKernel.is_broadcasted.<locals>.<genexpr>ù  s,   øè ø€ ò 
á%�	˜:ñ �YÓ¡8¨JÓ#7Õ7ñ
ùs   ƒ!)r­  rÚ   r  r°   r—   rÕ   r¾   rn   r€   rP   r6   r‘   r’   r  Úanyre  r‹   )r\   r€   Úindex_numelsrd   Úentryr  s        @r^   Úis_broadcastedzSIMDKernel.is_broadcastedè  s×   ø€ à×$Ñ$ UÔ+Øà�sœS §¡Ó-Ñ-ˆØ×(Ñ(ò 	=ˆFØ˜T×2Ñ2Ñ2àØ×)Ñ)¨&Ñ1ˆEÜ˜eŸl™lÔ,?Ô@Ð@Ð@Ø˜Ÿ™×+Ñ+Ó,°·±Ñ<Ô,ð	=ô —7‘7×#Ñ#×,Ñ,ˆÜó 
ä),¨\¸4¿;¹;×;MÑ;MÓ;OÓ)Pô
ó 
ð 	
r_   c                ó´   — t        |t        «      r)ddj                  t        | j                  |«      «      › d�S | j                  | j                  |«      «      S )a�  
        Convert an index expr to a string that can be used in output code.
        e.g. a sympy expression "s2" may actually appear as "ks1" in the generated kernel.

        Index expressions often need to be passed in as arguments to the triton kernel.
        Rename_indexing and codegen_indexing keep track of the needed indices and add
        new parameters to the function signature.
        rO  rˆ   rP  )rÕ   ÚlistrQ  ÚmapÚindex_to_strrô   Úrename_indexingr¬  s     r^   r¹  zSIMDKernel.index_to_strþ  sN   € ô �eœTÔ"Ø�t—y‘y¤ T×%6Ñ%6¸Ó!>Ó?Ð@ÀÐBÐBØ�z‰z˜$×.Ñ.¨uÓ5Ó6Ð6r_   c                ón  — | j                  |«      }t        |t        j                  j                  j
                  «      }t        |j                  t        j                  «      «      s(t        |j                  t        j                  «      «      r3|j                  t        j                  j                  j
                  «      }t        |j                  t        j                  «      «      r‰|j                  t        j                  «      D ]g  }|j                  }t        |«      dkD  sŒt        d„ |D «       «      sŒ1|t        j                  j                  j                  |«      i}t        ||«      }Œi | j                  |«      }t        |t         «      s|n|j"                  d   }| j%                  |«      S )Nr   c              3  óp   K  — | ].  }t        |t        j                  t        j                  f«      –— Œ0 y ­wrR   )r   r   rÐ   ÚPRECOMPUTED_SIZErÑ   s     r^   rÓ   z.SIMDKernel.prepare_indexing.<locals>.<genexpr>  s.   è ø€ ò ,àô # 1¤t§y¡y´$×2GÑ2GÐ&H×Iñ,ùs   ‚46)rþ   r2   r6   r‘   r’   Úprecomputed_replacementsrÚ   Úatomsrx   ÚfloorÚceilingÚsubsr°   rÛ   Úlookup_precomputed_sizerÕ   r   rØ   Úcodegen_indexing)r\   r€   ÚarÞ   ÚreplacementsÚ
simp_indexs         r^   Úprepare_indexingzSIMDKernel.prepare_indexing  sG  € ð ×&Ñ& uÓ-ˆÜ˜5¤!§'¡'×"2Ñ"2×"KÑ"KÓLˆäˆu�{‰{œ5Ÿ;™;Ó'Ô(¬C°·±¼E¿M¹MÓ0JÔ,KØ—J‘JœqŸw™w×/Ñ/×HÑHÓIˆEô ˆu�{‰{œ5Ÿ=™=Ó)Ô*Ø—[‘[¤§¡Ó/ò 	<�ð Ÿ.™.�Ü�w“< !Ó#¬ñ ,à$ô,õ )ð %&¤q§w¡w×'7Ñ'7×'OÑ'OÐPQÓ'RÐ#S�LÜ& u¨lÓ;‘Eð	<ð ×+Ñ+¨EÓ2ˆ
ô )¨´XÔ>‰JÀJÇOÁOÐTUÑDVð 	ð ×$Ñ$ ZÓ0Ð0r_   c                ól  — | j                   D �cg c]  }|j                  r| j                  sŒ|‘Œ }}|rut        |«      dkD  rgt	        d„ |D «       «      }dj                  d„ |d | D «       «      d| d  k(  s"J |d | D �cg c]  }|j                  ‘Œ c}«       ‚t        |d | «      |d | |S c c}w c c}w )Nr7   c              3  ó8   K  — | ]  }|j                   d v –— Œ y­w)ÚxyzN©rY   ©rÒ   Úts     r^   rÓ   z0SIMDKernel.active_range_trees.<locals>.<genexpr>6  s   è ø€ Ò9¨a˜Ÿ™ EÔ)Ñ9ùry  Ú c              3  ó4   K  — | ]  }|j                   –— Œ y ­wrR   rÌ  rÍ  s     r^   rÓ   z0SIMDKernel.active_range_trees.<locals>.<genexpr>7  s   è ø€ Ò;¨˜1Ÿ8�8Ñ;ùs   ‚Úzyx)rú   rb   r  rÚ   r  rQ  rY   rœ   )r\   ÚreorderrÎ  Útreesr  s        r^   Úactive_range_treeszSIMDKernel.active_range_trees1  sÈ   € à×'Ñ'ö
Ø¨q¯~ª~À×AVÓAVŠAð
ˆð 
ñ ”s˜5“z A’~ÜÑ9°5Ô9Ó9ˆEØ—7‘7Ñ;¨U°6°E¨]Ô;Ó;¸uÀeÀVÀW¸~ÒMð Ø"'¨¨ -öPØ�—“òPó ÐMô % U¨6¨E ]Ó3ˆE�&�5ˆMØˆùò
ùò
Ps   �B,­B,Á?B1c                ó4  — t         j                  j                  j                  || j	                  «       «      }t        |j                  t        ¬«      D ]Á  }|| j                  v sŒi }| j                  |   j                  «       D ].  }t         j                  j                  j                  |«      ||<   Œ0 t        |«      dkD  r5t        | j                  |   j                  |«      | j                  |   _        | j                  |   j                  «        ŒÃ |S )Nrª   r   )r6   r‘   r’   rù   rW   Úsortedr°   ri   r—   rÜ   rÃ  rÚ   r2   r™   rÂ   )r\   r™   ÚsymrÆ  Úpss        r^   rÄ  zSIMDKernel.codegen_indexing=  s÷   € Ü�w‰w×Ñ×4Ñ4°T¸4¿?¹?Ó;LÓMˆÜ˜$×+Ñ+´Ô5ò 	5ˆCØ�d×+Ñ+Ò+ð  "�Ø×/Ñ/°Ñ4×EÑEÓGò T�BÜ'(§w¡w×'7Ñ'7×'OÑ'OÐPRÓ'S�L Ò$ðTä�|Ó$ qÒ(Ü6@Ø×-Ñ-¨cÑ2×7Ñ7Ø$ó7�D×)Ñ)¨#Ñ.Ô3ð ×%Ñ% cÑ*×2Ñ2Õ4ð	5ð ˆr_   c                ó   — t        d«      ‚)NzNYI: codegen_nan_checkr  ra   s    r^   Úcodegen_nan_checkzSIMDKernel.codegen_nan_checkN  s   € Ü!Ð":Ó;Ð;r_   c                ó   — t        d«      ‚)NzNYI: call_kernelr  )r\   rU   r�   s      r^   Úcall_kernelzSIMDKernel.call_kernelQ  s   € Ü!Ð"4Ó5Ð5r_   c              #  óþ   K  — | j                   }| j                  }|rt        j                  ||«      }t	        j
                  |«      }|| _         || _        	 |–— || _         || _        y# || _         || _        w xY w­w)z:Context manager to add an additional mask to tl.load/storeN)Ú
_load_maskÚ_load_otherr4   Úlogical_andr5   Ú_unwrap)r\   r*  rí   r@  Ú	prior_vals        r^   Ú
mask_loadszSIMDKernel.mask_loadsT  sy   è ø€ ð
 —‘ˆØ×$Ñ$ˆ	ÙÜ—?‘? 4¨Ó/ˆDä×!Ñ! $Ó'ˆØˆŒØ ˆÔð	)àŠJà#ˆDŒOØ(ˆDÕøð $ˆDŒOØ(ˆDÕüs   ‚AA=ÁA* ÁA=Á*A:Á:A=c                ó(  — | j                   j                  «       D ��ci c]  \  }}||j                  “Œ }}}t        ||«      }i }| j                  D ]7  }t        |j                  «      }t        ||di«      t        ||di«      z
  ||<   Œ9 |S c c}}w )a\  
        This gets the stride of the index for each of the tiling variables
        (technically, it does it at index 0)

        For example, if
        xindex = x0 + 512*x1 + 1024*r0
        x0 = (xindex//512)
        x1 = (xindex % 512)
        r0 = rindex // 1024

        this function would return
        {xindex: 512, rindex: 1024}
        r7   r   )r—   rf   r™   r2   rú   r0   rU   )	r\   r€   ÚkÚvÚindex_to_tile_indexesÚindex_in_tile_varsÚstridesÚ
range_treer³   s	            r^   Úget_strides_of_loadzSIMDKernel.get_strides_of_loadh  s¡   € ð 8<×7LÑ7L×7RÑ7RÓ7T× U©t¨q°!  A§F¡F¡Ð UÐÑ UÜ'¨Ð/DÓEÐØˆØ×*Ñ*ò 	ˆJÜ" :§?¡?Ó3ˆAÜ#Ð$6¸¸A¸Ó?Ä*Ø" Q¨ FóCñ ˆG�AŠJð	ð
 ˆùó !Vs   žBc                ó\   — t        |t        «      rt        t        | |«      «      S  | |«      S rR   )rÕ   Útupler¸  )r©  rí   s     r^   Ú_map_tuple_or_scalarzSIMDKernel._map_tuple_or_scalar€  s'   € ä�eœUÔ#Üœ˜R ›Ó(Ð(Ù�%‹yÐr_   c           	     ó  — g }t        t        | j                  j                  j	                  «       «      «      }| j                  j                  «       \  }}}}| j                  j                  «       }t        j                  j                  j                  t        | j                  j	                  «       «      «      }t        |«      D �]2  \  }}||vr|j                  d«       Œt        j                  j!                  |«      }	t        j                  j                  j                  |	«      }
|
|kD  rwt#        t$           «       }d}||   D ]M  }t'        |t(        t*        f«      r|j-                  d|› �«       |dz  }Œ3|j-                  |j.                  «       ŒO t        |«      |z  }n|
}t        j                  j1                  |«      }t3        |«      }|j                  ||z  dt5        ||k  «      z   z  «       �Œ5 t7        |«      S )a+  
        Try the best to estimate the total size (in bytes) of the
        kernel's inputs and outputs, which is used for estimating the memory
        throughput of this kernel. This information is used for checking how
        far we are from the peak memory bandwidth. It's important that
        we want to avoid overestimating the sizes of the inputs and outputs,
        because it can wrongfully give us a very large memory traffic value,
        which may be even larger than the theoretical bandwidth and thus
        become very misleading. This is particularly problematic for cases
        where we slice some inputs. In those cases, we should only count
        the size of the "slices" instead of the original inputs, because
        only the slices contribute to the real memory traffic.
        r   Úno_index_dep_r7   )rÚ   r3   rØ   Úinplace_buffersr‹   Úpython_argdefsrÿ   Úbuf_accessesr6   r‘   r’   r¦   r1   r  r+  r˜   Ú	get_numelr   r   rÕ   r   r    r¯   r€   Ú	get_dtyper+   r¶   r  )r\   ÚnbytesÚninplace_argsrŒ  Ú	call_argsró  Ú	out_numelr7  rÝ   Ú	arg_numelÚbuf_sizer<  Úno_index_dep_countÚdeprX   r  Ú
dtype_sizes                    r^   Úestimate_kernel_num_bytesz$SIMDKernel.estimate_kernel_num_bytes†  sº  € ð ˆÜœF 4§9¡9×#<Ñ#<×#CÑ#CÓ#EÓFÓGˆØ!ŸY™Y×5Ñ5Ó7Ñˆˆ9�a˜Ø—}‘}×1Ñ1Ó3ˆô —G‘G×$Ñ$×.Ñ.¬}¸T¿[¹[×=OÑ=OÓ=QÓ/RÓSˆ	Ü 	Ó*ó 	M‰FˆAˆsð ˜,Ñ&Ø—‘˜aÔ ØÜŸ™×)Ñ)¨#Ó.ˆIÜ—w‘w×'Ñ'×1Ñ1°)Ó<ˆHØ˜)Ò#ô %¤S™/Ó+�Ø%&Ð"Ø'¨Ñ,ò /�CÜ! #¬´Ð'9Ô:ØŸ™ mÐ4FÐ3GÐ$HÔIØ*¨aÑ/Ñ*àŸ™ C§I¡IÕ.ð/ô ˜G› yÑ0‘à �Ü—G‘G×%Ñ% cÓ*ˆEÜ'¨Ó.ˆJØ�M‰M˜% *Ñ,°´C¸¸MÑ8IÓ4JÑ0JÑKÖLð9	Mô: �6‹{Ðr_   c           	     ó&  — t        | j                  j                  «      dk(  rEt        | j                  j                  «      dk(  r#t        | j                  j                  «      dk(  ry| j                  j                  «       \  }}}}d}|D �]F  }t        j                  j                  |«      }|sŒ&|j                  «       }	t        |	j                  «      dk(  sŒOt        |	j                  D �
cg c]
  }
|
dk(  sŒ	|
‘Œ c}
«      dk(  rŒ|t        j                  |	j                  «      }|€|}Œ ||k7  sŒ¦t        d|› d�d|› d	|› �z   «      }t        j!                  |«       |D �cg c]m  }t        j                  j                  |«      rJt        j                  t        j                  j#                  |«      j                  «       j                  «      nd‘Œo }}|D �cg c]Z  }t        j                  j                  |«      r7t        j                  j#                  |«      j                  «       j                  nd‘Œ\ }}|D �cg c]@  }|t        j                  j$                  v rd
n|t        j                  j&                  v rdnd‘ŒB }}|D �
cg c]  }
|
j(                  ‘Œ }}
t        d|› d|› d|› �d|› d|› d�z   «      }t        j!                  |«        y t+        d|› d�«      }t        j!                  |«       yc c}
w c c}w c c}w c c}w c c}
w )zr
        Print message if the kernel have mixed layout inputs.
        Only care about 4D tensor for now.
        r7   r   Né   r   zExpected stride order z, but found stride orderr�  z for kernel Ú
GraphInputÚIntermediateBufferz  param names z
  buf names z
  strides z	
  sizes z
  sources ú
z%All the inputs for the triton kernel z have uniform layout)rÚ   rØ   Úinput_buffersÚoutput_buffersrñ  rò  r6   r‘   Útry_get_bufferÚ
get_layoutr‡  r   Úget_stride_orderÚstrider%   ÚlogÚwarningÚ
get_bufferÚgraph_inputsÚname_to_bufferrU   r$   )r\   rö   Úargdefsrø  Ú
_signaturerŒ  Úuniform_stride_orderÚarg_nameÚbufÚlayoutrJ   Ústride_orderÚmsgrU   Ústride_order_listÚ	size_listÚsource_listÚargdef_namess                     r^   Úwarn_mix_layoutzSIMDKernel.warn_mix_layoutÅ  s  € ô �—	‘	×'Ñ'Ó(¨AÒ-Ü�D—I‘I×,Ñ,Ó-°Ò2Ü�D—I‘I×-Ñ-Ó.°!Ò3ð
 à,0¯I©I×,DÑ,DÓ,FÑ)ˆ�˜J¨Ø#ÐØ!ó 0	ˆHÜ—'‘'×(Ñ(¨Ó2ˆCÙØØ—^‘^Ó%ˆFÜ�6—;‘;Ó 1Ó$ä 6§;¡;Ö9˜a°!°q³&šÒ9Ó:¸aÒ?ØÜ!×2Ñ2°6·=±=ÓA�Ø'Ð/Ø+7Ñ(Ø)¨\Ó9Ü%Ø0Ð1EÐ0FÐF^Ð_Ø˜l˜^¨<¸°}ÐEñFó�Cô —K‘K Ô$ð %.ö)ð !ô Ÿ7™7×1Ñ1°$Ô7ô ×+Ñ+ÜŸG™G×.Ñ.¨tÓ4×?Ñ?ÓA×HÑHôð "ñ	"ð)Ð%ð )ð %.ö	!ð !ô Ÿ7™7×1Ñ1°$Ô7ô Ÿ™×*Ñ*¨4Ó0×;Ñ;Ó=×BÒBà!ñ"ð!�Ið !ð %.ö#ð !ð	  ¤1§7¡7×#7Ñ#7Ñ7ñ %ð  ¤1§7¡7×#9Ñ#9Ñ9ñ 2à!ñ	"ð#�Kð #ð 5<Ö#<¨q A§F£FÐ#<�LÐ#<Ü%Ø(¨¨°nÀYÀKÈ|Ð\mÐ[nÐoØ& y k°¸k¸]È"ÐMñNó�Cô —K‘K Ô$Ùða0	ôb Ø3°K°=Ð@TÐUó
ˆô 	�‰�CÕùò[ :ùò)ùò!ùò#ùò $=s'   Ã-
K:
Ã8K:
Å"A2K?ÇALÈ?AL	Ê
Lc                ó†  — t        j                  ||d|«      }d| _        t        j                  | j                  j
                  |«      }t        j                  ||«      }d| _        t        j                  ||«      }t        j                  ||«      }t        j                  ||d|«      }t        j                  |||f«      S )Nr  FT)r4   Ú	reductionr  Ú
index_exprrÿ   rŸ  ÚtruedivÚsubÚmulr5   rá  )	r\   r  rí   Úsum_ÚrnumelÚmeanÚdxÚdx2Úm2s	            r^   Úwelford_reduce_fallbackz"SIMDKernel.welford_reduce_fallback  s›   € Ü�}‰}˜U E¨5°%Ó8ˆØ %ˆÔÜ—‘ §¡× =Ñ =¸uÓEˆÜ�{‰{˜4 Ó(ˆà $ˆÔÜ�W‰W�U˜DÓ!ˆÜ�g‰g�b˜"‹oˆÜ�]‰]˜5 %¨°Ó4ˆÜ×!Ñ! 4¨¨VÐ"4Ó5Ð5r_   c                óæ   — t        j                  ||d|«      }t        j                  ||«      }t        j                  |«      }t        j                  ||d|«      }t	        j
                  ||f«      S )NÚmaxr  )r4   r  r!  Úexpr5   rá  )r\   r  rí   Úvmaxr!  r,  Úvsums          r^   Ú prepare_softmax_twopass_fallbackz+SIMDKernel.prepare_softmax_twopass_fallback  s\   € Ü�}‰}˜U E¨5°%Ó8ˆÜ�g‰g�e˜TÓ"ˆÜ�g‰g�c‹lˆÜ�}‰}˜U E¨5°#Ó6ˆÜ×!Ñ! 4¨ ,Ó/Ð/r_   c                ó   — t         ‚rR   r  ra   s    r^   Úcodegen_kernelzSIMDKernel.codegen_kernel  r  r_   c                 ó   — y rR   rÇ   ra   s    r^   rm  zSIMDKernel.codegen_body"  ó   € Ør_   c                 ó   — y rR   rÇ   )r\   r´  s     r^   rÍ   z)SIMDKernel.codegen_iteration_ranges_entry%  r3  r_   )NNN)r  údict[str, sympy.Expr]rÿ   rA   r‚   r·   r  úOptional[bool]r  r6  ro   rp   rå   )r  ztorch.dtypero   ri   r¹   rq   )r‚   r·   r  rr   rb   rr   r  r5  r  rr   ro   úlist[IterationRangesRoot])r‚   zdict[str, str]ro   rp   )r<  úSequence[sympy.Expr]ro   rp   )rU   ri   r€   rl   rí   r9   ro   rp   )ro   rk   )r7  r¶   ro   ri   )ro   z	list[str])r€   rl   ro   rl   )r€   rl   rý   rn   ro   rl   )ro   z'contextlib.AbstractContextManager[None])r�   rl   ro   rj   )r“  úIterable[sympy.Expr]r�   úSequence[Sequence[sympy.Expr]]ro   zStuple[list[list[sympy.Expr]], list[list[Callable[[list[sympy.Expr]], sympy.Expr]]]])r“  r9  r�   r:  rŸ  rl   ro   rr   )r�   r:  ro   úlist[list[sympy.Expr]])r“  r8  r�   r:  ro   r;  )r€   rl   ro   rr   )r€   rl   ro   ri   ©F)rÒ  rr   ro   r7  )r™   rl   ro   rl   rº   rR   )rU   ri   r�   zOptional[IRNode]ro   rp   )r*  zUnion[str, OpsWrapper]rí   úUnion[int, float]ro   zIterator[str])r€   rl   ro   rk   )r´  r”   )>rt   ru   rv   rw   Úpexprró   Ú__annotations__rõ   rT   r{   r)   r   r  r  r!  r  r8  r  r=  rA  r	  r  rW   rL  rR  rW  rZ  rü   rû   r`  rr  ru  Ústaticmethodrœ  Úclassmethodrx   ry   rz   r   r¤  r¢  r­  rµ  r¹  rÈ  rÔ  rÄ  rÚ  rÜ  rp  rq  rã  rë  rî  rÿ  r  r)  r/  r1  rm  rÍ   r|   r}   s   @r^   rm   rm   O  sì  ø… ñð */€EÐ&Ó.Ø&Ó&Ø!€O�TÓ!ØÓð /3Ø8<Ø9=ð-.à%ð-.ð %ð-.ð ,ð	-.ð
 (6ð-.ð )7ð-.ð 
õ-.ð^ ØØòJó ó ó ðJó"ð òEó ðEóð4à+ð4ð ð4ð ð	4ð
 &ð4ð ð4ð 
#ó4ól-óó*óóó
óRó'ó
ó'ó
ð$>Øð>Ø':ð>à	ó>ðØðØ':ðà	óó(ó0
ð ðJ1Ø$ðJ1Ø/MðJ1ð
òJ1ó ðJ1ðX ð
 ',§g¡g§k¡kð	à$ðð 0ðð $ð	ð
 
òó ðð,
VØ5ð
Và	ó
Vð ðOà$ðOð 0ðOð
 
 òOó ðOó84ó
ó,7ð$1àð$1ð 
ó$1ôL
óó"<ô6ð ×Ñð)Ø*ð)Ø3Dð)à	ò)ó ð)ó&ð0 ñó ðò
=ò~EòN
6ò0ò"ò÷r_   rm   c                  óÎ  — e Zd ZU eZded<   d„ Zd„ ZeZeZ	d„ Z
	 	 dd„Ze	 	 	 	 	 	 dd„«       Zd d„Z	 	 	 	 d!d	„Zd
„ Zddœ	 d"d„Zd„ Z	 d#	 	 	 	 	 	 	 	 	 	 	 d$d„Zd„ Ze ej.                  d«      d%d„«       «       Ze	 	 	 	 	 	 d&d„«       Ze	 	 	 	 	 	 d'd„«       Ze	 	 	 	 	 	 	 	 d(d„«       Ze	 	 d)d„«       Zeej<                  j>                  f	 d*d„«       Z d„ Z!d+d„Z"d#d„Z#d„ Z$d„ Z%y),ÚSIMDSchedulingz	type[Any]Úkernel_typec                ó&   — t        d„ |D «       «      S )Nc              3  ó€   K  — | ]6  }t         j                  j                  j                  t	        |«      «      –— Œ8 y ­wrR   r§  rÑ   s     r^   rÓ   z*SIMDScheduling.group_fn.<locals>.<genexpr>-  s*   è ø€ ÒPÀQ”Q—W‘W×%Ñ%×.Ñ.¬}¸QÓ/?×@ÑPùs   ‚<>)rí  rY  s     r^   Úgroup_fnzSIMDScheduling.group_fn,  s   € ÜÑPÈ%ÔPÓPÐPr_   c                óŽ  ‡‡— t        |t        j                  «      st        |t        j                  «      r t        j                  j                  ||«      S |j                  \  }\  }}|j                  \  }\  ŠŠt        ||«      }|j                  «       r)|j                  «       s|j                  «       rA |d«       n8|j                  «       r(|j                  «       s|j                  «       r |d«       |j                  «       r,|j                  «       r|‰k(  xr |‰k(  }|s |d|‰|‰«       |S |j                  «       �så|j                  «       �sÔ|‰k(  r|‰k(  s�|j                  «       s |d|‰|‰«       y|j                  «       D ]`  }|j                  «       r nN|j                  «       |j                  «       z  sŒ7|j                  \  }\  }	}
||	k(  r||
k(  rŒT |d||	||
«        y t        ||fd«      D ]D  \  }}|j                  «       sŒt        |j                  «       t        «      }|s ||› d�«       |c S  | j                  |j                  «       ||«      }| j                  |j                  «       ||«      }| j                  |j                  «       |j                  «       z   ||«      }t         j"                  j$                  rVd}t'        |«      d	kD  r%t'        |«      d	kD  r||cxk(  xr |k(  nc }n||k(  }nt'        |«      d	kD  r||k(  }|s |d
|||«       yy|j                  «       sÙ|j                  «       rÉ|dk(  r‰dk7  sJ ‚|‰‰z  k(  r£t)        ˆˆfd„|j                  «       D «       «      s	 |d«       yt         j"                  j*                  r\|j                  «       sLt-        | j                  |j                  «       |«      j/                  «       «      |df‰‰dffv }|s |d«       |S y|‰k7  r |d«       |‰k(  S |j                  «       r|j                  «       rJ ‚| j1                  ||«      S )zª
        Hook called by Scheduler to determine if the Triton backend
        can fuse node1 and node2.  These nodes might already be
        FusedSchedulerNodes.
        z&Split scan cannot fuse with reductionsz1numel/rnumel mismatch (reduce) (%s, %s), (%s, %s)z5numel/rnumel mismatch (non-reduce) (%s, %s), (%s, %s)Fz:numel/rnumel mismatch prologue mismatch (%s, %s), (%s, %s))Únode1Únode2z is not TritonTemplateBufferTr   ztiling mismatch (%s, %s, %s)r7   c              3  ój   •K  — | ]*  }t         j                  ‰‰f|j                  «       «      –— Œ, y ­wrR   )rm   r   Ú
get_ranges)rÒ   r´   Únumel2Úrnumel2s     €€r^   rÓ   z*SIMDScheduling.can_fuse.<locals>.<genexpr>š  s1   øè ø€ ò àô ×,Ñ,¨f°gÐ->ÀÇÁÃ×Oñùs   ƒ03z"nodes numel/rnumel incompatibilityzinvalid tiling for reductionznodes numel incompatibility)rÕ   r   ÚForeachKernelSchedulerNodeÚcan_fuser”  r(   Úis_split_scanrb   Úis_templateÚ	get_nodesÚused_buffer_namesÚget_buffer_namesre  Úget_template_noder"   Úselect_tilingr   ÚtritonÚ tiling_prevents_pointwise_fusionrÚ   rÛ   Ú tiling_prevents_reduction_fusionrí  r‹   Úcan_fuse_horizontal)r\   rI  rJ  rŒ  Únumel1Úrnumel1ÚwhyÚreduction_can_fuser�   Ú	pro_numelÚ
pro_rnumelr´   Ú	node_nameÚis_triton_templateÚtiling1Útiling2Útiling3ÚcondÚis_reduction_tiling_validrM  rN  s                      @@r^   rP  zSIMDScheduling.can_fuse/  sO  ù€ ô �eœY×AÑAÔBÄjØ”9×7Ñ7ôG
ô ×7Ñ7×@Ñ@ÀÈÓNÐNà$Ÿ{™{ÑˆÑˆF�GØ$Ÿ{™{ÑˆÑˆF�GÜ˜˜uÓ%ˆà×ÑÔ ¨×)<Ñ)<Ô)>Ø×!Ñ!Ô#ÙÐ<Õ=Ø× Ñ Ô"¨5×+>Ñ+>Ô+@Ø×!Ñ!Ô#ÙÐ<Ô=à×ÑÔ E×$6Ñ$6Ô$8Ø!'¨6Ñ!1Ò!H°gÀÑ6HÐÙ%ÙØGØØØØôð &Ð%à×!Ñ!Õ#¨E×,>Ñ,>Õ,@Ø˜fÒ$¨°GÒ);Ø×(Ñ(Ô*ÙØOØØØØôð !ð !&§¡Ó 1ò )˜à×+Ñ+Ô-Ù!ð  $×5Ñ5Ó7¸%×:PÑ:PÓ:RÒRØ$Ø59·Z±ZÑ2˜Ñ2˜I zØ &¨)Ò 3¸À:Ó8MÙØ \Ø &Ø )Ø 'Ø *ôñ $)ð#)ô& !$ U¨E NÐ4FÓ Gò 	.‘��9Ø—=‘=•?ô *4Ø×+Ñ+Ó-Ô/Có*Ð&ñ .Ù˜y˜kÐ)EÐFÔGØ-Ò-ð	.ð ×(Ñ(¨¯©Ó):¸FÀGÓLˆGØ×(Ñ(¨¯©Ó):¸FÀGÓLˆGØ×(Ñ(Ø—‘Ó! E§O¡OÓ$5Ñ5°v¸wóˆGô �}‰}×=Ò=Ø�Ü�w“< !Ò#Ü˜7“| aÒ'Ø&¨'Ö<°WÔ<™à&¨'Ñ1™Ü˜“\ AÒ%Ø" gÑ-�DÙÙØ6ØØØô	ð !àà×!Ñ!Ô#¨×(:Ñ(:Ô(<Ø˜a’< G¨q¢LÐ0Ð0Ø˜ 'Ñ)Ò)Üô à"Ÿ_™_Ó.ôô ñ Ð<Ô=Ø ä—M‘M×BÒBØ!×-Ñ-Ô/ä05Ø×*Ñ*¨5¯?©?Ó+<¸fÓE×LÑLÓNó1ð   ˜Ø ¨!Ð,ðð1Ð-ñ 5ÙÐ:Ô;Ø4Ð4Øà˜ÒÙÐ1Ô2Ø˜VÑ#Ð#à×!Ñ!Ô#¨E×,>Ñ,>Ô,@Ð@Ð@à×'Ñ'¨¨uÓ5Ð5r_   c           
     ó–  ‡‡‡‡‡‡‡— g Št        t        j                     «       Št        t           «       Št        t           «       Šd Šˆˆfd„}ˆˆfd„}ˆfd„}ˆˆˆˆfd„}t        j
                  ˆˆˆˆfd„«       }ˆˆfd„}	|D ]©  }
|
‰v rŒ‰j                  |
«        ||
«      r? |	|
‰«      r |«       5  	 d d d «       ‰r ||
«      s‰xs t        ‰«      Šnd Š ||
«       Œ` ||
«      r" |«       5  ‰j                  |
«       d d d «       ŒŠt        d‰› d‰› d	|
j                  d
   › �«      ‚ ‰S # 1 sw Y   Œ|xY w# 1 sw Y   ŒÃxY w)Nc                ób   •— | j                   \  }\  }}|‰k(  xr |‰k(  xs |‰‰z  k(  xr |dk(  S ©Nr7   ©r”  ©r´   rŒ  Ú
node_numelÚnode_rnumelrX   r$  s       €€r^   Úfits_in_main_bodyz@SIMDScheduling.generate_node_schedule.<locals>.fits_in_main_bodyÃ  sH   ø€ Ø+,¯7©7Ñ(ˆAÑ(�
˜KØ %Ñ'ÒA¨K¸6Ñ,Aò Ø˜e f™nÑ,ÒA°ÀÑ1Aðr_   c                óN   •— | j                   \  }\  }}|‰k(  xr |dk(  xr ‰dk7  S rk  rl  rm  s       €€r^   Úfits_outside_reductionzESIMDScheduling.generate_node_schedule.<locals>.fits_outside_reductionÉ  s4   ø€ Ø+,¯7©7Ñ(ˆAÑ(�
˜KØ Ñ&ÒK¨;¸!Ñ+;ÒKÀÈ!ÁÐKr_   c                ó\   •— | j                   j                  D ]  }|j                  ‰v sŒ y y)NTF)Úread_writesÚreadsrU   )r´   ÚreadÚcurrent_loop_buffer_usages     €r^   Úexpect_improved_memory_usagezKSIMDScheduling.generate_node_schedule.<locals>.expect_improved_memory_usageÍ  s1   ø€ ØŸ™×+Ñ+ò  �Ø—9‘9Ð 9Ò9Ùð ð r_   c                óŠ  •— ‰j                  | «       ‰j                  | «       ‰j                  | j                  j                  D �cg c]  }|j
                  ‘Œ c}«       | j                  «       rŒt        | t        j                  «      rrt        | j                  t        j                  «      rNt        | j                  j                  t        j                  «      s ‰j                  | j                  «       «       y ‰j                  | j                  j                   D �cg c]  }|j
                  ‘Œ c}«       y c c}w c c}w rR   )r¯   r˜   Úupdatert  ru  rU   rb   rÕ   r   ÚSchedulerNoder�   r   ÚComputedBufferÚdataÚScanÚget_nameÚwrites)r´   rJ   rw  ÚdoneÚnode_scheduleÚnot_ready_yet_nodess     €€€€r^   Úschedule_node_in_loopzDSIMDScheduling.generate_node_schedule.<locals>.schedule_node_in_loopÓ  sÐ   ø€ Ø�H‰H�QŒKØ× Ñ  Ô#Ø%×,Ñ,¸a¿m¹m×>QÑ>QÖ-R¸¨a¯f«fÒ-RÔSð
 —‘Ô Ü˜q¤)×"9Ñ"9Ô:Ü˜qŸv™v¤r×'8Ñ'8Ô9Ü" 1§6¡6§;¡;´·±Ô8à#×'Ñ'¨¯
©
«Õ5à)×0Ñ0À!Ç-Á-×BVÑBVÖ1W¸Q°!·&³&Ò1WÕXùò .Sùò 2Xs   ÁD;Ä E c               3  óL  •K  — ‰r‰d   t         u r‰j                  «        n‰j                  t        «       ‰r1‰j	                  ‰t        «       ‰j	                  ‰dz   t         «       d Šd –— ‰j                  t         «       ‰j                  «        ‰ j                  «        y ­w)Nrk  r7   )r?   Úpopr˜   r>   ÚinsertÚclear)rw  Úmaybe_split_indexr‚  rƒ  s   €€€€r^   Úend_current_reduction_loopzISIMDScheduling.generate_node_schedule.<locals>.end_current_reduction_loopä  s�   øè ø€ ñ  ¨rÑ!2´oÑ!EØ×!Ñ!Õ#à×$Ñ$Ô%5Ô6Ù Ø×$Ñ$Ð%6Ô8HÔIØ×$Ñ$Ð%6¸Ñ%:¼OÔLØ$(Ð!ÛØ× Ñ ¤Ô1Ø×%Ñ%Ô'Ø%×+Ñ+Õ-ùs   ƒB!B$c                ó€   •— ‰dk(  ry‰| j                   z  sy|rt        |d   t        t        f«      rJ ‚t	        ‰«      S )Nr7   Frk  )Ú	ancestorsrÕ   r?   r>   rr   )r�   r‚  rƒ  r$  s     €€r^   Ú#requires_closing_previous_reductionzRSIMDScheduling.generate_node_schedule.<locals>.requires_closing_previous_reductionô  sN   ø€ Ø˜Š{ØØ&¨¯©Ò7ØÙ ¬Ø˜bÑ!¤OÔ5EÐ#Fô*ð ð ô Ð+Ó,Ð,r_   zunexpected group: (rˆ   z) != r7   )r   r   r&   ri   rp  rq  r¯   rÚ   r˜   r  r”  )r\   r�   rX   r$  rp  rr  rx  r„  rŠ  r�  r�   rw  r�  r‰  r‚  rƒ  s     ``       @@@@@r^   Úgenerate_node_schedulez%SIMDScheduling.generate_node_scheduleº  sa  þ€ Ø#%ˆÜœ)×5Ñ5Ñ6Ó8ˆô )¬™oÓ/ÐÜ$.¬s¡OÓ$5Ð!Ø+/Ðõ	õ	Lô	÷	Yô" 
×	"Ñ	"ö	.ó 
#ð	.õ	-ð ò 	ˆDØ�t‰|ØØ�H‰H�TŒNá  Ô&Ù6°t¸]ÔKÙ3Ó5ñ Ø÷ñ -Ñ5QÐRVÔ5Wà(9Ò(O¼SÀÓ=OÑ%ð )-Ð%á% dÕ+Ù'¨Ô-Ù/Ó1ñ /Ø!×(Ñ(¨Ô.÷/ð /ô *Ø)¨%¨°°6°(¸%ÀÇ
Á
È1Á¸ÐOóð ð-	ð4 Ð÷'ð ú÷/ð /ús   Â6D3Ã4D?Ä3D<	Ä?E	c                óè   — |j                  «       }t        |d„ ¬«      j                  \  }\  }}| j                  |||«      }t        j                  d|«       | j                  t        |||«      «      S )zK
        Given a set of pre-fused nodes, generate a Triton kernel.
        c                ó4   — t        | j                  «       «      S rR   ©r¶   rb   r¨   s    r^   r©   z-SIMDScheduling.codegen_node.<locals>.<lambda>#  s   € ´c¸!¿.¹.Ó:JÓ6K€ r_   rª   zSchedule:
 %s)rS  r+  r”  rŽ  Úschedule_logÚdebugÚcodegen_node_schedulerA   )r\   r�   r�   rŒ  rX   r$  r‚  s          r^   Úcodegen_nodezSIMDScheduling.codegen_node  sp   € ð 04¯~©~Ó/?ˆä  Ñ,KÔL×RÑRÑˆ‰?ˆE�6à×3Ñ3°E¸5À&ÓIˆÜ×ÑÐ+¨]Ô;à×)Ñ)Ü˜}¨e°VÓ<ó
ð 	
r_   c                óÒ  — t        j                  t         j                  «      j                  }t	        | «      sy|D �cg c]0  }|j                  «       r|j                  «       j                  «       ‘Œ2 }}t        d„ |D «       «      syt        j                  j                  j                  | |«       |D ],  }t        j                  j                  j                  ||«       Œ. yc c}w )NFc              3  ó2   K  — | ]  }t        |«      –— Œ y ­wrR   )r*   )rÒ   r‡  s     r^   rÓ   z8SIMDScheduling.can_use_32bit_indexing.<locals>.<genexpr>>  s   è ø€ ÒF°DÔ)¨$×/ÑFùr  T)ÚtorchÚiinfoÚint32r+  r*   Úhas_tensor_outputr  Ústorage_sizerÛ   r6   r‘   r’   Ú	guard_leq)rX   ÚbuffersÚint_maxr  Ú	buf_sizesr‡  s         r^   Úcan_use_32bit_indexingz%SIMDScheduling.can_use_32bit_indexing,  sÂ   € ô
 —+‘+œeŸk™kÓ*×.Ñ.ˆä% eÔ,Øð ö
àØ×$Ñ$Ô&ð �N‰NÓ×)Ñ)Õ+ð
ˆ	ð 
ô ÑF¸IÔFÔFØô 	
�‰×Ñ×"Ñ" 5¨'Ô2Øò 	6ˆDÜ�G‰G×Ñ×&Ñ& t¨WÕ5ð	6àùò
s   ¾5C$c                óÒ  — |j                   }| j                  ||j                  |j                  «      }| j	                  ||gd|i«      }|D ]  }| j                  ||«       Œ t        j                  |«       |D ]–  }t        j                  |«      5  |j                  «       }d d d «       | j                  ||«      }t        j                  j                  rt        ||«       t         j#                  d|«       ||_        t'        |«      |_        Œ˜ ~t)        |«      dkD  rt        |«      }n|\  }t        j                  |«      5  |j+                  «       D ]  }	|	j-                  «        Œ 	 d d d «       | j/                  |«       |j1                  |j$                  «       t        j2                  r|j5                  «        t        j6                  r|j7                  |d   j$                  «       t        j8                  xj:                  |j:                  z  c_        t        j8                  xj<                  |j<                  z  c_        t        j8                  j>                  j@                  rÐt        jB                  rÀ|d   jD                  jG                  «       }
|j+                  «       D ]�  }	|	jI                  «       }||
vrŒ|	jJ                  €J ‚|	jJ                  jM                  «       }|€ŒCtN        d   dxx   dz  cc<   t        j8                  j>                  jQ                  d|jR                  ›d|› d	�«       Œ’ | jU                  «        y # 1 sw Y   �Œ®xY w# 1 sw Y   �ŒðxY w)
Nrÿ   z+Generating kernel code with kernel_name: %sr7   r   ÚinductorÚintermediate_hookszrun_intermediate_hooks(rˆ   rÄ   )+r‚  rW  rX   rŸ  Úcreate_kernel_choicesÚ!codegen_node_schedule_with_kernelr=   Úmerge_workspaces_inplacer6   Úset_kernel_handlerr1  Údefine_kernelr   ÚtraceÚenabledr/   r  r“  rö   r   rÚ   Úscheduler_nodesÚmark_runÚcodegen_commentrÜ  Únan_assertsrÚ  r  r‘   Úremoved_buffersÚinplaced_to_removeÚwrapper_codeÚsupports_intermediate_hooksÚgenerate_intermediate_hooksrØ   Úlive_output_buffersr  r�   Úget_origin_noder   Ú	writelinerU   Úfree_buffers_in_scheduler)r\   Úkernel_featuresr‚  r  ÚkernelsrZ   Úsrc_coderö   Úfinal_kernelr�   Ú	live_outsrU   Úorigin_nodes                r^   r”  z$SIMDScheduling.codegen_node_scheduleH  sð  € Ø'×5Ñ5ˆØ×#Ñ#Ø˜?×0Ñ0°/×2QÑ2Qó
ˆð ×,Ñ,Ø˜f˜X¨
°OÐ'Dó
ˆð ò 	JˆFØ×2Ñ2°=À&ÕIð	Jä×,Ñ,¨WÔ5Øò 	3ˆFÜ×%Ñ% fÓ-ñ 3Ø!×0Ñ0Ó2�÷3à×,Ñ,¨X°}ÀfÓMˆKÜ�|‰|×#Ò#Ü7Ø!Øôô �I‰IÐCÀ[ÔQØ!,ˆFÔÜ(¨Ó2ˆFÕð	3ð ô ˆw‹<˜!ÒÜ& wÓ/‰Là%‰Oˆ\ä×!Ñ! ,Ó/ñ 	 Ø'×7Ñ7Ó9ò  �Ø—‘•ñ ÷	 ð 	×Ñ˜]Ô+Ø× Ñ  ×!9Ñ!9Ô:ä×ÒØ×*Ñ*Ô,Ü×!Ò!Ø×(Ñ(¨°©×)?Ñ)?Ô@ä	�‰×Ò <×#?Ñ#?Ñ?ÕÜ	�‰×"Ò" l×&EÑ&EÑEÕ"ô �G‰G× Ñ ×<Ò<Ü×2Ò2ð   ™
Ÿ™×;Ñ;Ó=ˆIØ'×7Ñ7Ó9ò 
�Ø—}‘}“�Ø˜yÑ(ØØ—y‘yÐ,Ð,Ð,Ø"Ÿi™i×7Ñ7Ó9�ØÑ*Ü˜ZÑ(Ð)=Ó>À!ÑCÓ>Ü—G‘G×(Ñ(×2Ñ2Ø1°+×2BÑ2BÐ1EÀRÈÀvÈQÐOõð
ð 	×&Ñ&Õ(÷k3ñ 3ú÷&	 ñ 	 ús   ÂMÅ&MÍM	ÍM&c                ó(   —  | j                   |i |¤ŽgS rR   )rD  )r\   r¹  Úkernel_argsÚkernel_kwargss       r^   r¥  z$SIMDScheduling.create_kernel_choices‹  s)   € ð ˆD×ÑØðàñð
ð 	
r_   c           	     ó<  — |5  t        j                  «       }i }|D ]½  }|t        u r |j                  |j	                  «       «       Œ+|t
        u r|j                  «        ŒD|j                  «        |j                  |j                  «       «      }|j                  t        j                  |j                  j                  |«      j                  «       «      «       Œ¿ |j!                  |j#                  «       «       |D ]ˆ  }|t        u r |j                  |j	                  «       «       Œ+|t
        u r|j                  «        ŒDt%        |j                  «       |j                  |j                  «       «      }|j'                  |«       ŒŠ 	 d d d «       y # 1 sw Y   y xY wrR   )rp  Ú	ExitStackr>   Úenter_contextrr  r?   ÚcloseÚdecide_inplace_updater¤  rL  rz  rG  ÚfromkeysÚ_bodyÚindexing_from_argsr‹   r=  Úkeysr#   rÂ   )r\   r‚  rZ   ÚstackÚall_indexingr�   r­   s          r^   r¦  z0SIMDScheduling.codegen_node_schedule_with_kernel•  sS  € Øñ 	-Ü×(Ñ(Ó*ˆEØˆLð &ò �ØÔ+Ñ+Ø×'Ñ'¨×(@Ñ(@Ó(BÕCØœ_Ñ,Ø—K‘K•Mà×.Ñ.Ô0Ø!'×!<Ñ!<¸T¿_¹_Ó=NÓ!O�JØ ×'Ñ'ÜŸ™Ø ŸJ™J×9Ñ9¸*ÓE×LÑLÓNóõðð ×$Ñ$ \×%6Ñ%6Ó%8Ô9ð &ò 	-�ØÔ+Ñ+Ø×'Ñ'¨×(@Ñ(@Ó(BÕCØœ_Ñ,Ø—K‘K•Mô 6°d·j±jÔAØ!'×!<Ñ!<¸T¿_¹_Ó=NÓ!O�JØ—L‘L Õ,ñ	-÷-	-÷ 	-ñ 	-ús   ƒFFÆFF©Úonly_gen_src_codec               óf
  — |j                   \  }\  }}|dk(  sJ ‚|j                  j                  |j                  «      \  }}	i }
|j                  «       }g }|D ]  }|j	                  «       }|j                  |«       ||z  sŒ*t        |«      dk(  sJ ‚||
t        t        |«      «      <   |j                  j                  t        t        |«      «      «       g }Œ� t        |«      dk(  sJ ‚|5  |s|g|¢D ]  }|j                  «        Œ  |	«       }|j                  d«      5  |D ]0  }|j                  |j                  |j                  «       «      «       Œ2 |j                   j#                  t%        «       «       ddd«       |j&                  j)                  «       D �]+  \  }}d|› d�}|
j+                  |j-                  «       g «      x}sŒ0t/        d„ |D «       «      }t1        j2                  d| «      5  |j                  |«      5  |D ]ˆ  }t        |j	                  «       «      dk(  r<t        |«      dk(  r.t5        |«      r#|xj6                  |j	                  «       z  c_        |j                  |j                  |j                  «       «      «       ŒŠ |j                   j#                  t%        «       «       ddd«       ddd«       �Œ. 	 ddd«       t9        t:        «      s$|j=                  d	«       |j=                  d
d¬«       t?        j@                  |«      5  |j&                  jC                  «       D ]  }d|› d�}|j=                  |d¬«       Œ |j                  d«      5  t9        |t:        «      r|}n|j=                  d«       |jD                  }ddd«       g |¢|‘|¢}t0        jF                  rH|jI                  «       dz  }|jK                  «       › d› d|jM                  |«      jO                  «       › �}|rcddd«       S | jQ                  ||«      }t0        jR                  jT                  rtW        ||«       ddd«       | jY                  «       |j[                  |j                  «       t>        j\                  xj^                  |j^                  z  c_/        t>        j\                  xj`                  |j`                  z  c_0        | jc                  «        y# 1 sw Y   �ŒnxY w# 1 sw Y   �Œ=xY w# 1 sw Y   �ŒnxY w# 1 sw Y   �ŒCxY w# 1 sw Y   �Œ„xY w# 1 sw Y   ŒâxY w)z�
        Codegen a triton template

        If `only_gen_src_code` the src code will be returned instead of codegen'd into the wrapper
        r7   r   z<STORE_OUTPUT>Nz<LOAD_INPUT_ú>c              3  ó<   K  — | ]  }|j                  «       –— Œ y ­wrR   )Úcan_codegen_without_upcasts)rÒ   Úp_ns     r^   rÓ   z2SIMDScheduling.codegen_template.<locals>.<genexpr>æ  s   è ø€ ò 5Ø>A˜×7Ñ7×9ñ5ùó   ‚ztriton.codegen_upcast_to_fp32z<DEF_KERNEL>z	<ARGDEFS>F)Ústrictg    eÍÍAr  )2r”  r�   Úmake_kernel_renderrT  rU  r˜   rÚ   r•   ÚiterÚprologue_fused_inputsr¯   r­  Úset_subgraph_bodyrÂ   r¤  rL  ÚcseÚ
invalidater   Únamed_input_nodesrf   r±   r  rÛ   r   Úpatchr   Ú#prologue_fused_inputs_preserve_zerorÕ   ri   Úfinalize_hookr6   r¨  rÊ  ÚcodeÚbenchmark_kernelrÿ  Úimports_for_benchmark_kernelÚcodegen_kernel_benchmarkÚgetvaluer©  rª  r«  r/   r®  rÜ  r‘   r°  r±  r¸  )r\   Útemplate_nodeÚepilogue_nodesÚprologue_nodesrÎ  rŒ  Ú_numelr$  rZ   ÚrenderÚbuf_name_to_prologue_groupÚtemplate_readsÚprologue_groupÚprologueÚnamesr�   Úpartial_codeÚ
input_nameÚbufferÚsubgraph_nameÚcan_codegen_without_upcastÚprologue_noder»  r‚  Únum_gbrö   s                             r^   Úcodegen_templatezSIMDScheduling.codegen_template·  s  € ð ,×1Ñ1ÑˆÑˆF�FØ˜Š{Ðˆ{Ø&×+Ñ+×>Ñ>¸}×?QÑ?QÓR‰ˆ�à%'Ð"Ø&×8Ñ8Ó:ˆØˆØ&ò 	$ˆHØ×-Ñ-Ó/ˆEØ×!Ñ! (Ô+à�~Ó%Ü˜5“z Q’Ð&�Ø@NÐ*¬4´°U³Ó+<Ñ=Ø×,Ñ,×0Ñ0´´d¸5³kÓ1BÔCØ!#‘ð	$ô �>Ó" aÒ'Ð'Ð'àñ ,	@Ù$ð +Ð<¨^Ð<ò $�DØ—M‘M•Oð$ñ "›8ˆLà×)Ñ)Ð*:Ó;ñ 4Ø*ò Q�DØ—L‘L ×!<Ñ!<¸T¿_¹_Ó=NÓ!OÕPðQà—
‘
×%Ñ%¤j£lÔ3÷4ð
 '-×&>Ñ&>×&DÑ&DÓ&Fó @Ñ"�
˜FØ".¨z¨l¸!Ð <�Ø%?×%CÑ%CØ—O‘OÓ% ró&ð �>ñ ô 25ñ 5ØESô5ó 2Ð.ô  Ÿ™Ø7Ð=WÐ9Wóñ @ð $×5Ñ5°mÓDñ @Ø1?ò " ä$'¨×(FÑ(FÓ(HÓ$IÈQÒ$NÜ(+¨NÓ(;¸qÒ(@ä'CÀMÔ'RØ(.×(RÒ(RØ,9×,JÑ,JÓ,Lñ)*Õ(Rð !.× 5Ñ 5Ø$*×$?Ñ$?Ø(5×(@Ñ(@Ó(Bó%&õ!"ð"ð #ŸJ™J×1Ñ1´*³,Ô?÷!@÷@ñ @ñ@÷,	@ô\ ˜,¬Ô,Ø×&Ñ& ~Ô6Ø×&Ñ& {¸5Ð&ÔAô ×!Ñ! &Ó)ñ 	Tð %×6Ñ6×;Ñ;Ó=ò H�
Ø".¨z¨l¸!Ð <�Ø×*Ñ*¨=ÀÐ*ÕGðHð ×)Ñ)Ð*:Ó;ñ 1Ü˜l¬CÔ0Ø+‘Hà ×.Ñ.Ð/?Ô@Ø+×0Ñ0�H÷1ð N˜nÐM¨mÐM¸nÐMˆMä×&Ò&Ø×9Ñ9Ó;¸cÑA�à×:Ñ:Ó<Ð=¸RØ�j Ø×6Ñ6°vÓ>×GÑGÓIÐJðLð ñ !Ø÷1	Tñ 	Tð4 ×,Ñ,¨X°}ÀfÓMˆKä�|‰|×#Ò#Ü7¸À{ÔS÷;	Tð> 	×Ñ˜]Ô+Ø×Ñ˜;¨×(:Ñ(:Ô;ä	�‰×Ò 6×#9Ñ#9Ñ9ÕÜ	�‰×"Ò" f×&?Ñ&?Ñ?Õ"Ø×&Ñ&Ô(Ø÷_4ñ 4ú÷&@ñ @ú÷@ñ @ú÷3,	@ñ ,	@ú÷t1ñ 1ú÷	Tð 	Tús‰   Ã,5TÄ!AS&Å:ATÇ)TÇ5T ÈB1S3	Ê8T Ë TÌA
T'Í'1TÎA+T'Ð9T'Ó&S0	Ó+TÓ3S=Ó8T Ô T
ÔTÔTÔT$	ÔT'Ô'T0c                ó¢   — t         j                  j                  j                  t         j                  j                  j                  «       «       y rR   )r6   r‘   r²  r·  Ú
device_opsÚsynchronizera   s    r^   Úcodegen_synczSIMDScheduling.codegen_sync-  s-   € Ü	�‰×Ñ×&Ñ&¤q§w¡w×'9Ñ'9×'EÑ'EÓ'GÕHr_   c           
     ó¸  — ddl m} |D �cg c]  }|j                  «       ‘Œ }}i i }
}	t        ||«      D ]u  \  }}t	        |d„ ¬«      j
                  \  }\  }}| j                  |||«      }| j                  |||«      }||||f|
|<   |j                  |t        |||«      | ¬«      |	|<   Œw |j                  || ||	|
¬«      }t        j                  dt        |«      |D �cg c]  }t        |«      ‘Œ c}«       g }|D �]>  }|D �cg c]  }|j                  «       ‘Œ }} |||¬«      }t        ||«      D ]á  \  }}| j                  |
|   d	   |j                  |	|   «      «       |	|   }|
|   d	   }|sIt!        j"                  |«      5  t%        j&                  |«      D ]  }|j)                  «        Œ 	 d d d «       t         j*                  xj,                  |j,                  z  c_        t         j*                  xj.                  |j.                  z  c_        Œã |j1                  «       }|j3                  |||f«       �ŒA |S c c}w c c}w c c}w # 1 sw Y   Œ�xY w)
Nr7   )ÚComboKernelc                ó4   — t        | j                  «       «      S rR   r‘  r¨   s    r^   r©   z;SIMDScheduling.generate_combo_kernel_code.<locals>.<lambda>=  ó   € ¼#¸a¿n¹nÓ>NÓ:O€ r_   rª   )rÿ   Úoptimize_mask)r�   Útriton_schedulingÚcustom_algorithmÚ
kernel_mapÚnode_info_mapz1ComboKernels: %d nodes partitioned into %s groups)Úenable_autotuneÚmixed_sizesr   )Útriton_combo_kernelrü  rS  re  r+  r”  rŽ  rW  Úcreate_triton_kernelrA   Úhorizontal_partitionr  r“  rÚ   r¦  Úcreate_sub_kernelr6   r¨  r@   Ú
only_nodesr­  r‘   r°  r±  r1  r˜   )r\   Úsubkernel_nodesÚcustom_part_algorithmr  r  rÎ  rü  r�   Úfused_node_listsÚsubkernel_mapÚnode_schedule_mapÚpnr�   rŒ  rX   r$  r‚  r  Ú
partitionsÚpÚkernel_code_listÚ
node_grouprZ   Ú	subkernelr»  s                            r^   Úgenerate_combo_kernel_codez)SIMDScheduling.generate_combo_kernel_code0  s‹  € õ 	5à9HÖI°˜DŸN™NÕ,ÐIÐÐIØ+-¨rÐ(ˆÜ˜_Ð.>Ó?ò 		‰IˆB�Ü!$ UÑ0OÔ!P×!VÑ!VÑˆA‰��vØ ×7Ñ7¸¸uÀfÓMˆMØ×'Ñ'¨°u¸fÓEˆFØ$1°6¸5À&Ð$HÐ˜bÑ!Ø +× @Ñ @ØÜ+¨M¸5À&ÓIØ"-˜oð !Aó !ˆM˜"Òð		ð !×5Ñ5Ø!Ø"Ø2Ø$Ø+ð 6ó 
ˆ
ô 	�	‰	Ø?Ü�Ó Ø'Ö(˜ŒS��VÒ(ô	
ð
 ÐØ$ó 	DˆJØ=GÖH°T §¡Õ 0ÐHÐÐHÙ Ø /Ø'ôˆFô
 ! Ð-=Ó>ò K‘	��EØ×6Ñ6Ø% bÑ)¨!Ñ,Ø×,Ñ,¨]¸2Ñ->Ó?ôð *¨"Ñ-�	Ø 1°"Ñ 5°aÑ 8�Ù(Ü×-Ñ-¨iÓ8ñ ,Ü$6×$AÑ$AÀ-Ó$Pò ,˜DØ ŸM™M�Oñ,÷,ô —‘×'Ò'¨9×+DÑ+DÑDÕ'Ü—‘×*Ò*¨i×.JÑ.JÑJÖ*ðKð ×,Ñ,Ó.ˆHØ×#Ñ# X¨v°zÐ$BÖCð-	Dð.  Ðùòc Jùò. )ùò  I÷,ð ,ús   ‹IÃ IÄIÆ+IÉIc                óÖ  — |j                  «       }|j                  }|j                  }t        j                  dkD  xs t        j                  dk(  xr |}| j                  ||||«      }|D ]l  \  }}}	| j                  ||g|«      }
| j                  |g«       t        j                  d|
«       |j                  t        j                  j                  |
«       Œn | j                  «        y )Nr7   z"ComboKernels: generated kernel %s.)Úget_subkernel_nodesÚuse_custom_partition_algor  r   Úcombo_kernel_allow_mixed_sizesr  r©  r®  r  r“  rÜ  r6   r‘   r²  r¸  )r\   Úcombo_kernel_noder  r  r  r  r  r»  rZ   rŒ  rö   s              r^   Úcodegen_combo_kernelz#SIMDScheduling.codegen_combo_kernelm  së   € Ø+×?Ñ?ÓAˆØ 1× KÑ KÐØ+×;Ñ;ˆÜ×;Ñ;¸aÑ?ò 
Ü×1Ñ1°QÑ6ÒPÐ;Pð 	ð  ×:Ñ:ØÐ2°OÀ[ó
Ðð $4ò 	BÑˆH�f˜aØ×,Ñ,¨XÐ8IÐ7JÈFÓSˆKØ× Ñ Ð"3Ð!4Ô5Ü�I‰IÐ:¸KÔHØ×ÑœqŸw™w×3Ñ3°[ÕAð		Bð 	×&Ñ&Õ(r_   é    c           
     ó†  ‡ ‡‡
— ‰dk(  }dˆ ˆˆ
fd„}|j                  «       \  }Š
t        |«      dk  rt        ‰
«      dk  rg S |j                  «       \  }Š
 |||r|n‰
|j                  |«      «      }|D �cg c]?  }t        ‰ j	                  |j
                  |‰«      |j                  |j                  ¬«      ‘ŒA }	}|	S c c}w )Nr7   c                ód  •— t        |j                  «      t        |«      k(  sJ d|j                  ›d|›�«       ‚|j                  |j                  g}t	        d„ t
        j                  j                  |«      D «       «      sJ ‚t
        j                  j                  |«      D �cg c]:  }|j                  t        j                  j                  vrt        |t        «      r|‘Œ< }}t        |j                  D �cg c]  }|j                  ‘Œ c}«      }dd„}t        ‰j!                   ||«      g| «      dd¬«      g}|D �]Ô  }t        j                  j"                  j%                  |j&                  |j                  «      }	t        |	«      t        |«      k(  sJ ‚	 |	j'                  d«      dz   }
|
t        |«      k(  rŒt	        d	„ |	|
d
 D «       «      rŒ•	  ||d
|
 «       |||
d
 «      f}t        j                  j"                  j+                  t-        d„ t/        ||	«      D «       «      «      }|j                  |v r|dz  }t        j1                  |d   «      r|dz  }t        j1                  |d   «      r|dz  }t        j                  j"                  j+                  |t-        t        j                  |‰«      «      z
  «      dk\  s�ŒŠ|j3                  t        ‰j!                   ||d
|
 «       |||
d
 «      g‰«      ||j                  ¬«      «       �Œ× |S c c}w c c}w # t(        $ r Y �Œðw xY w)zX
            Compute tiling candidates by dividing up the iteration ranges.
            zrw.range_vars=z ranges=c              3  óH   K  — | ]  }t        |t        t        f«      –— Œ y ­wrR   )rÕ   r   r   )rÒ   rý  s     r^   rÓ   zHSIMDScheduling.candidate_tilings.<locals>.tile_ranges.<locals>.<genexpr>‘  s$   è ø€ ò àô ˜3¤¬GÐ 4×5ñùs   ‚ "c                óf   — t         j                  j                  j                  t	        | «      «      S rR   r§  )rt  s    r^   Úcollapse_rangeszNSIMDScheduling.candidate_tilings.<locals>.tile_ranges.<locals>.collapse_ranges�  s"   € Ü—w‘w×'Ñ'×0Ñ0´¸vÓ1FÓGÐGr_   Únoner   )r  rU   Úscorer7   c              3  ó&   K  — | ]	  }|d k(  –— Œ y­wrx  rÇ   rÑ   s     r^   rÓ   zHSIMDScheduling.candidate_tilings.<locals>.tile_ranges.<locals>.<genexpr>³  s   è ø€ Ò; a˜1 �6Ñ;ùs   ‚Nc              3  ó2   K  — | ]  \  }}|d k7  sŒ|–— Œ y­wrx  rÇ   )rÒ   r‡  r
  s      r^   rÓ   zHSIMDScheduling.candidate_tilings.<locals>.tile_ranges.<locals>.<genexpr>Â  s    è ø€ ò "Ù!-  vÈÐSTËœñ"ùs   ‚�r   ©r  r$  rU   )rt  r8  ro   rl   )rÚ   Ú
range_varsru  r€  rÛ   r  rH  rI  rU   r6   r‘   r°  rÕ   r   r   ÚCandidateTilingÚcreate_partial_tilingr’   Ústride_hintsr€   Ú
ValueErrorr¦   r1   re  Úis_good_sizer˜   )Úis_pointwisert  ÚrwÚdep_sourcesrý  ÚdepsÚwrite_namesr"  Útilingsré  ÚsplitÚtiled_groupsr$  rž  rŸ  Úreduction_rangess                €€€r^   Útile_rangesz5SIMDScheduling.candidate_tilings.<locals>.tile_ranges†  s  ø€ ô �r—}‘}Ó%¬¨V«Ò4ÐS¸¸¿¹Ð8HÈ	È&ÈÐ6SÓSÐ4ð Ÿ8™8 R§Y¡YÐ/ˆKÜñ ä$Ÿ?™?×8Ñ8¸ÓEôô ð ð ô %Ÿ?™?×8Ñ8¸ÓEöàØ—8‘8¤1§7¡7×#:Ñ#:Ñ:Ü˜s¤IÔ.ò ðˆDð ô %¸"¿)¹)Ö%D°3 c§h£hÒ%DÓEˆKóHô
  Ø×4Ñ4Ù(¨Ó0Ð1°<óð  ØôðˆGð ó 4�ÜŸ'™'×*Ñ*×7Ñ7¸¿	¹	À2Ç=Á=ÓQ�Ü˜7“|¤s¨6£{Ò2Ð2Ð2ð
Ø#ŸM™M¨!Ó,¨qÑ0�EØ¤ F£Ò+Ø ÜÑ;¨7°5°6¨?Ô;Ô;ð !ð <ñ $ F¨6¨E NÓ3Ù# F¨5¨6 NÓ3ð �ô Ÿ™×(Ñ(×2Ñ2Ü!ñ "Ü14°V¸WÓ1Eô"ó ó�ð
 —8‘8˜{Ñ*à˜Q‘J�EÜ"×/Ñ/°¸Q±Ô@Ø˜Q‘J�EÜ"×/Ñ/°¸Q±Ô@Ø˜Q‘J�Eô —G‘G×$Ñ$×.Ñ.Ø¤¬i¯o©o¸fÐFVÓ.WÓ XÑXóð ôð
 —N‘NÜ'Ø#&×#<Ñ#<á$3°F¸6¸E°NÓ$CÙ$3°F¸5¸6°NÓ$Cð!"ð !0ó$ð #(Ø!$§¡ô
öðQ4ðl ˆNùò[ùò &Eøô: "ò Úðús$   Â$?LÃ8LÆ"L"Æ=L"Ì"	L/Ì.L/r'  )r.  rr   ro   úlist[CandidateTiling])rL  rÚ   Ú"pointwise_or_reduction_read_writesr)  Úcomplete_partial_tilingr  r$  rU   )rž  r�   rX   rŸ  r.  r7  Úpointwise_rangesÚpartial_tilingsr  Úfull_tilingsr6  s   `  `      @r^   Úcandidate_tilingsz SIMDScheduling.candidate_tilings�  sà   ú€ ð '¨!Ñ+ˆ÷\	ð| .2¯_©_Ó->Ñ*ÐÐ*ÜÐÓ  AÒ%¬#Ð.>Ó*?À1Ò*DØˆIð .2¯_©_Ó->Ñ*ÐÐ*Ù%ØÙ ,ÑÐ2BØ×3Ñ3°LÓAó
ˆð *ö	
ð ô Ø×2Ñ2Ø—M‘M 5¨/óð —l‘lØ—[‘[öð	
ˆð 	
ð Ðùò	
s   Á6AB>c                óŠ   — g d¢t        |«       d }ddgdt        |«       }t        g t        ||«      ¢t        ||«      ¢«      S )zK
        Create a tiling dict from pointwise and reduction splits.
        )rH   rI   rJ   NrK   rL   )rÚ   r   re  )rž  Ú	pw_tilingÚreduction_tilingÚpw_prefixesÚreduction_prefixess        r^   Úcreate_tilingzSIMDScheduling.create_tilingþ  sY   € ò &¤s¨9£~ oÐ&7Ð8ˆØ# U˜^Ð,C¬cÐ2BÓ.CÐDÐÜØVŒc�+˜yÓ)ÐV¬CÐ0BÐDTÓ,UÐVó
ð 	
r_   c                ó>   — | j                  |r|ng |s|«      S g «      S rR   )rD  )rž  r  r.  s      r^   r*  z$SIMDScheduling.create_partial_tiling  s0   € ð × Ñ Ù"‰F¨Ù&ˆFó
ð 	
à,.ó
ð 	
r_   c                ó–   — t        |j                  «       «      }d|v }||z  }|t        |«      z  g}|r||fn||f} | j                  |Ž S )zb
        Given a tiling for only pointwise or reduction dimensions, adds the missing one.
        rJ   )r·  r‹   r1   rD  )	rž  r  rX   rŸ  Úsplitsr.  Útotal_numelÚmissing_tilingÚtiling_argss	            r^   r:  z&SIMDScheduling.complete_partial_tiling  sf   € ô �f—m‘m“oÓ&ˆØ˜f�}ˆà˜oÑ-ˆØ%¬°fÓ(=Ñ=Ð>ˆñ )5ˆV�^Ñ$¸>È6Ð:Rð 	ð !ˆs× Ñ  +Ð.Ð.r_   c           
     óÌ  — |dk(  }t        t        t        t        j                  f      «       }t        j                  |«      D �]†  }t        |t        j                  «      sŒ|j                  «       }|st        |d   «      dk(  rŒC||rdnd   }|g}	|j                  j                  «       D �
cg c],  }
t        |
t        «      rt        |
j                  «      dkD  r|
‘Œ. }}
|D �]U  }
g |
j                  j!                  «       ¢}t        j"                  j$                  }t&        j(                  j*                  }t-        |«      D ]!  \  }\  }}||z  }|j/                  ||«      sŒ! n |j1                  ||«      sŒ–dz   }|r|d| n||d }g }|D ]—  \  }}t3        j4                  |
j6                  |«      }t9        d|j;                  t<        «      |j;                  t>        «      z   t        |«      «      }t3        j@                  ||||«      }|�|d   n|g}|jC                  |«       Œ™ |	jE                  |«       �ŒX |	D ]‰  }t9        dt        |«      tF        jH                  jJ                  z
  «      }|dz   }tM        |d| «      }|ftO        ||d «      z   }|jQ                  | jS                  | jU                  ||«      ||«      «       Œ‹ �Œ‰ tW        |t        d¬«      }|S c c}
w )zÑ
        Creates N-dimensional tiling candidiates, attempting to simplify loads/stores
        by tiling the kernel into higher dimensions.

        Returns a list of tilings ranked by dimensionality.
        r7   r   Nr   T)r«   Úreverse),r   rG  ri   rx   ÚExprr?   ÚfilterrÕ   r   r{  rL  rÚ   rt  Úreads_and_writesr   rt  rf   ry   rz   r6   r‘   r’   r+  Ústatically_known_geqr“   r8   Úget_subexpr_involving_symbolr€   r+  r  r   r   Úmatch_mod_div_block_exprr:  r˜   r   rX  Ú	max_tilesr1   rí  r¯   r:  r*  rÖ  )rž  r‚  Úpointwise_numelrŸ  r.  r3  r�   Únode_rangesÚranges_to_tileÚnode_tilingsrý  Úmemory_depsÚall_var_rangesÚpointwise_vars_numelr’   Úpointwise_end_idxÚvarrX   Úreduction_start_idxrW   Úindex_tilingr€   Únum_dimsÚmatch_resultÚdimsÚnode_tilingÚnum_leading_dimsÚfirst_trailing_dimÚcollapsed_leading_dimÚcollapsed_splitsÚranked_tilingss                                  r^   Úget_nd_tilingszSIMDScheduling.get_nd_tilings+  s*  € ð '¨!Ñ+ˆÜœT¤#¤u§z¡z /Ñ2Ñ3Ó5ˆÜ#×*Ñ*¨=Ó9ó W	ˆDÜ˜d¤I×$;Ñ$;Ô<Øð Ÿ/™/Ó+ˆKÙ¤C¨°A©Ó$7¸1Ò$<Øð )©l©ÀÑBˆNØ*Ð+ˆLð  ×+Ñ+×<Ñ<Ó>öàÜ˜c¤9Ô-´#°c·j±j³/ÀAÒ2Eò ðˆKð ð
 #ó 12�ð "7 3§:¡:×#3Ñ#3Ó#5Ð!6�Ü',§w¡w§{¡{Ð$ÜŸ7™7×+Ñ+�Ü7@ÀÓ7Pò Ñ3Ð%¡|¨¨UØ(¨EÑ1Ð(Ø×4Ñ4Ø,¨oõñ ðð  ×7Ñ7Ø(¨/ôð ð '8¸!Ñ&;Ð#ñ $ð #Ð#7Ð$7Ñ8à'Ð(;Ð(<Ð=ð ð  "�Ø",ò .‘J�C˜Ü/×LÑLØŸ	™	 3ó�Eô
  #ØØŸ™¤HÓ-°·±¼OÓ0LÑLÜ˜NÓ+ó �Hô $7×#OÑ#OØ˜s E¨8ó$�Lð /;Ð.F˜<¨š?ÈUÈG�DØ ×'Ñ'¨Õ-ð%.ð( ×#Ñ# LÖ1ðc12ðh  ,ò �Ü#& q¬#¨kÓ*:¼V¿]¹]×=TÑ=TÑ*TÓ#UÐ Ø%5¸Ñ%9Ð"Ü(5°kÐBUÐCUÐ6VÓ(WÐ%Ø$9Ð#;¼eØÐ 2Ð 3Ð4ó?ñ $Ð ð —‘Ø×/Ñ/Ø×1Ñ1Ð2BÀLÓQØ'Ø'óõòðUW	ôv  ØÜØô
ˆð Ðùòas   Â.1K!c                ób  ‡‡— ‰dk(  }| j                  |g‰g«      }|st        j                  j                  rt        j                  j                  dk  r¢t
        j                  t        j                  k  rt        j                  |«      D ]g  }t        j                  j                  rŒt        | j                  ||‰«      «      dkD  sŒ>t
        j                  t        j                  d«      «        |S  |S t!        t"           «       }t%        j&                  «       }t        j                  |«      D ]g  }| j                  ||‰«      D ]O  }	|	j(                  |v rŒ|	j(                  �|j+                  |	j(                  «       ||	xx   |	j,                  z  cc<   ŒQ Œi |j/                  «       D �	�
cg c]  \  }	}
|	j0                  ‘Œ }}	}
t        j                  j                  dk\  r?|r=	 	 	 	 	 	 dd„}t3        dt        |«      «      D ]  } ||d   ||   «      }|€Œ|g|z   } n t        |«      dkD  rt
        j                  d|«       t        j                  j4                  r| j7                  ||‰«      |z   }|D ]-  Št9        ‰t:        «      sJ ‚t=        ˆˆfd„|D «       «      sŒ+‰c S  |S c c}
}	w )	zÓ
        Heuristics to decide how to tile kernels.
        Currently, we tile based on stride-1 dimensions.

        Returns:
            `(tile1, tile2, reduction_numel)` s.t. `tile1 * tile2 == numel`

        r7   r   z½
                                Reduction over non-contiguous dims.
                                Consider setting config.triton.tile_reductions to True.
                                r   c                ó  — | d   | j                  dd«      }}|d   |j                  dd«      }}t        j                  j                  j	                  ||z
  «      dk(  ry t        j                  j                  j	                  ||z
  «      dk  r||f||fc\  }}\  }}t        j                  j                  j	                  ||z
  «      dkD  sJ ‚t        j                  j                  j                  ||«      sy |t        ||«      || d   dœ}|S )NrJ   rI   r7   r   rK   )rH   rI   rJ   rK   )r±   r6   r‘   r’   r¦   r{  r   )Útiling0rd  Úa0Úa1Úb0Úb1Ú
new_tilings          r^   Úconvert_tiling_to_3dz:SIMDScheduling.select_tiling.<locals>.convert_tiling_to_3dÙ  s  € ð ! ™ w§{¡{°3¸Ó':�B�Ø  ™ w§{¡{°3¸Ó':�B�Ü—7‘7×#Ñ#×-Ñ-¨b°2©gÓ6¸!Ò;ØÜ—7‘7×#Ñ#×-Ñ-¨b°2©gÓ6¸Ò:à*,¨b¨°B¸°8Ð&‘H�R˜™h˜r 2ä—w‘w×'Ñ'×1Ñ1°"°r±'Ó:¸QÒ>Ð>Ð>Ü—w‘w×'Ñ'×DÑDÀRÈÔLØð Ü! " bÓ)ØØ" 5™>ñ	�
ð "Ð!r_   zpossibly bad tiling: %sc              3  óº   •K  — | ]R  }t        |t        j                  «      r6t        j	                  ‰j                  «       |j                  «       ‰¬ «      –— ŒT y­w))rŸ  N)rÕ   r   r{  rm   r   r‹   rL  )rÒ   r�   rŸ  r  s     €€r^   rÓ   z/SIMDScheduling.select_tiling.<locals>.<genexpr>  sO   øè ø€ ò ð Ü˜d¤I×$;Ñ$;Ô<ô	 ×(Ñ(Ø—M‘M“O T§_¡_Ó%6Èð )÷ ñùs   ƒAA)rk  r5  rd  r5  ro   zOptional[dict[str, sympy.Expr]])rD  r   rX  Útile_reductionsrS  Úperf_hint_logÚlevelÚloggingÚWARNINGr?   rN  rÚ   r>  ÚinfoÚtextwrapÚdedentr   ri   Úcollectionsr   rU   r¯   r$  Úmost_commonr  ÚrangeÚprefer_nd_tilingrh  rÕ   rG  rÛ   )rž  r‚  rX   rŸ  r.  Údefault_tilingr�   Ú
seen_namesÚcandidate_tilesÚcandidate_tilingr$  rg  rq  r7  Únew_3d_tilingr  s      `           @r^   rW  zSIMDScheduling.select_tiling�  s¾  ù€ ð '¨!Ñ+ˆð ×*Ñ*¨E¨7°_Ð4EÓFˆá¤V§]¡]×%BÒ%BÜ�]‰]×$Ñ$¨Ò)ä×"Ñ"¤g§o¡oÒ5Ü+×2Ñ2°=ÓAò �Dä"ŸM™M×9Ó9Ü × 5Ñ 5°d¸EÀ?Ó SÓTÐWXÓXä%×*Ñ*Ü$ŸO™Oð!$óôð Ø!Ð!ðð "Ð!ä¤‘_Ó&ˆ
Ü4?×4GÑ4GÓ4IˆÜ#×*Ñ*¨=Ó9ò 	LˆDØ$'×$9Ñ$9¸$ÀÀÓ$Wò LÐ Ø#×(Ñ(¨JÑ6ØØ%×*Ñ*Ð6Ø—N‘NÐ#3×#8Ñ#8Ô9ØÐ 0Ó1Ð5E×5KÑ5KÑKÔ1ñLð	Lð ,;×+FÑ+FÓ+H÷7
á'Ð  %ð ×#Ó#ð7
ˆñ 7
ô
 �=‰=×"Ñ" aÒ'©Lð"Ø.ð"Ø9Nð"à0ó"ô0 ˜1œc .Ó1Ó2ò �Ù 4Ø" 1Ñ% ~°aÑ'8ó!�ð !Ñ,Ø&3 _°~Ñ%E�NÙðô ˆ~Ó Ò"Ü×ÑÐ8¸.ÔIô �=‰=×)Ò)à×"Ñ" =°%¸ÓIØ ñ!ð ð
 %ò 		ˆFÜ˜f¤dÔ+Ð+Ð+Üô ð *ô	õ ð ’ð		ð ÐùóC7
s   Æ.J+c                 ó   — y rR   rÇ   ra   s    r^   ÚflushzSIMDScheduling.flush  r3  r_   c                 ó   — yr#  rÇ   ra   s    r^   Úready_to_flushzSIMDScheduling.ready_to_flush  r$  r_   c                óà  — t        d„ |D «       «      sÂt        |d„ ¬«      j                  \  }\  }}| j                  |||«      }| j	                  |||«      }| j                  |t        |||«      ¬«      }| j                  ||«       t        j                  d|«      5  t        j                  |«      5  |j                  «       }	d d d «       d d d «       nL|d   j                  |«      \  }
}}t        j                  d|«      5  | j                  |||
d¬«      }	d d d «       	j                  t!        t"        j$                  «      d	«      }	|	S # 1 sw Y   Œ‰xY w# 1 sw Y   Œ@xY w# 1 sw Y   ŒLxY w)
Nc              3  ó<   K  — | ]  }|j                  «       –— Œ y ­wrR   )rR  )rÒ   r´   s     r^   rÓ   zASIMDScheduling.generate_kernel_code_from_nodes.<locals>.<genexpr>  s   è ø€ Ò2 q�1—=‘=—?Ñ2ùrÔ  c                ó4   — t        | j                  «       «      S rR   r‘  r¨   s    r^   r©   z@SIMDScheduling.generate_kernel_code_from_nodes.<locals>.<lambda>  rþ  r_   rª   )rÿ   rá  r   TrÍ  Útriton_)r²  r+  r”  rŽ  rW  rD  rA   r¦  r   rÝ  r6   r¨  r1  Úget_prologue_template_epiloguerö  Úreplaceri   r-   ÚKERNEL_NAME)r\   r�   rá  rŒ  rX   r$  r‚  r  rZ   r»  rí  ÚtemplateÚepilogues                r^   Úgenerate_kernel_code_from_nodesz.SIMDScheduling.generate_kernel_code_from_nodes  ss  € ÜÑ2¨EÔ2Ô2Ü!$ UÑ0OÔ!P×!VÑ!VÑˆA‰��vØ ×7Ñ7¸¸uÀfÓMˆMØ×'Ñ'¨°u¸fÓEˆFØ×%Ñ%ØÜ+¨M¸5À&ÓIð &ó ˆFð ×2Ñ2°=À&ÔIä—‘Ð/Ð1AÓBñ3ä×$Ñ$ VÓ,ñ3ð "×0Ñ0Ó2�÷	3÷ 3ð 3ð ,1°©8×+RÑ+RØó,Ñ(ˆH�h ô —‘Ð0Ð2BÓCñ Ø×0Ñ0ØØØØ&*ð	 1ó �÷ð ×#Ñ#¤C¬×(?Ñ(?Ó$@À)ÓLˆØˆ÷%3ð 3ú÷ 3ð 3ú÷ð ús0   ÂEÂ3EÃEÄE$ÅE	ÅEÅE!Å$E-c                 ó   — y rR   rÇ   )r\   r‚  s     r^   r®  zSIMDScheduling.codegen_comment4  r3  r_   c                ó   — t         ‚rR   r  )r\   r»  r‚  rZ   s       r^   r©  zSIMDScheduling.define_kernel7  r  r_   N)r�   z<Union[scheduler.FusedSchedulerNode, scheduler.SchedulerNode])rX   rl   rž  z<Iterable[Union[ir.Buffer, ir.TensorBox, ir.TorchBindObject]]ro   rr   )r¹  rA   )r¹  rA   ro   zlist[SIMDKernel])ro   úOptional[str]r<  )r  zlist[BaseSchedulerNode]r  rr   r  rr   r  rr   rÎ  rr   ro   zlist[tuple[str, Any, Any]])ro   r8  )r@  r8  rA  r8  ro   r5  )r  r8  r.  rr   ro   r5  )r  r5  rX   rl   rŸ  rl   ro   r5  )ro   z"list[dict[str, tuple[sympy.Expr]]])ro   r5  rq   )&rt   ru   rv   rm   rD  r?  rG  rP  Úcan_fuse_verticalr[  rŽ  r•  r@  r¡  r”  r¥  r¦  rö  rú  r  r  rA  r¿   rÀ   r>  rD  r*  r:  rh  rx   ry   rz   rW  r…  r‡  r‘  r®  r©  rÇ   r_   r^   rC  rC  )  s  … Ø'€K�Ó'òQòF6ðP !ÐØ"Ðò^ð@
ØPó
ð$ ðØðàMðð 
òó ðó6A)ðF
Ø1ð
à	ó
ò -ðF SXñtà	ótòlIð #(ð; à0ð; ð  $ð; ð ð	; ð
 ð; ð  ð; ð 
$ó; òz)ð( Ø€Y×Ñ˜Óòyó ó ðyðv ð

Ø,ð

Ø@Tð

à	ò

ó ð

ð ð
à$ð
ð ð
ð 
ò	
ó ð
ð ð/à%ð/ð ð/ð $ð	/ð
 
ò/ó ð/ð( ðoð
 
,òoó ðoðb à38·7±7·;±;ðpà	òpó ðpòdóóò<ó"r_   rC  T)Úfrozenc                  ó@   — e Zd ZU ded<   ded<   dZded<   ed„ «       Zy)	r)  r5  r  r¶   r$  Nr”  rU   c                ór   — t         j                  j                  j                  | «      } | dk\  xr | dz  dk(  S )z@Somewhat arbitrary heuristic used to boost scores for some sizesr  r   rŽ  )r³   s    r^   r-  zCandidateTiling.is_good_sizeA  s5   € ô �G‰G×Ñ×&Ñ& qÓ)ˆØ�B‰wÒ(˜A ™F a™KÐ(r_   )rt   ru   rv   r?  rU   r@  r-  rÇ   r_   r^   r)  r)  ;  s)   … à!Ó!ØƒJØ€Dˆ-Óàñ)ó ñ)r_   r)  c                  ó   — e Zd Zy)r|  N)rt   ru   rv   rÇ   r_   r^   r|  r|  H  s   „ Ør_   r|  )rí   r=  ro   ri   )vÚ
__future__r   r{  rp  Údataclassesr¿   r  rv  rê   r‘  ry  r   Útypingr   r   r   r   r	   r
   r   Útyping_extensionsr   rx   r˜  Útorch._loggingÚtorch.fx.immutable_collectionsr   Útorch.utils._ordered_setr   Útorch.utils._sympy.functionsr   r   r   Útorch.utils._sympy.symbolr   r   r   r   Ú_dynamo.utilsr   rÏ  r   r   r   Úanalyze_preserves_zero_maskr   Ú	codecacher   Údependenciesr   r   r    r!   r"   Úoptimize_indexingr#   Úruntime.runtime_utilsr$   r%   r&   r'   r(   Úutilsr)   r*   r+   r,   r-   r.   r/   r0   r1   r2   r3   Úvirtualizedr4   r5   r6   Úblock_analysisr8   Úcommonr9   r:   r;   r<   Úmulti_kernelr=   Úsimd_kernel_featuresr>   r?   r@   rA   Úcollections.abcrB   rC   rD   Ú	getLoggerrt   r  Ú_loggingÚgetArtifactLoggerrt  r’  Ú
fusion_logÚdoprintr>  r/  Ú	dataclassrN   rn   r”   rî   rï   rm   rC  r)  Ú	Exceptionr|  rÇ   r_   r^   ú<module>r·     sÒ  ðå "ã Û Û Û Û Û Û Û Û Ý ß X× XÑ XÝ %ã ã Û Ý 9Ý /ß LÑ L÷ó õ &ß $Ñ $Ý FÝ !ß 6Ñ 6ß -Ý Aß ;ß DÑ D÷÷ ÷ ñ ÷ -Ñ ,Ý /ß PÓ PÝ %÷ó ñ ß<Ñ<ð €g×Ñ˜Ó!€Ø—‘×0Ñ0°¸<ÓH€Ø�~‰~×/Ñ/°¸*ÓE€Ø�^‰^×-Ñ-¨h¸ÓA€
ñ 	‹×Ñ€áÒ7Ó8€ð ×Ñ÷5+ð 5+ó ð5+ôpx;˜/ô x;ôv;'˜?ô ;'ó|ñ Ð+°;ÈÔT€ôW�˜Ñ(¨'°/Ñ*Bô WôtO"�^ô O"ðd  €×Ñ˜dÔ#÷	)ð 	)ó $ð	)ô	�	õ 	r_   