Ë
    g^(h‡ò  ã                  ó^  — d dl mZ d dlZd dlZd dlZd dlZd dlZd dlmZ d dl	m
Z
 d dlmZmZmZmZmZmZ d dlZd dlZd dlZddlmZ ddlmZ dd	lmZmZ dd
lmZmZ ddlm Z  ddl!m"Z"m#Z# ddl$m%Z% ddl#m&Z& ddl'm(Z(m)Z) ddl*m+Z+ ddl,m-Z-m.Z. ddl/m0Z0m1Z1m2Z2m3Z3m4Z4 ddl5m6Z7m8Z8 ddl9m:Z:m;Z;m<Z<m=Z=m>Z>m?Z?m@Z@mAZAmBZB ddlCmDZD ddlEmFZF ddlGmHZHmIZImJZJ erd dlKmLZL ddl*mMZMmNZN  ejž                  eP«      ZQd„ ZR G d„ deS«      ZT G d„ d e@«      ZU eU«       j¬                  ZW e@«       j¬                  ZXej²                  d!ej´                  d"ej¶                  d#ej¸                  d$ejº                  d%ej¼                  d&ej¾                  d'ejÀ                  d(ejÂ                  d)ejÄ                  d*ejÆ                  d+ejÈ                  d,ejÊ                  d-iZfd.„ Zgd/„ Zh G d0„ d1e?«      ZieijÕ                  d2«        G d3„ d4e;«      ZkejØ                   G d5„ d6«      «       Zmd7„ Znd8„ Zo G d9„ d:eI«      Zp G d;„ d<eJ«      Zqy)=é    )ÚannotationsN)Údefaultdict)Úinf)ÚAnyÚCallableÚcastÚOptionalÚTYPE_CHECKINGÚUnioné   )Úis_integer_dtype)Ú
OrderedSet)ÚFloorDivÚModularIndexing)Úsymbol_is_typeÚSymT)ÚValueRangesé   )ÚconfigÚir)ÚHalideCodeCache)Úget_reduction_combine_fn)Úis_metric_table_enabledÚlog_kernel_metadata)ÚAddParenHandler)ÚHalideInputSpecÚ
HalideMeta)Úget_bounds_index_exprÚget_kernel_metadataÚparallel_num_threadsÚsympy_index_symbolÚ
sympy_subs)Ú_opsÚVé   )	ÚBackendFeatureÚCSEVariableÚDeferredLineÚIndentedBufferÚKernelArgTypeÚOpOverridesÚPythonPrinterÚSizeArgÚ	TensorArg)ÚDTYPE_TO_CPP)Úcexpr)Úconstant_reprÚ
SIMDKernelÚSIMDScheduling)ÚSequence)ÚReductionTypeÚ	StoreModec                ó"  — t        | t        «      rVd| cxk  rdk  sKn t        j                  t        j                  «      }| |j
                  k(  ry| |j                  k(  ryd| ›d�S t        | t        «      rdt        | «      › d�S t        | «      S )Ni   €iÿÿÿzhl.Int(64).min()zhl.Int(64).max()zhl.i64(ú)zhl.f64()
Ú
isinstanceÚintÚtorchÚiinfoÚint64ÚminÚmaxÚfloatr1   Úrepr)ÚvalÚinfos     ú\/var/www/skyplay_api_hub/venv/lib/python3.12/site-packages/torch/_inductor/codegen/halide.pyÚhalide_constantrE   =   s~   € Ü�#”sÔ [°CÔ%E¸:Ô%EÜ�{‰{œ5Ÿ;™;Ó'ˆØ�$—(‘(Š?Ø%Ø�$—(‘(Š?Ø%Ø˜˜˜qÐ!Ð!Ü�#”uÔØœ sÓ+Ð,¨AÐ.Ð.Ü�‹9Ðó    c                  ó    ‡ — e Zd Zdˆ fd„Zˆ xZS )ÚUnsupportedc                ó*   •— t         ‰| �  d|› �«       y )Nz!halide backend does not support: )ÚsuperÚ__init__)ÚselfÚthingÚ	__class__s     €rD   rK   zUnsupported.__init__K   s   ø€ Ü‰ÑÐ<¸U¸GÐDÕErF   ©ÚreturnÚNone)Ú__name__Ú
__module__Ú__qualname__rK   Ú__classcell__©rN   s   @rD   rH   rH   J   s   ø„ ÷Fñ FrF   rH   c                  óÊ   ‡ — e Zd Zed„ «       Zed„ «       Zd„ Zd„ Zd„ Zd„ Z	e	Z
d„ Zd„ Zd	„ Zd
„ Zd„ Zd„ Zd„ Zd„ Zd„ Zd„ Zd„ Zd„ Zd„ Zd„ Zd„ Zˆ fd„Zd„ ZeZd„ Zd„ Zˆ xZS )ÚHalidePrinterc                óD   — dt         j                  j                  › d| › d�S )Núhl.cast(ú, r8   )r$   ÚkernelÚindex_dtype©Úexprs    rD   Ú
cast_indexzHalidePrinter.cast_indexP   s"   € àœ!Ÿ(™(×.Ñ.Ð/¨r°$°°qÐ9Ð9rF   c                ó   — d| › d�S )Núhl.cast(hl.Float(32), r8   © r^   s    rD   Ú
cast_floatzHalidePrinter.cast_floatT   s   € à'¨ v¨QÐ/Ð/rF   c                ó   — d|› d�S )Núhl.f32(r8   rc   ©rL   r_   s     rD   Ú_print_FloatzHalidePrinter._print_FloatX   s   € Ø˜˜˜aÐ Ð rF   c                óz   — t        |j                  «      dk(  sJ ‚d| j                  |j                  d   «      › d�S )Nr%   rf   r   r8   ©ÚlenÚargsÚ_printrg   s     rD   Ú_print_ToFloatzHalidePrinter._print_ToFloat[   s9   € Ü�4—9‘9‹~ Ò"Ð"Ð"Ø˜Ÿ™ T§Y¡Y¨q¡\Ó2Ð3°1Ð5Ð5rF   c                ó˜   — t        |j                  «      dk(  sJ ‚| j                  d| j                  |j                  d   «      › d�«      S )Nr%   ú	hl.floor(r   r8   ©rk   rl   r`   rm   rg   s     rD   Ú_print_floorzHalidePrinter._print_floor_   óB   € Ü�4—9‘9‹~ Ò"Ð"Ð"Ø�‰ ¨4¯;©;°t·y±yÀ±|Ó+DÐ*EÀQÐGÓHÐHrF   c                ó˜   — t        |j                  «      dk(  sJ ‚| j                  d| j                  |j                  d   «      › d�«      S )Nr%   ú	hl.trunc(r   r8   rq   rg   s     rD   Ú_print_TrunczHalidePrinter._print_Truncc   rs   rF   c                ó˜   — t        |j                  «      dk(  sJ ‚| j                  d| j                  |j                  d   «      › d�«      S )Nr%   úhl.ceil(r   r8   rq   rg   s     rD   Ú_print_ceilingzHalidePrinter._print_ceilingi   sB   € Ü�4—9‘9‹~ Ò"Ð"Ð"Ø�‰ ¨$¯+©+°d·i±iÀ±lÓ*CÐ)DÀAÐFÓGÐGrF   c                óJ   — d| j                  | j                  |«      «      › d�S ©Nzhl.sqrt(r8   )rd   rm   rg   s     rD   Ú_helper_sqrtzHalidePrinter._helper_sqrtm   s$   € Ø˜$Ÿ/™/¨$¯+©+°dÓ*;Ó<Ð=¸QÐ?Ð?rF   c                óÎ   — | j                  |j                  d   «      }| j                  |j                  d   «      }| j                  |j                  d   «      }d|› d|› d|› d�S )Nr   r%   r   ú
hl.select(r[   r8   )Údoprintrl   )rL   r_   ÚcÚpÚqs        rD   Ú_print_WherezHalidePrinter._print_Wherep   s_   € Ø�L‰L˜Ÿ™ 1™Ó&ˆØ�L‰L˜Ÿ™ 1™Ó&ˆØ�L‰L˜Ÿ™ 1™Ó&ˆØ˜A˜3˜b   2 a S¨Ð*Ð*rF   c                óh  — t        |j                  «      dk(  r| j                  |j                  d   «      S t        |j                  «      dz  }| j                  t        j                  |j                  d | Ž «      }| j                  t        j                  |j                  |d  Ž «      }d|› d|› d�S )Nr%   r   r   úhl.min(r[   r8   )rk   rl   rm   ÚsympyÚMin©rL   r_   ÚmidÚaÚbs        rD   Ú
_print_MinzHalidePrinter._print_Minv   s’   € Üˆt�y‰y‹>˜QÒØ—;‘;˜tŸy™y¨™|Ó,Ð,ä�$—)‘)‹n Ñ!ˆØ�K‰KœŸ	™	 4§9¡9¨T¨c ?Ð3Ó4ˆØ�K‰KœŸ	™	 4§9¡9¨S¨T ?Ð3Ó4ˆØ˜˜˜2˜a˜S Ð"Ð"rF   c                óh  — t        |j                  «      dk(  r| j                  |j                  d   «      S t        |j                  «      dz  }| j                  t        j                  |j                  d | Ž «      }| j                  t        j                  |j                  |d  Ž «      }d|› d|› d�S )Nr%   r   r   úhl.max(r[   r8   )rk   rl   rm   r†   ÚMaxrˆ   s        rD   Ú
_print_MaxzHalidePrinter._print_Max   s’   € Üˆt�y‰y‹>˜QÒØ—;‘;˜tŸy™y¨™|Ó,Ð,ä�$—)‘)‹n Ñ!ˆØ�K‰KœŸ	™	 4§9¡9¨T¨c ?Ð3Ó4ˆØ�K‰KœŸ	™	 4§9¡9¨S¨T ?Ð3Ó4ˆà˜˜˜2˜a˜S Ð"Ð"rF   c                ó˜   — t        |j                  «      dk(  sJ ‚| j                  d| j                  |j                  d   «      › d�«      S )Nr%   úhl.abs(r   r8   rq   rg   s     rD   Ú
_print_AbszHalidePrinter._print_Abs‰   sB   € Ü�4—9‘9‹~ Ò"Ð"Ð"Ø�‰ ¨¯©°T·Y±Y¸q±\Ó)BÐ(CÀ1ÐEÓFÐFrF   c                óz   — t        |j                  «      dk(  sJ ‚d| j                  |j                  d   «      › d�S )Nr%   zhl.cos((r   r8   rj   rg   s     rD   Ú_print_OpaqueUnaryFn_cosz&HalidePrinter._print_OpaqueUnaryFn_cos�   ó9   € Ü�4—9‘9‹~ Ò"Ð"Ð"Ø˜$Ÿ+™+ d§i¡i°¡lÓ3Ð4°AÐ6Ð6rF   c                óz   — t        |j                  «      dk(  sJ ‚d| j                  |j                  d   «      › d�S )Nr%   z	hl.cosh((r   r8   rj   rg   s     rD   Ú_print_OpaqueUnaryFn_coshz'HalidePrinter._print_OpaqueUnaryFn_cosh‘   ó9   € Ü�4—9‘9‹~ Ò"Ð"Ð"Ø˜4Ÿ;™; t§y¡y°¡|Ó4Ð5°QÐ7Ð7rF   c                óz   — t        |j                  «      dk(  sJ ‚d| j                  |j                  d   «      › d�S )Nr%   z	hl.acos((r   r8   rj   rg   s     rD   Ú_print_OpaqueUnaryFn_acosz'HalidePrinter._print_OpaqueUnaryFn_acos•   r™   rF   c                óz   — t        |j                  «      dk(  sJ ‚d| j                  |j                  d   «      › d�S )Nr%   zhl.sin((r   r8   rj   rg   s     rD   Ú_print_OpaqueUnaryFn_sinz&HalidePrinter._print_OpaqueUnaryFn_sin™   r–   rF   c                óz   — t        |j                  «      dk(  sJ ‚d| j                  |j                  d   «      › d�S )Nr%   z	hl.sinh((r   r8   rj   rg   s     rD   Ú_print_OpaqueUnaryFn_sinhz'HalidePrinter._print_OpaqueUnaryFn_sinh�   r™   rF   c                óz   — t        |j                  «      dk(  sJ ‚d| j                  |j                  d   «      › d�S )Nr%   z	hl.asin((r   r8   rj   rg   s     rD   Ú_print_OpaqueUnaryFn_asinz'HalidePrinter._print_OpaqueUnaryFn_asin¡   r™   rF   c                óz   — t        |j                  «      dk(  sJ ‚d| j                  |j                  d   «      › d�S )Nr%   zhl.tan((r   r8   rj   rg   s     rD   Ú_print_OpaqueUnaryFn_tanz&HalidePrinter._print_OpaqueUnaryFn_tan¥   r–   rF   c                óz   — t        |j                  «      dk(  sJ ‚d| j                  |j                  d   «      › d�S )Nr%   z	hl.tanh((r   r8   rj   rg   s     rD   Ú_print_OpaqueUnaryFn_tanhz'HalidePrinter._print_OpaqueUnaryFn_tanh©   r™   rF   c                óz   — t        |j                  «      dk(  sJ ‚d| j                  |j                  d   «      › d�S )Nr%   z	hl.atan((r   r8   rj   rg   s     rD   Ú_print_OpaqueUnaryFn_atanz'HalidePrinter._print_OpaqueUnaryFn_atan­   r™   rF   c                ó  •— |j                   rt        ‰| �	  |«      S |j                  \  }}| j	                  | j                  |«      «      }| j	                  | j                  |«      «      }| j                  d|› d|› d�«      S )Nrp   z / r8   )Ú
is_integerrJ   Ú_print_FloorDivrl   rd   r   r`   )rL   r_   ÚxÚdivrN   s       €rD   rª   zHalidePrinter._print_FloorDiv±   sr   ø€ Ø�?Š?Ü‘7Ñ*¨4Ó0Ð0à—‘‰ˆˆ3Ø�O‰O˜DŸL™L¨›OÓ,ˆØ�o‰o˜dŸl™l¨3Ó/Ó0ˆØ�‰ ¨1¨#¨S°°°QÐ7Ó8Ð8rF   c                ó˜   — t        |j                  «      dk(  sJ ‚| j                  d| j                  |j                  d   «      › d�«      S )Nr%   ú	hl.round(r   r8   rq   rg   s     rD   Ú_print_RoundzHalidePrinter._print_Roundº   rs   rF   c                ó2   — |j                   \  }}d|› d|› d�S )Nú(ú) / (z+hl.f32(0)))rl   )rL   r_   rŠ   r‹   s       rD   Ú_print_IntTrueDivzHalidePrinter._print_IntTrueDivÀ   s$   € Ø�y‰y‰ˆˆ1à�1�#�U˜1˜#˜[Ð)Ð)rF   c                ó~   — |j                   \  }}| j                  |«      }t        |«      }dd| z  ›d|› dd|z  ›d�S )Nrf   g      $@z)*hl.round((z	)*hl.f32(ú)))rl   rm   r:   )rL   r_   rB   Úns       rD   Ú_print_RoundDecimalz!HalidePrinter._print_RoundDecimalÅ   sL   € Ø—‘‰ˆˆQØ�k‰k˜#ÓˆÜ�‹FˆØ˜ 1 "™Ð(¨°S°E¸À4ÈÁ7À+ÈRÐPÐPrF   ) rR   rS   rT   Ústaticmethodr`   rd   rh   rn   rr   rv   Ú_print_TruncToIntry   r|   rƒ   rŒ   r�   r“   r•   r˜   r›   r�   rŸ   r¡   r£   r¥   r§   rª   r¯   Ú_print_RoundToIntr³   r·   rU   rV   s   @rD   rX   rX   O   s³   ø„ Øñ:ó ð:ð ñ0ó ð0ò!ò6òIòIð %ÐòHò@ò+ò#ò#òGò7ò8ò8ò7ò8ò8ò7ò8ò8ô9òIð %Ðò*ö
QrF   rX   z	hl.Bool()zhl.BFloat(16)zhl.Float(16)zhl.Float(32)zhl.Float(64)z	hl.Int(8)z
hl.Int(16)z
hl.Int(32)z
hl.Int(64)z
hl.UInt(8)zhl.UInt(16)zhl.UInt(32)zhl.UInt(64)c                ó   — t         |    S ©N)Ú_halide_type©Údtypes    rD   Úhalide_typerÀ   á   s   € Ü˜ÑÐrF   c                óð   — t        | «      r/| j                  r#| t        j                  k7  rt        j                  } | t        j
                  t        j                  fv rt        j                  } t        | «      S r¼   )	r   Ú	is_signedr;   r=   Úint32Úfloat16Úbfloat16Úfloat32rÀ   r¾   s    rD   Úhalide_acc_typerÇ   å   sM   € Ü˜Ô 5§?¢?°uÄÇÁÒ7KÜ—‘ˆØ”—‘¤§¡Ð/Ñ/Ü—‘ˆÜ�uÓÐrF   c                  ó  — e Zd Ze	 	 dB	 	 	 dCd„«       ZedDd„«       Zed„ «       Zed„ «       Zed„ «       Z	ed„ «       Z
ed„ «       Zed	„ «       Zed
„ «       Zed„ «       Zed„ «       Zed„ «       Zed„ «       Zed„ «       Zed„ «       Zed„ «       Zed„ «       Zed„ «       Zed„ «       Zed„ «       Zed„ «       Zed„ «       Zed„ «       Zed„ «       Zed„ «       Zed„ «       Zed„ «       Zed„ «       Z ed„ «       Z!ed„ «       Z"ed „ «       Z#ed!„ «       Z$ed"„ «       Z%ed#„ «       Z&ed$„ «       Z'ed%„ «       Z(ed&„ «       Z)ed'„ «       Z*ed(„ «       Z+ed)„ «       Z,ed*„ «       Z-ed+„ «       Z.ed,„ «       Z/ed-„ «       Z0ed.„ «       Z1ed/„ «       Z2ed0„ «       Z3ed1„ «       Z4ed2„ «       Z5ed3„ «       Z6ed4„ «       Z7ed5„ «       Z8ed6„ «       Z9ed7„ «       Z:ed8„ «       Z;ed9„ «       Z<ed:„ «       Z=ed;„ «       Z>ed<„ «       Z?ed=„ «       Z@edEd>„«       ZAed?„ «       ZBed@„ «       ZCedA„ «       ZDy)FÚHalideOverridesNc                óX   — |t         j                  k(  rd| › d�S dt        |«      › d| › d�S )Nr±   z != 0)rZ   r[   r8   )r;   ÚboolrÀ   )r«   r¿   Ú	src_dtypeÚuse_compute_typess       rD   Úto_dtypezHalideOverrides.to_dtypeî   s9   € ð ”E—J‘JÒØ�q�c˜�=Ð Øœ+ eÓ,Ð-¨R°¨s°!Ð4Ð4rF   c                óâ   — |t         j                  t         j                  fv rdt        |«      › d| › d�} dt        |«      › d| › d�}|t         j                  t         j                  fv rd|› d�}|S )NrZ   r[   r8   zhl.reinterpret(rb   )r;   rÄ   rÅ   rÀ   )r«   r¿   rÌ   Úlines       rD   Úto_dtype_bitcastz HalideOverrides.to_dtype_bitcastù   ss   € àœŸ™¬¯©Ð7Ñ7Øœ; yÓ1Ð2°"°Q°C°qÐ9ˆAØ ¤¨UÓ!3Ð 4°B°q°c¸Ð;ˆØ”U—]‘]¤E§N¡NÐ3Ñ3Ø+¨D¨6°Ð3ˆDØˆrF   c                ó8   — | j                  t        |«      |«      S r¼   )rÎ   rE   )ÚclsÚvaluer¿   s      rD   ÚconstantzHalideOverrides.constant  s   € à�|‰|œO¨EÓ2°EÓ:Ð:rF   c                ó   — d| › d�S )Nr’   r8   rc   ©r«   s    rD   ÚabszHalideOverrides.abs  ó   € à˜˜˜1ˆ~ÐrF   c                óR   — t        | d«      sd| › d�S d| › d| j                  › d| › d�S )NÚnameúhl.exp(r8   z"hl.fast_exp(hl.cast(hl.Float(32), z)) if z!.type().bits() <= 32 else hl.exp(©ÚhasattrrÛ   r×   s    rD   ÚexpzHalideOverrides.exp
  s=   € ä�q˜&Ô!Ø˜Q˜C˜q�>Ð!Ø3°A°3°f¸Q¿V¹V¸HÐDeÐfgÐehÐhiÐjÐjrF   c                ó   — d| › d�S )NrÜ   r8   rc   r×   s    rD   Úlibdevice_expzHalideOverrides.libdevice_exp  rÙ   rF   c                ó   — d| › d�S r{   rc   r×   s    rD   ÚsqrtzHalideOverrides.sqrt  ó   € à˜!˜˜AˆÐrF   c                óœ   — t        | d«      s	d| › d|› d�S d| j                  › d|› d�}d| › d|› d	| › d
| › d|› d| j                  › d| › d|› d�S )NrÛ   r…   r[   r8   rZ   ú	.type(), úhl.select((ú<ú)|hl.is_nan(ú), ú) if z.type().is_float() else hl.min(rÝ   ©rŠ   r‹   s     rD   ÚminimumzHalideOverrides.minimum  ó   € ô �q˜&Ô!Ø˜Q˜C˜r !  AÐ&Ð&Ø�q—v‘v�h˜i¨ s¨!Ð,ˆØ˜Q˜C˜q   <°¨s°#°a°S¸¸1¸#¸UÀ1Ç6Á6À(ÐJiÐjkÐilÐlnÐopÐnqÐqrÐsÐsrF   c                óœ   — t        | d«      s	d| › d|› d�S d| j                  › d|› d�}d| › d|› d	| › d
| › d|› d| j                  › d| › d|› d�S )NrÛ   rŽ   r[   r8   rZ   ræ   rç   ú>ré   rê   rë   z.type().is_float() else hl.max(rÝ   rì   s     rD   ÚmaximumzHalideOverrides.maximum   rî   rF   c                óX   — t        |d«      rd|j                  › d|› d�}d| › d|› d|› d�S )NrÛ   rZ   ræ   r8   r~   r[   rÝ   )rŠ   r‹   r€   s      rD   ÚwherezHalideOverrides.where(  s?   € ä�1�fÔØ˜1Ÿ6™6˜( )¨A¨3¨aÐ0ˆAØ˜A˜3˜b   2 a S¨Ð*Ð*rF   c                ó   — d| › d�S )Nzhl.cos(r8   rc   r×   s    rD   ÚcoszHalideOverrides.cos.  rÙ   rF   c                ó   — d| › d�S )Nzhl.sin(r8   rc   r×   s    rD   ÚsinzHalideOverrides.sin2  rÙ   rF   c                ó   — t        d«      ‚)NÚlgamma©rH   r×   s    rD   rù   zHalideOverrides.lgamma6  ó   € ä˜(Ó#Ð#rF   c                ó   — d| › d�S )Nzhl.erf(r8   rc   r×   s    rD   ÚerfzHalideOverrides.erf:  rÙ   rF   c                ó   — d| › d�S )Nzhl.cosh(r8   rc   r×   s    rD   ÚcoshzHalideOverrides.cosh>  rä   rF   c                ó   — d| › d�S )Nzhl.sinh(r8   rc   r×   s    rD   ÚsinhzHalideOverrides.sinhB  rä   rF   c                ó   — d| › d�S )Nzhl.acos(r8   rc   r×   s    rD   ÚacoszHalideOverrides.acosF  rä   rF   c                ó   — d| › d�S )Nz	hl.acosh(r8   rc   r×   s    rD   ÚacoshzHalideOverrides.acoshJ  ó   € à˜1˜#˜QÐÐrF   c                ó   — d| › d�S )Nzhl.asin(r8   rc   r×   s    rD   ÚasinzHalideOverrides.asinN  rä   rF   c                ó   — d| › d�S )Nz	hl.asinh(r8   rc   r×   s    rD   ÚasinhzHalideOverrides.asinhR  r  rF   c                ó   — d| › d|› d�S )Nz	hl.atan2(r[   r8   rc   ©r«   Úys     rD   Úatan2zHalideOverrides.atan2V  ó   € à˜1˜#˜R ˜s !Ð$Ð$rF   c                ó   — d| › d�S )Nzhl.atan(r8   rc   r×   s    rD   ÚatanzHalideOverrides.atanZ  rä   rF   c                ó   — d| › d�S )Nz	hl.atanh(r8   rc   r×   s    rD   ÚatanhzHalideOverrides.atanh^  r  rF   c                ó   — t        d«      ‚)NÚcopysignrú   r  s     rD   r  zHalideOverrides.copysignb  s   € ä˜*Ó%Ð%rF   c                ó   — t        d«      ‚)NÚerfinvrú   r×   s    rD   r  zHalideOverrides.erfinvf  rû   rF   c                ó   — d| › d|› d�S )Nz	hl.hypot(r[   r8   rc   r  s     rD   ÚhypotzHalideOverrides.hypotj  r  rF   c                ó   — t        d«      ‚)NÚ	nextafterrú   r  s     rD   r  zHalideOverrides.nextaftern  s   € ä˜+Ó&Ð&rF   c                ó   — | › d|› �S ©Nz & rc   rì   s     rD   Úlogical_andzHalideOverrides.logical_andr  ó   € à��C˜�sˆ|ÐrF   c                ó   — | › d�S )Nz == 0rc   ©rŠ   s    rD   Úlogical_notzHalideOverrides.logical_notv  s   € à��Eˆ{ÐrF   c                ó   — | › d|› �S ©Nz | rc   rì   s     rD   Ú
logical_orzHalideOverrides.logical_orz  r  rF   c                ó   — d| › d|› d�S )Nr±   ú ^ r8   rc   rì   s     rD   Úlogical_xorzHalideOverrides.logical_xor~  s   € à�1�#�S˜˜˜1ˆ~ÐrF   c                ó   — | › d|› �S r  rc   rì   s     rD   Úbitwise_andzHalideOverrides.bitwise_and‚  r  rF   c                ó   — d| › �S )Nú~rc   r!  s    rD   Úbitwise_notzHalideOverrides.bitwise_not†  s   € à�1�#ˆwˆrF   c                ó   — | › d|› �S r$  rc   rì   s     rD   Ú
bitwise_orzHalideOverrides.bitwise_orŠ  r  rF   c                ó   — | › d|› �S )Nr'  rc   rì   s     rD   Úbitwise_xorzHalideOverrides.bitwise_xorŽ  r  rF   c                ó   — | › d|› �S )Nz << rc   rì   s     rD   Úbitwise_left_shiftz"HalideOverrides.bitwise_left_shift’  ó   € à��D˜˜ˆ}ÐrF   c                ó   — | › d|› �S )Nz >> rc   rì   s     rD   Úbitwise_right_shiftz#HalideOverrides.bitwise_right_shift–  r4  rF   c                ó   — d| › d|› d�S )Nzhalide_helpers.rand(r[   r8   rc   ©ÚseedÚoffsets     rD   ÚrandzHalideOverrides.randš  s   € à% d V¨2¨f¨X°QÐ7Ð7rF   c                ó   — d| › d|› d�S )Nzhalide_helpers.randn(r[   r8   rc   r8  s     rD   ÚrandnzHalideOverrides.randnž  s   € à& t f¨B¨v¨h°aÐ8Ð8rF   c           	     ó    — d| › d|› d|› d|› d�	S )Nzhalide_helpers.randint64(r[   r8   rc   )r9  r:  ÚlowÚhighs       rD   Ú	randint64zHalideOverrides.randint64¢  s#   € à*¨4¨&°°6°(¸"¸S¸EÀÀDÀ6ÈÐKÐKrF   c                óˆ   — t        j                  | d«      › dt        j                  j                  j                  d|«      › �S )Nr   ú + Úload_seed_offset)ÚopsÚloadr$   r\   rl   Úseed_offset)rÛ   r:  s     rD   Ú	load_seedzHalideOverrides.load_seed¦  s7   € ä—(‘(˜4 Ó#Ð$ C¬¯©¯©×(AÑ(AÐBTÐV\Ó(]Ð'^Ð_Ð_rF   c                ó   — d| › d�S )Nz1./hl.sqrt(r8   rc   r×   s    rD   ÚrsqrtzHalideOverrides.rsqrtª  s   € ð ˜Q˜C˜qÐ!Ð!rF   c                ó   — d| › d�S )Nzhl.tan(r8   rc   r×   s    rD   ÚtanzHalideOverrides.tan¯  rÙ   rF   c                ó   — d| › d�S )Nzhl.tanh(r8   rc   r×   s    rD   ÚtanhzHalideOverrides.tanh³  rä   rF   c                ó   — d| › d�S )Nz3(hl.reinterpret(hl.UInt(32), hl.cast(hl.Float(32), z)) >> 31) != 0rc   r×   s    rD   ÚsignbitzHalideOverrides.signbit·  s   € àDÀQÀCÀ~ÐVÐVrF   c                ó   — | › d| › d|› d|› �S )Nz - hl.trunc(ú/z)*rc   rì   s     rD   ÚfmodzHalideOverrides.fmod»  s!   € ð ��L   1 Q C r¨!¨Ð-Ð-rF   c                ó   — d| › d|› d�S )Nzhl.pow(r[   r8   rc   rì   s     rD   ÚpowzHalideOverrides.powÀ  s   € à˜˜˜2˜a˜S Ð"Ð"rF   c                ó   — d| › d�S )Nzhl.log(r8   rc   r×   s    rD   ÚlogzHalideOverrides.logÄ  rÙ   rF   c                ó   — d| › d�S )Nz hl.is_inf(hl.cast(hl.Float(32), rµ   rc   r×   s    rD   ÚisinfzHalideOverrides.isinfÈ  ó   € ð 2°!°°BÐ7Ð7rF   c                ó   — d| › d�S )Nz hl.is_nan(hl.cast(hl.Float(32), rµ   rc   r×   s    rD   ÚisnanzHalideOverrides.isnanÍ  rZ  rF   c                ó   — d| › d�S )Nr®   r8   rc   r×   s    rD   ÚroundzHalideOverrides.roundÒ  r  rF   c                ó   — d| › d�S )Nrp   r8   rc   r×   s    rD   ÚfloorzHalideOverrides.floorÖ  r  rF   c                ó   — d| › d|› d�S )Nr±   r²   z + hl.f32(0))rc   rì   s     rD   Úint_truedivzHalideOverrides.int_truedivÚ  s   € à�1�#�U˜1˜#˜]Ð+Ð+rF   c                ó.   — d| j                   › d| › d|› d�S )Nz"hl.floor(hl.cast(hl.Float(max(32, ú.type().bits())), ú) / r8   ©rÛ   rì   s     rD   ÚfloordivzHalideOverrides.floordivÞ  s)   € ð 1°·±°Ð8JÈ1È#ÈTÐRSÐQTÐTUÐVð	
rF   c                ó4  — t        j                  t        j                  d|«      t        j                  «      }t        j                  t        j                  |d«      t        j                  «      }t        j
                  ||«      }d|j                  › d|› d�S )NÚ0rZ   ræ   r8   )rE  rÎ   Últr;   Úint8ÚsubrÛ   )rÓ   r«   ÚleftÚrightrl  s        rD   ÚsignzHalideOverrides.signå  sg   € ä�|‰|œCŸF™F 3¨›N¬E¯J©JÓ7ˆÜ—‘œSŸV™V A s›^¬U¯Z©ZÓ8ˆÜ�g‰g�d˜EÓ"ˆØ˜!Ÿ&™&˜ ¨3¨%¨qÐ1Ð1rF   c                ó   — d| › d�S )Nru   r8   rc   r×   s    rD   ÚtrunczHalideOverrides.truncì  r  rF   c                ó.   — d| j                   › d| › d|› d�S )Nz"hl.trunc(hl.cast(hl.Float(max(32, rd  re  r8   rf  rì   s     rD   ÚtruncdivzHalideOverrides.truncdivð  s)   € ð
 1°·±°Ð8JÈ1È#ÈTÐRSÐQTÐTUÐVð	
rF   c                ó   — d| › d�S )Nrx   r8   rc   r×   s    rD   ÚceilzHalideOverrides.ceilø  rä   rF   c                ó   — d| › d�S )NrŽ   z, 0)rc   r×   s    rD   ÚreluzHalideOverrides.reluü  s   € à˜˜˜4Ð Ð rF   c                ó~  — t         j                  j                  |«      }t         j                  j                  t         j                  j	                  |«      t         j                  j                  |«      t        |«      ¬«      }|t        j                  t        j                  fvrt        j                  ||«      S |S ©N©Úbounds)r$   r\   Úprepare_indexingÚgenfuncÚindex_to_strÚused_dims_from_indexr   r;   rÃ   r=   rE  rÎ   )rÓ   r_   r¿   ÚindexÚvars        rD   Ú
index_exprzHalideOverrides.index_expr   sˆ   € ä—‘×)Ñ)¨$Ó/ˆÜ�h‰h×ÑÜ�H‰H×!Ñ! %Ó(Ü�H‰H×)Ñ)¨%Ó0Ü(¨Ó.ð ó 
ˆð
 œŸ™¤e§k¡kÐ2Ñ2Ü—<‘<  UÓ+Ð+Øˆ
rF   c                ó®   — t        j                  |t        j                  «      }t        j                  |||«      }||_        t        t        |«      «      S r¼   )rE  rÎ   r;   rÃ   Úhalide_clampÚindirect_indexing_sizer!   Ústr)rÓ   Ú	index_varÚsizeÚcheckÚwrap_negs        rD   Úindirect_indexingz!HalideOverrides.indirect_indexing  sC   € ô —L‘L ¬E¯K©KÓ8ˆ	Ü×$Ñ$ Y°°eÓ<ˆ	Ø+/ˆ	Ô(Ü!¤# i£.Ó1Ð1rF   c                óø   — t         j                  j                  t         j                  j                  |«      dz
  «      }t	        |t
        t        j                  f«      sd|j                  › d|› d�}d|› d|› d�S )Nr%   rZ   ræ   r8   z	hl.clamp(z, 0, )	r$   r\   ÚkexprÚrename_indexingr9   r:   r†   ÚIntegerrÛ   )rÓ   rÔ   rˆ  r‰  Úends        rD   r„  zHalideOverrides.halide_clamp  sj   € ä�h‰h�n‰nœQŸX™X×5Ñ5°dÓ;¸aÑ?Ó@ˆÜ˜$¤¤e§m¡mÐ 4Ô5Ø˜UŸZ™Z˜L¨	°#°°aÐ8ˆCð ˜5˜'  s e¨1Ð-Ð-rF   c                óˆ  — t         j                  j                  | |«      5 } |«       }d d d «       j                  j                  rt        |«      }t         j                  j                  d|j                  › dt        |«      › d�g t        j                  |«      ¬«      }t        j                  ||«      S # 1 sw Y   Œ�xY w)NrZ   ræ   r8   rz  )r$   r\   Ú
mask_loadsr{  Úis_boolrË   r}  rÛ   rE   r   ÚwraprE  ró   )ÚmaskÚbodyÚotherÚnew_maskÚresults        rD   ÚmaskedzHalideOverrides.masked  s©   € ä�X‰X× Ñ   uÓ-ð 	°Ù“VˆF÷	ð �=‰=× Ò Ü˜“KˆEô —‘× Ñ Ø�v—{‘{�m 9¬_¸UÓ-CÐ,DÀAÐFØÜ×#Ñ# EÓ*ð !ó 
ˆô �y‰y˜ 6¨5Ó1Ð1÷	ð 	ús   ¡B8Â8Cc                ó   — t        d«      ‚)NÚfrexp)ÚNotImplementedErrorr×   s    rD   rœ  zHalideOverrides.frexp.  s   € ä! 'Ó*Ð*rF   )NT)r¿   útorch.dtyperÌ   úOptional[torch.dtype])r¿   rž  rÌ   rž  )TT)ErR   rS   rT   r¸   rÎ   rÑ   ÚclassmethodrÕ   rØ   rß   rá   rã   rí   rñ   ró   rõ   r÷   rù   rý   rÿ   r  r  r  r  r
  r  r  r  r  r  r  r  r  r"  r%  r(  r*  r-  r/  r1  r3  r6  r;  r=  rA  rH  rJ  rL  rN  rP  rS  rU  rW  rY  r\  r^  r`  rb  rg  ro  rq  rs  ru  rw  r‚  r‹  r„  rš  rœ  rc   rF   rD   rÉ   rÉ   í   s&  „ Øð ,0Øð	5àð5ð )ò5ó ð5ð òó ðð ñ;ó ð;ð ñó ðð ñkó ðkð
 ñó ðð ñó ðð ñtó ðtð ñtó ðtð ñ+ó ð+ð
 ñó ðð ñó ðð ñ$ó ð$ð ñó ðð ñó ðð ñó ðð ñó ðð ñ ó ð ð ñó ðð ñ ó ð ð ñ%ó ð%ð ñó ðð ñ ó ð ð ñ&ó ð&ð ñ$ó ð$ð ñ%ó ð%ð ñ'ó ð'ð ñó ðð ñó ðð ñó ðð ñó ðð ñó ðð ñó ðð ñó ðð ñó ðð ñó ðð ñó ðð ñ8ó ð8ð ñ9ó ð9ð ñLó ðLð ñ`ó ð`ð ñ"ó ð"ð ñó ðð ñó ðð ñWó ðWð ñ.ó ð.ð ñ#ó ð#ð ñó ðð ñ8ó ð8ð ñ8ó ð8ð ñ ó ð ð ñ ó ð ð ñ,ó ð,ð ñ
ó ð
ð ñ2ó ð2ð ñ ó ð ð ñ
ó ð
ð ñó ðð ñ!ó ð!ð ñ	ó ð	ð ò2ó ð2ð ñ.ó ð.ð ñ2ó ð2ð  ñ+ó ñ+rF   rÉ   Úhalidec                  ól   ‡ — e Zd Z ej                  d«      Z	 d	 	 	 	 	 dˆ fd„Zd„ Zd„ Zd	d„Z	d„ Z
ˆ xZS )
ÚHalideCSEVariablez\b(tmp\d+)\[\?\]c                ó6   •— t         ‰| �  |||«       d | _        y r¼   )rJ   rK   Ú	used_dims)rL   rÛ   r{  r¿   rN   s       €rD   rK   zHalideCSEVariable.__init__9  s   ø€ ô 	‰Ñ˜˜v uÔ-Ø7;ˆ�rF   c                óT  — t        | j                  xs d«      }t        j                  ||j	                  «       «      D ]D  }t        |t        «      sŒ|j                  €
J |||f«       ‚|j                  |j                  «       ŒF t        j                  j                  |«      | _        y )Nrc   )r   r¥  Ú	itertoolsÚchainÚvaluesr9   r£  Úupdater$   r\   Úsort_used_dims)rL   rÛ   rl   ÚkwargsÚusedÚargs         rD   Úupdate_on_argsz HalideCSEVariable.update_on_argsB  s„   € Ü˜$Ÿ.™.Ò.¨BÓ/ˆÜ—?‘? 4¨¯©«Ó9ò 	+ˆCÜ˜#Ô0Õ1Ø—}‘}Ð0ÐC°4¸¸dÐ2CÓCÐ0Ø—‘˜CŸM™MÕ*ð	+ô Ÿ™×0Ñ0°Ó6ˆ�rF   c                óš   — t        |«      dk(  r| j                  › d�S | j                  › ddj                  t        t        |«      «      › d�S )Nr   z[()]ú[r[   ú])rk   rÛ   ÚjoinÚmapr†  )rL   Údimss     rD   Ú	index_strzHalideCSEVariable.index_strJ  sE   € Üˆt‹9˜Š>Ø—i‘i�[ Ð%Ð%à—)‘)�˜A˜dŸi™i¬¬C°«Ó7Ð8¸Ð:Ð:rF   c                ón   — | j                   €| j                  › d�S | j                  | j                   «      S )Nz[?])r¥  rÛ   r¶  )rL   s    rD   Ú__str__zHalideCSEVariable.__str__P  s0   € Ø�>‰>Ð!à—i‘i�[ Ð$Ð$Ø�~‰~˜dŸn™nÓ-Ð-rF   c           	     óÒ   — | j                   �t        d„ | j                   D «       «      sJ ‚| j                  | j                   D �cg c]  }|j                  ||«      ‘Œ c}«      S c c}w )Nc              3  óP   K  — | ]  }t        |t        j                  «      –— Œ  y ­wr¼   ©r9   r†   ÚExpr©Ú.0r«   s     rD   ú	<genexpr>z-HalideCSEVariable.subs_str.<locals>.<genexpr>W  s    è ø€ ò 2
Ø*+ŒJ�qœ%Ÿ*™*×%ñ2
ùó   ‚$&)r¥  Úallr¶  Úget)rL   Úreplacementsr¶   s      rD   Úsubs_strzHalideCSEVariable.subs_strV  s^   € Ø�~‰~Ð)¬cñ 2
Ø/3¯~©~ô2
ô /
ð 	
ð 
ð �~‰~¸t¿~¹~ÖN¸!˜|×/Ñ/°°1Õ5ÒNÓOÐOùÒNs   ÁA$r¼   )r{  zValueRanges[Any]r¿   rŸ  rP   rQ   )rP   r†  )rR   rS   rT   ÚreÚcompileÚundefined_rerK   r¯  r¶  r¸  rÄ  rU   rV   s   @rD   r£  r£  6  sQ   ø„ Ø�2—:‘:Ð1Ó2€Lð (,ð	<ð !ð<ð %ð	<ð
 
õ<ò7ò;ó.öPrF   r£  c                  óH   ‡ — e Zd ZU ded<   ded<   ded<   dˆ fd„Zd	d„Zˆ xZS )
ÚDimensionInfozOptional[sympy.Expr]r_   ú
sympy.Exprrˆ  Ústridec                ó¬   •— t         ‰| �  «        t        j                  j                  j                  |d«      r| }| }|| _        || _        || _        y ©Nr   )	rJ   rK   r$   ÚgraphÚsizevarsÚstatically_known_ltr_   rˆ  rË  )rL   r_   rˆ  rË  rN   s       €rD   rK   zDimensionInfo.__init__c  sK   ø€ Ü‰ÑÔÜ�7‰7×Ñ×/Ñ/°¸Ô:Ø�WˆFØ�5ˆDØˆŒ	ØˆŒ	Øˆ�rF   c                óè  — | j                   €J ‚| j                   }|r|dk(  ry|r°i |¥}|j                  D ]‘  }t        |t        j                  «      sŒt        |t        j                  «      sJ ‚t        j                  j                  |j                  «      }t        |t        «      sJ ‚t        |j                  |«      «      ||<   Œ“ t        ||«      }t        j                  j!                  |«      S )Nr   úhl.Var())r_   Úfree_symbolsr   r   ÚTMPr9   r†   ÚSymbolr$   r\   Úlookup_cse_varrÛ   r£  r!   rÄ  r"   r~  )rL   rÃ  Ú	zero_varsr_   Úsymr�  s         rD   r¶  zDimensionInfo.index_strl  sÑ   € Ø�y‰yÐ$Ð$Ð$Ø�y‰yˆÙ˜ šØÙØ+˜lÐ+ˆLØ×(Ñ(ò W�Ü! #¤t§x¡xÕ0Ü% c¬5¯<©<Ô8Ð8Ð8ÜŸ(™(×1Ñ1°#·(±(Ó;�CÜ% cÔ+<Ô=Ð=Ð=Ü(:¸3¿<¹<ÈÓ;UÓ(V�L Ò%ðWô ˜d LÓ1ˆDÜ�x‰x×$Ñ$ TÓ*Ð*rF   rO   ©NF)rR   rS   rT   Ú__annotations__rK   r¶  rU   rV   s   @rD   rÉ  rÉ  ]  s   ø… à
ÓØ
ÓØÓõ÷+rF   rÉ  c                ó„  — t         j                  j                  j                  | |«      ry	 t         j                  j                  j	                  | «      }t         j                  j                  j	                  |«      }||k(  r*t         j                  j                  j                  | |«       ||k(  S # t
        $ r Y yw xY w©NTF)r$   rÎ  rÏ  Ústatically_known_equalsÚ	size_hintÚ	TypeErrorÚguard_equals)rm  rn  rŠ   r‹   s       rD   Úeqrá  }  s˜   € Ü‡w�w×Ñ×/Ñ/°°eÔ<ØðÜ�G‰G×Ñ×&Ñ& tÓ,ˆÜ�G‰G×Ñ×&Ñ& uÓ-ˆð 	ˆA‚vÜ	�‰×Ñ×%Ñ% d¨EÔ2Ø�‰6€Møô	 ò Ùðús   ­AB3 Â3	B?Â>B?c                óÈ  — t         j                  j                  j                  | |«      ry	 t         j                  j                  j	                  | «      }t         j                  j                  j	                  |«      }||k  r*t         j                  j                  j                  | |«       ||k  S # t
        $ r% t        j                  | |«      }|| k(  r| |k7  cY S Y yw xY wrÜ  )	r$   rÎ  rÏ  rÐ  rÞ  rß  r†   ÚgcdÚguard_lt)rm  rn  rŠ   r‹   rã  s        rD   rj  rj  Š  s¹   € Ü‡w�w×Ñ×+Ñ+¨D°%Ô8ØðÜ�G‰G×Ñ×&Ñ& tÓ,ˆÜ�G‰G×Ñ×&Ñ& uÓ-ˆð 	ˆ1‚uÜ	�‰×Ñ×!Ñ! $¨Ô.Øˆq‰5€Løô ò Ü�i‰i˜˜eÓ$ˆØ�$Š;Ø˜5‘=Ò Ùð	ús   ­AB3 Â3)C!Ã C!c                  ó˜  ‡ — e Zd ZU eZeZded<   	 	 	 	 d"ˆ fd„Zd#d„Z	d$d„Z
d%ˆ fd„Zd„ Zd„ Z	 	 d&ˆ fd	„Zd
„ Zd'd„Zd„ Zd„ Zd&d„Zd„ Zd(d„Zd)d„Zd*d„Z	 d+	 	 	 	 	 	 	 	 	 d,d„Z	 	 	 	 	 	 	 	 	 	 d-d„Zd„ Z	 	 	 	 	 	 	 	 d.d„Z ej:                  «       dœ	 d/d„Zd/d„Zd*d„Z d„ Z!d0d„Z"d+d„Z#e$d„ «       Z%d+d*d„Z&d „ Z'	 	 	 	 	 	 	 	 d1d!„Z(ˆ xZ)S )2ÚHalideKernelzCallable[[sympy.Expr], str]r�  c                óx  •— t        ‰| �  |fi |¤Ž | j                  | _        | j                  | _        | j                  | _        t        «       | _        | j                  | _	        | j                  | _
        i | _        i | _        i | _        i | _        i | _        i | _        t#        t$        «      | _        d| _        y rÙ  )rJ   rK   r–  ÚcomputeÚloadsÚstoresr)   Úindexing_code_domÚinside_reductionÚneeds_dom_indexingÚhas_reductionÚbuffer_dimensionsÚbuffer_offsetsÚhalide_varsÚindex_replacementsÚreduction_renamesÚdom_renamesr   ÚlistÚbuffer_aliasesÚhas_indirect_indexing)rL   Útilingr¬  rN   s      €rD   rK   zHalideKernel.__init__ž  s¤   ø€ ô
 	‰Ñ˜Ñ* 6Ò*à—y‘yˆŒØ—Y‘YˆŒ
Ø—i‘iˆŒÜ!/Ó!1ˆÔØ"&×"7Ñ"7ˆÔØ!×2Ñ2ˆÔØACˆÔØ57ˆÔà;=ˆÔà@BˆÔàCEˆÔàHJˆÔä4?ÄÓ4EˆÔØ%*ˆÕ"rF   c                ó   — t        |«      S r¼   )rÀ   )rL   r¿   s     rD   Údtype_to_strzHalideKernel.dtype_to_str¹  s   € Ü˜5Ó!Ð!rF   c                ó^   — | j                   j                  |› d|›d�«       t        |||«      S )Nz = hl.Func(r8   )r–  Ú	writeliner£  )rL   rÛ   r{  r¿   s       rD   Úcreate_cse_varzHalideKernel.create_cse_var¼  s0   € Ø�	‰	×Ñ˜t˜f K°¨x°qÐ9Ô:Ü   v¨uÓ5Ð5rF   c           
     óÜ  •‡‡‡— | j                   s| j                  s| j                  rJ ‚t        j                  t
        j                  j                  j                  t        ¬«      Št        j                  t        t        ‰ | �8  |«      «      }t        t            «       Št"        j$                  j'                  | j(                  D �cg c]  }|j*                  j-                  «       ‘Œ c}«      D �ci c]  }|j/                  «       |“Œ c}Šd„ }ˆˆfd„}ˆˆfd„}|D ]ì  }|j1                  t2        «      rV|j5                  t3        t7        j8                  d«      t7        j8                  d«      t7        j8                  d«      «      |«       |j1                  t:        «      rB|j5                  t;        t7        j8                  d«      t7        j8                  d«      «      |«       ‰j=                  t        ‰ | �9  |«      j>                  «       Œî tA        d„ ‰D «       «      | _!        d	}tE        | j(                  «      D �]%  }|j*                  j-                  «       D �cg c]  }|j/                  «       ‰v sŒ|‘Œ }	}|	jG                  ˆfd
„¬«       |	s+|	jI                  |jK                  d|jL                  «      «       d}
t6        jN                  jP                  }g }|
tS        |	«      k  �r®tU        |jL                  |«      �s—|	D �cg c]+  }tU        |jV                  |«      sŒ ||jX                  «      ‘Œ- }}|
tS        |«      z  }
|sJ |	«       ‚|t        jZ                  t
        j                  j                  j\                  |«      z  }|j_                  |	D �cg c]C  }ta        ||jV                  «      r+ta        |jV                  |«      r ||jV                  |z  «      ‘ŒE c}«       |�rˆt        jZ                  t6        jb                  |«      }tU        |d«      r2 ||jL                  |z  «      }tU        |d«      rJ ‚g }tS        |	«      }
d}te        dtS        | j                  «      › �«      }|jf                  r.te        dtS        | j                  «      › �«      | j                  |<   || j                  |<   |jI                  ||f«       ||z  }|	D �cg c]%  }tU        |jV                  |«      sŒ|jX                  ‘Œ' }}|
tS        |«      z  }
tS        |«      }|D �cg c]&  }tU        ||«      st7        jh                  ||z  «      ‘Œ( }}tS        |«      |k  s|dk(  sJ ‚|j_                  |«       |r�Œˆ|
tS        |	«      k  rtU        |jL                  |«      s�Œ—|	D ]Æ  }	 d}d}tU        |jV                  |«      s)||   \  }}|dz  }||z  }tU        |jV                  |«      sŒ)d}t6        jN                  jj                  }tU        |jX                  |«      s1||   \  }}|dz  }|||z  z  }||z  }tU        |jX                  |«      sŒ1|| j                   |j/                  «       <   ŒÈ �Œ( | j                  D ]-  }| jp                  js                  |› d|jt                  ›d�«       Œ/ | j                  rL| jw                  d| j                  jy                  «       D ��ci c]  \  }}|| j                  |   “Œ c}}«       yyc c}w c c}w c c}w c c}w c c}w c c}w c c}w # tl        $ rÁ |sJ ‚t6        jN                  jj                  }t6        jN                  jP                  }|D ]  \  }}|||z  z  }||z  }Œ t
        j                  j                  jo                  t3        ||jV                  |jX                  «      | j                  «      | j                   |j/                  «       <   Y �ŒMw xY wc c}}w )añ  
        Hook called right before codegen with every index that will be
        used in the fused kernel.

        This populates self.halide_vars/index_replacements/reduction_renames which is an alternate indexing
        scheme that avoids using divide and modulus.  Instead of xindex/yindex/rindex
        we base indexing on a larger number of vars whose product combines to those.

        This function populates self.halide_vars, self.index_replacements, and self.reduction_renames
        ©Úfallbackc                óz   — t        j                  t        j                  j                  j                  | «      «      S r¼   )r†   Úsimplifyr$   rÎ  rÏ  Úremove_precomputed_replacementsr^   s    rD   r  z0HalideKernel.finalize_indexing.<locals>.simplifyØ  s+   € Ü—>‘>Ü—‘× Ñ ×@Ñ@ÀÓFóð rF   c                ó  •— | ‰v rˆ‰|    }‰j                  |j                  j                  |j                  |z  t        j
                  j                  j                  |t        |j                  |«      «      «      j                  «       «       y y r¼   )ÚaddÚrootÚlookupÚdivisorr$   rÎ  rÏ  Úevaluate_minr   ÚlengthÚsymbol)Úbaser  ÚmodulusÚnodeÚall_used_symbolsÚsym_to_nodes       €€rD   Úvisit_modular_indexingz>HalideKernel.finalize_indexing.<locals>.visit_modular_indexingÝ  sw   ø€ Ø�{Ñ"Ø" 4Ñ(�Ø ×$Ñ$Ø—I‘I×$Ñ$ØŸ™ wÑ.ÜŸ™×(Ñ(×5Ñ5Ø#¤X¨d¯k©k¸7Ó%Cóó÷
 ‘f“hõð #rF   c           	     óÎ   •— | ‰v r`‰|    }‰j                  |j                  j                  |j                  |z  t	        |j
                  |«      «      j                  «       «       y y r¼   )r  r  r  r  r   r
  r  )r  r  r  r  r  s      €€rD   Úvisit_floor_divz7HalideKernel.finalize_indexing.<locals>.visit_floor_divé  s]   ø€ Ø�{Ñ"Ø" 4Ñ(�Ø ×$Ñ$Ø—I‘I×$Ñ$ØŸ™ wÑ.Ü  §¡¨gÓ6ó÷ ‘f“hõ	ð #rF   r  r  r  c              3  óP   K  — | ]  }t        |t        j                  «      –— Œ  y ­wr¼   )r   r   ÚINDIRECT)r¾  rØ  s     rD   r¿  z1HalideKernel.finalize_indexing.<locals>.<genexpr>  s    è ø€ ò )
Ø36ŒN˜3¤§¡×.ñ)
ùrÀ  Fc                ó(   •—  ‰| j                   «      S r¼   )r  )r¶   rÞ  s    €rD   ú<lambda>z0HalideKernel.finalize_indexing.<locals>.<lambda>  s   ø€ ¡Y¨q¯y©yÓ%9€ rF   ©Úkeyr%   r   TÚhÚhrz
 = hl.Var(r8   ÚrdomN)=rò  rñ  ró  Ú	functoolsÚpartialr$   rÎ  rÏ  rÞ  r   ÚdictÚfromkeysr´  rJ   r|  r   r   r§  r¨  Úfrom_iterableÚrange_treesÚnodesr©  r  Úhasr   Úreplacer†   ÚWildr   rª  rÓ  Úanyr÷  ÚreversedÚsortÚappendr  ÚnumelÚSÚOnerk   rá  r  r
  ÚreduceÚevaluate_maxÚextendrj  rã  r!   Úis_reductionr  ÚZeroÚ
IndexErrorÚsimplify_with_rangesÚindexing_coderü  rÛ   Úcodegen_rdomÚitems)!rL   ÚindicesÚtreer¶   r  r  r  r€  Úhad_fallbackr#  Úhandled_countr  Úadded_sym_sizeÚsizes_to_addr�  Ú	next_sizerØ  Ú	new_sizesÚ	prior_lenÚsr  Úidxrˆ  r
  r_   Ú
full_indexrË  ÚvÚrvr  rÞ  r  rN   s!                                @@@€rD   Úfinalize_indexingzHalideKernel.finalize_indexingÀ  sý  û€ ð ×#Ò# t×'7Ò'7¸4×;QÒ;Qð	
ð 
ô ×%Ñ%¤a§g¡g×&6Ñ&6×&@Ñ&@Ì3ÔOˆ	Ü—-‘-¤¤E¡GÑ$<¸gÓ FÓGˆÜ%¤c™?Ó,Ðô —_‘_×2Ñ2Ø15×1AÑ1AÖB¨�—‘×"Ñ"Õ$ÒBóö
àð �H‰H‹J˜‰Mò
ˆò	õ

	õ	ð ò 	RˆEØ�y‰yœÔ)Ø—‘Ü#ÜŸ
™
 6Ó*ÜŸ
™
 9Ó-ÜŸ
™
 9Ó-óð
 +ôð �y‰yœÔ"Ø—‘ÜÜŸ
™
 6Ó*ÜŸ
™
 9Ó-óð $ôð ×#Ñ#¤E¡GÑ$<¸UÓ$C×$PÑ$PÕQð%	Rô( &)ñ )
Ø:Jô)
ó &
ˆÔ"ð ˆÜ˜T×-Ñ-Ó.ó S	ˆDØ $§
¡
× 1Ñ 1Ó 3ÖV˜1°q·x±x³zÐEUÒ7U’QÐVˆEÐVØ�J‰JÓ9ˆJÔ:ÙØ—‘˜TŸ[™[¨¨D¯J©JÓ7Ô8ØˆMÜ—g‘g—k‘kˆGØˆNà¤# e£*Ó,´R¸¿
¹
ÀGÕ5Là05ö Ø+,¼¸A¿I¹IÀwÕ9O‘H˜QŸX™XÕ&ð �ð  ð ¤ \Ó!2Ñ2�Ù#Ð* UÓ*�|Ø¤	× 0Ñ 0Ü—G‘G×$Ñ$×1Ñ1°<ó!ñ �ð ×#Ñ#ð "'öàÜ˜g q§y¡yÔ1´b¸¿¹ÀCÔ6Hñ ! §¡¨WÑ!4Õ5òôò #Ü )× 0Ñ 0´·±¸LÓ I�IÜ˜) QÔ'ñ %-¨T¯Z©Z¸'Ñ-AÓ$B˜	Ü#% i°Ô#3Ð3Ð3Ø')˜Ü(+¨E«
˜Ø'+˜Ü,¨q´°T×5EÑ5EÓ1FÐ0GÐ-HÓI�CØ×(Ò(Ü6HØ ¤ T×%5Ñ%5Ó!6Ð 7Ð8ó7˜×.Ñ.¨sÑ3ð -6�D×$Ñ$ SÑ)Ø"×)Ñ)¨3°	Ð*:Ô;Ø˜yÑ(�GØ38Ö S¨a¼B¸q¿y¹yÈ'Õ<R §£Ð S�IÐ SØ!¤S¨£^Ñ3�MÜ # LÓ 1�Ið ".ö$àÜ! ! YÔ/ô Ÿ™ q¨9¡}Õ5ð$�Lð $ô
 ˜|Ó,¨yÒ8¸IÈºNÐJÐJØ ×'Ñ'¨	Ô2ó7 #ð!  ¤# e£*Ò,´R¸¿
¹
ÀGÖ5Lð\ ò �ðØ�CØ�GÜ  §¡¨wÔ7Ø$2°3Ñ$7™	˜˜TØ˜q™˜Ø 4™˜ô ! §¡¨wÕ7ð �FÜ Ÿ7™7Ÿ<™<�DÜ  §¡¨fÔ5Ø$2°3Ñ$7™	˜˜TØ˜q™˜Ø ¨¡Ñ,˜Ø $™˜ô	 ! §¡¨fÕ5ð
 >B�D×+Ñ+¨D¯K©K«MÒ:òðoS	ðl ×#Ñ#ò 	JˆCØ×Ñ×(Ñ(¨C¨5°
¸3¿8¹8¸,ÀaÐ)HÕIð	Jà×!Ò!Ø×ÑØØ6:×6LÑ6L×6RÑ6RÓ6T×U©U¨Q°��T×%Ñ% aÑ(Ñ(ÓUõð "ùòc Cùò
ùòz Wùò ùòùò0 !Tùò$øô2 "ò Ù'Ð'˜<Ü!&§¡§¡�JÜ"ŸW™WŸ[™[�FØ%3ò '™	˜˜TØ" f¨s¡lÑ2˜
Ø $™™ð'ô Ÿ™×(Ñ(×=Ñ=Ü+¨J¸¿¹ÀdÇkÁkÓRØ ×,Ñ,óð ×+Ñ+¨D¯K©K«MÔ:ðüó( Vsd   Ã![8Ã1[=É\É2\Ì\Ì\ÎA\
Ó\Ó+\Ô+\ÖA\×A"\Ù\Û_(ÜC_%ß$_%c           
     ó  — | j                   rdnd}|| j                  v r| j                  |   S i }| j                  j                  «       D ]c  }| j                   s|| j                  v rŒt        j                  d|j                  «      }|sJ ‚t        d|› |j                  d«      › �«      ||<   Œe | j                  |› d�|j                  «       D ��ci c]  \  }}|| j                  |   “Œ c}}«       || j                  |<   |S c c}}w )zCRDom based indexing uses explicit iteration ranges for Func updatesÚiÚoz^h(\d+)$r  r%   Údom)rì  rô  rñ  Úkeysró  rÅ  ÚmatchrÛ   r!   Úgroupr6  r7  )rL   ÚprefixÚrenamesr�  ÚmrD  rE  s          rD   Úsetup_dom_indexingzHalideKernel.setup_dom_indexingk  s  € à×-Ò-‘°3ˆØ�T×%Ñ%Ñ%Ø×#Ñ# FÑ+Ð+àˆØ×#Ñ#×(Ñ(Ó*ò 	HˆCØ×(Ò(¨S°D×4JÑ4JÑ-JØÜ—‘˜ c§h¡hÓ/ˆAÙˆH�1Ü-°°&°¸!¿'¹'À!»*¸Ð.FÓGˆG�CŠLð	Hð 	×ÑØˆh�cˆNÀ'Ç-Á-Ã/×R¹¸¸B˜R ×!1Ñ!1°!Ñ!4Ñ4ÓRô	
ð $+ˆ×Ñ˜Ñ Øˆùó	 Ss   ÃDc           	     óv  — |j                  «       D �cg c]&  }d| j                  | j                  |«      «      › d�‘Œ( }}| j                  j	                  |› ddj                  |«      › d�«       t        |j                  «       «      D ])  \  }}| j                  j	                  |› d|› d|› d�«       Œ+ y c c}w )	Núhl.Range(0, r8   z = hl.RDom([r[   ú])ú = r±  r²  )r©  r�  rŽ  r5  rü  r³  Ú	enumeraterK  )rL   rÛ   Úvarsrˆ  ÚrsizesrH  Úrsyms          rD   r6  zHalideKernel.codegen_rdom€  s¾   € ð Ÿ™›ö
àð ˜4Ÿ:™: d×&:Ñ&:¸4Ó&@ÓAÐBÀ!ÒDð
ˆð 
ð 	×Ñ×$Ñ$¨ v¨\¸$¿)¹)ÀFÓ:KÐ9LÈBÐ%OÔPÜ  §¡£Ó-ò 	B‰GˆAˆtØ×Ñ×(Ñ(¨D¨6°°T°F¸!¸A¸3¸aÐ)@ÕAñ	Bùò
s   “+B6c                ó¶   •— t         ‰| �  |«      }t        || j                  «      }t        j
                  j                  j                  || j                  «      S r¼   )	rJ   r|  r"   rò  r$   rÎ  rÏ  r4  rñ  )rL   r€  rN   s     €rD   r|  zHalideKernel.prepare_indexing‰  sI   ø€ ô ‘Ñ(¨Ó/ˆÜ˜5 $×"9Ñ"9Ó:ˆÜ�w‰w×Ñ×4Ñ4°U¸D×<LÑ<LÓMÐMrF   c                óž   — t        |t        j                  «      r%| j                  |j                  «      j
                  S | j                  |   S )zThe size of an index symbol)r   r   rÔ  rÖ  rÛ   r…  rñ  )rL   rØ  s     rD   Úsym_sizezHalideKernel.sym_size‘  s<   € ä˜#œtŸx™xÔ(Ø×&Ñ& s§x¡xÓ0×GÑGÐGØ×Ñ Ñ$Ð$rF   c           	     ó	  ‡ ‡‡— g Št        |j                  d„ ¬«      D ]~  }t        |t        j                  t        j
                  f«      r‰j                  |«       Œ?t        |t        j                  t        j                  t        j                  f«      rŒyJ |«       ‚ t        j                  j                  }‰D �ci c]  }|t        j                  j                  “Œ }}g }t        j                  ‰ j                  |«      «      }t        |t        j                   «      r|j"                  n|gD ]Ù  }	|	j                  D �
cg c]	  }
|
|v sŒ|
‘Œ }}
t%        |«      dk(  r||	z  }Œ5t%        |«      dk(  r||d   xx   |	z  cc<   ŒTg }t'        t%        |«      «      D ]e  }||   €J ‚||   \  }}t)        |«      t)        |«      z  r*|j+                  |D �
cg c]	  }
|
|vsŒ|
‘Œ c}
«       |	|z  }	ŒS|j                  ||f«       Œg g |¢||	f‘}ŒÛ ˆˆ ˆfd„}g }|D ]8  \  }}|D ]  }
||j-                  |
«      z  }Œ |j                   |||«      «       Œ: |j/                  «       D ]  \  }}|j                   |||g«      «       Œ  |j1                  d„ ¬«       |sA‰ j2                  r²|j                  t5        t        j                  j                  dd«      «       n}t6        j8                  j:                  j=                  |d   j>                  d«      sF|jA                  dt5        t        j                  j                  ‰rdn|d   j>                  d«      «       |r·‰sµ|‰ jB                  v rit6        j8                  j:                  jE                  |‰ jB                  |   «      r2‰ jG                  ||‰ jB                  |   z
  «       ‰ jB                  |   }n>t6        j8                  j:                  jI                  |d«      r‰ jG                  ||«       d}|}tK        jL                  «       D ]W  }‰ jO                  |||‰«      r||fc S ‰rJ ‚|› d|› �}|‰ jP                  |   vsŒ:‰ jP                  |   j                  |«       ŒY yc c}w c c}
w c c}
w )	zEConvert address-based indexing into dimensions using self.halide_varsc                ó   — | j                   S r¼   rf  r×   s    rD   r  z5HalideKernel.indexing_to_dimensions.<locals>.<lambda>š  s
   € ¸A¿F¹F€ rF   r  r   r%   Nc                ó   •— t        j                  | «      } t        |«      dk(  rUt        j                  d‰
¬«      }| j	                  ||d   z  «      }|r%t        |d   ‰	j                  |d   «      ||   «      S ‰rJ | «       ‚t        j                  t        | |D �ci c]  }|‰	j                  |«      dz
  “Œ c}«      dz   «      }t         j                  j                  }t        | t         j                  «      rt| j                  D ]e  }t        |t         j                  «      sŒ||z  }t        j                  | |z  «      } t        j                  t        j                  ||z  «      «      }Œg t        | ||«      S c c}w )Nr%   Úwild)Úexcluder   )r†   Úfactorrk   r&  rL  rÉ  r\  r  r"   r,  r-  r9   ÚMulrl   r�  Úceiling)r_   ÚsymsÚstride_wildrP  rØ  r
  rË  ÚtermÚis_storerL   Úsymbolss           €€€rD   Úexpr_to_dimensionz>HalideKernel.indexing_to_dimensions.<locals>.expr_to_dimension¾  sG  ø€ Ü—<‘< Ó%ˆDÜ�4‹y˜AŠ~Ü#Ÿj™j¨¸ÔA�Ø—J‘J˜{¨T°!©WÑ4Ó5�ÙÜ(Ø˜Q™ §¡¨t°A©wÓ!7¸¸;¹óð ñ  Ð% Ó%�<Ü—^‘^Ü˜4ÈÖ!NÀ# # t§}¡}°SÓ'9¸AÑ'=Ñ"=Ò!NÓOÐRSÑSóˆFô —W‘W—[‘[ˆFÜ˜$¤§	¡	Ô*Ø ŸI™Iò N�DÜ! $¬¯©Õ6Ø $™˜Ü$Ÿ~™~¨d°T©kÓ:˜Ü!&§¡´·±¸fÀt¹mÓ0LÓ!M™ð	Nô
 !  v¨vÓ6Ð6ùò "Os   ÂE;c                ót   — t         j                  j                  j                  | j                  t
        ¬«      S )Nrÿ  )r$   rÎ  rÏ  rÞ  rË  r   )Úds    rD   r  z5HalideKernel.indexing_to_dimensions.<locals>.<lambda>Ü  s$   € ¤§¡× 0Ñ 0× :Ñ :¸1¿8¹8ÌcÐ :Ó R€ rF   Ú_view))ÚsortedrÓ  r   r   ÚHALIDErÔ  r*  ÚUNBACKED_INTÚSIZEÚPRECOMPUTED_SIZEr†   r,  r2  ÚexpandrŽ  r9   ÚAddrl   rk   Úranger   r0  Úpopr7  r)  r÷  rÉ  r$   rÎ  rÏ  rÝ  rË  Úinsertrð  Ústatically_known_geqÚapply_offset_to_dimensionÚstatically_known_gtr§  ÚcountÚinstall_dimsrö  )rL   r�  r€  rh  rØ  r:  rA  Ú
split_exprÚsplit_failedÚpartrD  Ú	part_varsÚnew_split_failedrH  Ú
other_varsÚ
other_partrj  rµ  re  r_   Úorig_varri  s   `  `                 @rD   Úindexing_to_dimensionsz#HalideKernel.indexing_to_dimensions—  s5  ú€ àˆÜ˜%×,Ñ,Ñ2BÔCò 	ˆCÜ˜c¤D§K¡K´·±Ð#:Ô;Ø—‘˜sÕ#ä%Øä×)Ñ)ÜŸ	™	Ü×-Ñ-ðõð ð óð ð		ô —‘—‘ˆØ/6Ö7¨!�aœŸ™Ÿ™‘oÐ7ˆ
Ð7ØDFˆÜ—‘˜T×1Ñ1°%Ó8Ó9ˆÜ",¨U´E·I±IÔ">�E—J’JÀUÀGò 	FˆDØ$(×$5Ñ$5ÖI˜q¸¸jºšÐIˆIÐIÜ�9‹~ Ò"Ø˜$‘‘Ü�Y“ 1Ò$Ø˜9 Q™<Ó(¨DÑ0Ô(à#%Ð Üœs <Ó0Ó1ò J�AØ'¨™?Ð6Ð6Ð6Ø-9¸!©_Ñ*�J 
Ü! *Ó-´
¸9Ó0EÒEØ!×(Ñ(°ZÖ)V°À1ÈIÒCUª!Ò)VÔWØ 
Ñ*™à(×/Ñ/°¸ZÐ0HÕIðJð  FÐ!1ÐE°I¸tÐ3DÐE‘ð!	Fö$	7ð. ˆØ&ò 	7‰JˆD�$Øò *�Ø˜
Ÿ™ qÓ)Ñ)‘ð*à�K‰KÑ)¨$°Ó5Õ6ð	7ð $×)Ñ)Ó+ò 	8‰IˆC�Ø�K‰KÑ)¨$°°Ó6Õ7ð	8à�	‰	ÑRˆ	ÔSáØ×)Ò)à—‘œM¬%¯'©'¯,©,¸¸1Ó=Õ>Ü—‘×!Ñ!×9Ñ9¸$¸q¹'¿.¹.È!ÔLà�K‰KØ”=¤§¡§¡±H©qÀ$ÀqÁ'Ç.Á.ÐRSÓTôñ ™Ø�d×)Ñ)Ñ)¬a¯g©g×.>Ñ.>×.SÑ.SØ˜×+Ñ+¨CÑ0ô/ð ×.Ñ.¨t°V¸d×>QÑ>QÐRUÑ>VÑ5VÔWØ×,Ñ,¨SÑ1‘Ü—‘×!Ñ!×5Ñ5Ø˜ôð ×.Ñ.¨t°VÔ<Ø�àˆÜ—‘Ó"ò 	:ˆAØ× Ñ   d¨F°HÔ=Ø˜D�yÒ ÙÐ�<Ø�J˜e A 3Ð'ˆCØ˜$×-Ñ-¨hÑ7Ò7Ø×#Ñ# HÑ-×4Ñ4°SÕ9ñ	:ùò] 8ùò Jùò *Ws   Â="R Å	RÅRÇ	R
ÇR
c                óf  — || j                   vr|| j                   |<   || j                  |<   y| j                  |   |k7  s$t        | j                   |   «      t        |«      k7  ry|r| j                   |   |k(  S t        | j                   |   |«      D ]�  \  }}|j                  |j                  k7  r y|j
                  |j
                  k7  s|j                  |j                  k7  sŒTt        j                  j                  j                  |j
                  |j
                  «      |_        d|_        ŒŸ y)z>Try to set self.buffer_dimensions[var], return True on successTFN)rï  rð  rk   ÚziprË  rˆ  r_   r$   rÎ  rÏ  r/  )rL   r�  rµ  r:  rh  ÚoldÚnews          rD   r|  zHalideKernel.install_dimsÿ  s  € à�d×,Ñ,Ñ,Ø*.ˆD×"Ñ" 3Ñ'Ø'-ˆD×Ñ Ñ$ØØ×Ñ˜sÑ# vÒ-´Ø×"Ñ" 3Ñ'ó2
ä�‹Yò2ð ÙØ×)Ñ)¨#Ñ.°$Ñ6Ð6Ü˜D×2Ñ2°3Ñ7¸Ó>ò 	 ‰HˆC�Ø�z‰z˜SŸZ™ZÒ'ÙØ�x‰x˜3Ÿ8™8Ò# s§x¡x°3·8±8Ó';ÜŸ7™7×+Ñ+×8Ñ8¸¿¹À3Ç8Á8ÓL�”Ø�•ð	 ð rF   c                ó‚  — |dk(  ry t        t        t        |«      «      «      D ]’  }||   j                  dk(  s8t        j
                  j                  j                  |||   j                  «      sŒMt        |||   j                  «      }||||   j                  z  z  }||   xj                  |z  c_	        Œ” |dk(  sJ ‚y )Nr   r%   )
r(  ru  rk   rË  r$   rÎ  rÏ  rx  r   r_   )rL   rµ  r:  rH  r  s        rD   ry  z&HalideKernel.apply_offset_to_dimension  sª   € Ø�QŠ;ØÜœ%¤ D£	Ó*Ó+ò 	%ˆAØ�A‰w�~‰~ Ò"¤a§g¡g×&6Ñ&6×&KÑ&KØ˜˜Q™Ÿ™õ'ô   ¨¨Q©¯©Ó7�Ø˜$  a¡§¡Ñ/Ñ/�Ø�Q‘—’ Ñ$–ð	%ð ˜Š{Ð‰{rF   c                óš  — t        t        j                     «       }|j                  D �]  }t	        |t        j                  «      sJ ‚t        |t        j                  «      rU| j                  |j                  «      }t	        |t        «      r|j                  €J ‚|j                  |j                  «       Œ�t        |t        j                  «      r|j                  |«       Œ»t        |t        j                  t        j                   t        j"                  t        j$                  f«      r�Œt'        d|› �«      ‚ | j)                  |«      S )zIDetect which range trees are used to populate HalideCSEVariable.used_dimszunhandled symbol )r   r†   rÕ  rÓ  r9   r   r   rÔ  rÖ  rÛ   r£  r¥  rª  ro  r  rp  rq  rr  ÚINDEXr�  r«  )rL   r€  r¥  rØ  Úcse_vars        rD   r  z!HalideKernel.used_dims_from_index  s  € äœuŸ|™|Ñ,Ó.ˆ	Ø×%Ñ%ó 	EˆCÜ˜c¤5§<¡<Ô0Ð0Ð0Ü˜c¤4§8¡8Ô,à×-Ñ-¨c¯h©hÓ7�ä˜wÔ(9Ô:Ø×)Ñ)Ð5ðð6ð × Ñ  ×!2Ñ!2Õ3Ü ¤T§[¡[Ô1Ø—‘˜cÕ"ÜØ”d×'Ñ'¬¯©´D×4IÑ4IÌ4Ï:É:ÐVôñ ä)Ð,=¸c¸UÐ*CÓDÐDð#	Eð$ ×"Ñ" 9Ó-Ð-rF   c                óü   — t        d„ |D «       «      sJ ‚t        j                  | j                  | j                  j                  «       «      D �cg c]  }||v r|‘Œ
 }}t        |«      t        |«      k(  sJ ‚|S c c}w )Nc              3  óP   K  — | ]  }t        |t        j                  «      –— Œ  y ­wr¼   r»  r½  s     rD   r¿  z.HalideKernel.sort_used_dims.<locals>.<genexpr>7  s   è ø€ Ò@°”:˜a¤§¡×,Ñ@ùrÀ  )rÁ  r§  r¨  rñ  ró  r©  rk   )rL   r¥  rØ  Úordereds       rD   r«  zHalideKernel.sort_used_dims6  s„   € ÜÑ@°iÔ@Ô@Ð@Ð@ô !—‘Ø× Ñ  $×"8Ñ"8×"?Ñ"?Ó"Aóö
àð �iÑò	 ð
ˆð 
ô �7‹|œs 9›~Ò-Ð-Ð-Øˆùò
s   ÁA9c                óŠ   ‡‡— dj                  ˆˆfd„|D «       «      }t        |«      dk(  rd}|S t        |«      dk(  r|› d�}|S )Nr[   c              3  óB   •K  — | ]  }|j                  ‰‰«      –— Œ y ­wr¼   )r¶  )r¾  rl  rÃ  r×  s     €€rD   r¿  z.HalideKernel.make_index_str.<locals>.<genexpr>C  s   øè ø€ ÒQÀq˜aŸk™k¨,¸	×BÑQùs   ƒr   ú()r%   ú,)r³  rk   )rL   rµ  rÃ  r×  r¶  s     `` rD   Úmake_index_strzHalideKernel.make_index_strB  sM   ù€ Ø—I‘IÔQÈDÔQÓQˆ	Üˆt‹9˜Š>ØˆIð Ðô �‹Y˜!Š^à$˜+ Q˜ˆIØÐrF   c                ó"  — | j                   j                  |«      }| j                  |«      }| j                  ||d«      \  }}|› d| j	                  |«      › d�}t
        j                  j                  |«      }|t        j                  t        j                  fv rt        j                  }d|› d�}| j                  �r²t        | j                  t        «      r| j                  j                  €J ‚t!        g | j#                  |«      ¢| j                  j                  ¢­«      }| j%                  | j'                  |«      «      }|j                  rå| j(                  j+                  |j,                  › d�«       | j(                  j+                  |j,                  › d| j                  › d�«       | j/                  | j0                  xs d«      }	| j(                  j+                  |› d	t3        |«      › d
|	› d�«       | j(                  j+                  |› d|› dt3        |«      › d
|j,                  › d�«       |S | j(                  j+                  |› d| j                  › d
|› dt3        |«      › d�«       |S | j5                  || j#                  |«      «      S )z"Codegen a load from an InputBufferFr±  r²  rb   r8   z!_mask = hl.RDom([hl.Range(0, 1)])z_mask.where(r   z = hl.cast(r[   rU  z + hl.cast(z_mask)z = hl.select(z
, hl.cast(z, 0)))rl   Úinputr|  r…  r•  r$   rÎ  Ú	get_dtyper;   rÄ   rÅ   rÆ   Ú
_load_maskr9   r£  r¥  r   r  Únewfuncr«  r–  rü  rÛ   r�  Ú_load_otherrÀ   r}  )
rL   rÛ   r€  r�  rµ  rÐ   r¿   r¥  r™  r—  s
             rD   rF  zHalideKernel.loadK  sL  € à�i‰i�o‰o˜dÓ#ˆØ×%Ñ% eÓ,ˆØ×/Ñ/°°U¸EÓB‰	ˆˆTØ��a˜×+Ñ+¨DÓ1Ð2°!Ð4ˆÜ—‘×!Ñ! $Ó'ˆØ”U—]‘]¤E§N¡NÐ3Ñ3Ü—M‘MˆEØ+¨D¨6°Ð3ˆDà�?‹?ä˜4Ÿ?™?Ô,=Ô>Ø—O‘O×-Ñ-Ð9ðð:ô #ØO�$×+Ñ+¨EÓ2ÐO°T·_±_×5NÑ5NÑOóˆIð —\‘\ $×"5Ñ"5°iÓ"@ÓAˆFØ×ÒØ—	‘	×#Ñ# v§{¡{ mÐ3TÐ$UÔVØ—	‘	×#Ñ# v§{¡{ m°<ÀÇÁÐ?PÐPQÐ$RÔSØŸ
™
 4×#3Ñ#3Ò#8°qÓ9�Ø—	‘	×#Ñ#Ø�h˜k¬+°eÓ*<Ð)=¸RÀ¸wÀaÐHôð —	‘	×#Ñ#Ø�h˜c $  {´;¸uÓ3EÐ2FÀbÈÏÉÈÐU[Ð\ôð ˆMð —	‘	×#Ñ#Ø�h˜m¨D¯O©OÐ+<¸B¸t¸fÀJÌ{Ð[`ÓOaÐNbÐbgÐhôð ˆMà—<‘<  d×&?Ñ&?ÀÓ&FÓGÐGrF   c                ó^   — | j                   j                  t        j                  dd|«         S )Nz\[.*Ú )ÚcseÚvarname_maprÅ  rl  ©rL   rÛ   s     rD   rÖ  zHalideKernel.lookup_cse_varr  s$   € Ø�x‰x×#Ñ#¤B§F¡F¨7°B¸Ó$=Ñ>Ð>rF   c                ó(  — t        |t        «      sJ ‚| j                  j                  |«      }| j	                  |«      }| j                  ||d«      \  }}| j                  |«      s|�„| j                  «       }| j                  ||«      }|j                  |«      }	dj                  dgt        |«      z  «      xs d}
| j                  j                  t        ||› d|
› d|› d�«      «       n| j                  |d¬	«      }t        |«      }	t         j"                  j%                  |«      }|€|› d|› d
t'        |«      › d|	› d�}n+|dk(  r|› d|› dt'        |«      › d|	› d�}nt)        d|› �«      ‚| j                  j                  t        ||«      «       y)z"Codegen a store to an OutputBufferTNr[   rÒ  r“  r±  z] = hl.undef(z.type()))r×  z] = hl.cast(r8   Ú
atomic_addz] += hl.cast(zstore mode=)r9   r£  rl   Úoutputr|  r…  Úis_indirect_indexingrQ  r•  rÄ  r³  rk   r–  rü  r(   r†  r$   rÎ  r˜  rÀ   r�  )rL   rÛ   r€  rÔ   Úmoder�  rµ  rÃ  r¶  Ú	value_strÚ
undef_dimsr¿   rÐ   s                rD   ÚstorezHalideKernel.storeu  s•  € ô ˜%Ô!2Ô3Ð3Ð3Ø�i‰i×Ñ˜tÓ$ˆØ×%Ñ% eÓ,ˆØ×/Ñ/°°U¸DÓA‰	ˆˆTØ×$Ñ$ UÔ+¨tÐ/?Ø×2Ñ2Ó4ˆLØ×+Ñ+¨D°,Ó?ˆIØŸ™ |Ó4ˆIØŸ)™) Z L´3°t³9Ñ$<Ó=ÒFÀ$ˆJØ�I‰I×ÑÜ˜T c U¨!¨J¨<°}ÀSÀEÈÐ#RÓSõð ×+Ñ+¨D¸DÐ+ÓAˆIÜ˜E›
ˆIä—‘×!Ñ! $Ó'ˆØˆ<Ø�U˜!˜I˜; l´;¸uÓ3EÐ2FÀbÈÈÐSTÐU‰DØ�\Ò!Ø�U˜!˜I˜; m´KÀÓ4FÐ3GÀrÈ)ÈÐTUÐV‰Dä%¨°D°6Ð&:Ó;Ð;Ø�	‰	×ÑœL¨¨tÓ4Õ5rF   c           	     ó^  — | j                   sJ ‚| j                  rJ ‚|||f}|| j                  j                  v r| j                  j                  |   S t	        |t
        «      r1|dk(  sJ ‚ | j                  |Ž x| j                  j                  |<   }|S t	        |t        «      r|j                  €J ‚t        | j                  «      }| j                  |j                  D �cg c]	  }||vsŒ|‘Œ c}«      }	|t        |j                  «      z
  r:| j                  |› | j                  t        g |j                  ¢|¢­«      «      «      }|j                  | j                  «      }
t        j                   j#                  ||«      }t%        |«      }|dv rÄ|	j&                  › d|› �}| j(                  j+                  |› d|› d|
› d�«       g }d}t-        | j                  «      D ]C  \  }}|j/                  |› d	|› d
�«       |dk7  r|dxx   d|› �z  cc<   || j0                  |   z  }ŒE | j(                  j+                  |	› ddj3                  |«      › �«       n®|dk(  r| j5                  ||«      }	n–t7        ||«      }t9        j:                  t=        t?        «       «      «      5   ||	|
«      }ddd«       d|› dtA        |«      › d�}| j(                  j+                  |	› d|› �«       | j(                  j+                  |	› d› �«       |	| j                  j                  |<   |	S c c}w # 1 sw Y   Œ{xY w)zCodegen a reduction operationÚwelford_combineN)ÚargmaxÚargminÚ_z = hl.z(rdom, r8   r%   r±  r²  éÿÿÿÿÚ*rU  rC  Úwelford_reducerZ   r[   )!rì  r™  rž  Úreduction_cacher9   ÚtupleÚwelford_combine_implr£  r¥  r   ró  rš  r}  r«  rÄ  r   Ú	ReductionÚdefault_accumulatorrÇ   rÛ   r–  rü  rV  r*  rñ  r³  Úwelford_reduce_fallbackr   r$   Úset_ops_handlerr   rÉ   rE   )rL   r¿   rÌ   Úreduction_typerÔ   Ú	cache_keyÚresult_tupleÚreduction_varsrD  Ú
result_varr¦  ÚdefaultÚacc_typer€  ÚpartsrË  rH  rØ  Ú
combine_fnÚcombine_strÚdefault_strs                        rD   Ú	reductionzHalideKernel.reduction’  s!  € ð ×$Ò$Ð$Ð$Ø—?’?Ð"Ð"Ø °Ð6ˆ	Ø˜Ÿ™×0Ñ0Ñ0Ø—8‘8×+Ñ+¨IÑ6Ð6ä�eœUÔ#Ø!Ð%6Ò6Ð6Ð6à)�×)Ñ)¨5Ð1ðˆD�H‰H×$Ñ$ YÑ/°,ð  Ðä˜%Ô!2Ô3¸¿¹Ð8SÐSÐSÜ# D×$:Ñ$:Ó;ˆØ—\‘\ØŸ™ÖC�1¨1°NÒ+BŠQÒCó
ˆ
ð œJ u§¡Ó7Ò7Ø—L‘LØ�'Ø×#Ñ#¤JÐ/R°·±Ð/RÀ>Ñ/RÓ$SÓTóˆEð —N‘N 4×#9Ñ#9Ó:ˆ	Ü—,‘,×2Ñ2°>À9ÓMˆÜ" 5Ó)ˆàÐ1Ñ1Ø!—‘Ð' q¨Ð(8Ð9ˆEØ�I‰I×Ñ 5 '¨°Ð/?¸wÀyÀkÐQRÐ SÔTàˆEØˆFÜ# D×$:Ñ$:Ó;ò 0‘��3Ø—‘ ˜w a¨ s¨!˜_Ô-Ø˜Q’;Ø˜"“I 1 V H Ñ-“IØ˜$×*Ñ*¨3Ñ/Ñ/‘ð	0ð
 �I‰I×Ñ : ,¨c°%·*±*¸UÓ2CÐ1DÐ EÕFØÐ/Ò/à×5Ñ5°e¸UÓC‰Jä1°.À(ÓKˆJÜ×"Ñ"¤?´?Ó3DÓ#EÓFñ @Ù(¨°YÓ?�÷@à$ X J¨b´ÀÓ1IÐ0JÈ!ÐLˆKØ�I‰I×Ñ : ,¨c°+°Ð ?Ô@Ø�I‰I×Ñ : ,¨c°+°Ð ?Ô@à.8ˆ�‰× Ñ  Ñ+ØÐùòG D÷8@ð @ús   Ã 	LÃ*LÊ
L#Ì#L,c                óà  — t        |t        «      r|j                  €J ‚t        |t        «      r|j                  €J ‚t        |t        «      r|j                  €J ‚t        g |j                  ¢|j                  ¢|j                  ¢­xs | j                  «      }|t        | j
                  «      z  }| j                  | j                  |«      «      }|||fD �cg c]  }d|j                  › d�‘Œ }}|j                  }| j                  j                  |› ddj                  |«      › d�«       | j                  j                  |› d|› d�«       | j                  j                  |› d|› d	�«       | j                  j                  |› d
|› d�«       | j                  j                  |› d|j                  | j
                  «      › �«       | j                  j                  |› d|j                  | j
                  «      › �«       | j                  j                  |› d|j                  | j
                  «      › �«       | j                  j                  |› d|› d|› d�«       | j                  j                  |› d|› d|› d�«       | j                  j                  |› d|› d|› d|› d�«       |› d|› d|› d�|› d|› d|› d|› d|› d|› d�|› d�g}	| j                  j                  |› ddj                  |	«      › d�«       g }
t        d «      D ]S  }|
j                  | j                  |j                  «      «       | j                  j                  |
d!   › d"|› d#|› d$�«       ŒU t        |
«      S c c}w )%NrZ   z.type(), 0)z = hl.Tuple([r[   rT  z
_mean_1 = z[0]z_m2_1 = z[1]z_weight_1 = z[2]z
_mean_2 = z_m2_2 = z_weight_2 = z	_delta = z
_mean_2 - Ú_mean_1z_new_weight = z_weight_1 + Ú	_weight_2z_w2_over_w = hl.select(z_new_weight == 0.0, 0.0, z_weight_2 / z_new_weight)z
_mean_1 + z	_delta * Ú
_w2_over_wz_m2_1 + z_m2_2 + z_weight_1 * Ú_new_weightr   r®  rU  r±  r²  )r9   r£  r¥  r   rñ  ró  rš  r«  rÛ   r–  rü  r³  rÄ  ru  r*  r²  )rL   ÚmeanÚm2Úweightr¥  r¼  r«   r½  Úpfxrª  ÚunpackedrH  s               rD   r³  z!HalideKernel.welford_combine_implÏ  se  € Ü˜$Ô 1Ô2°t·~±~Ð7QÐQÐQÜ˜"Ô/Ô0°R·\±\Ð5MÐMÐMÜ˜&Ô"3Ô4¸×9IÑ9IÐ9UÐUÐUÜØ?ˆd�n‰nÐ?˜rŸ|™|Ð?¨f×.>Ñ.>Ñ?ÒSÀ4×CSÑCSó
ˆ	ð 	”Z × 6Ñ 6Ó7Ñ7ˆ	Ø—\‘\ $×"5Ñ"5°iÓ"@ÓAˆ
Ø<@À"ÀfÐ;MÖN°a�X˜aŸf™f˜X [Ò1ÐNˆÐNØ�o‰oˆØ�	‰	×Ñ˜z˜l¨-¸¿	¹	À'Ó8JÐ7KÈ2ÐNÔOØ�	‰	×Ñ˜s˜e :¨j¨\¸Ð=Ô>Ø�	‰	×Ñ˜s˜e 8¨J¨<°sÐ;Ô<Ø�	‰	×Ñ˜s˜e <°
¨|¸3Ð?Ô@Ø�	‰	×Ñ˜s˜e :¨d¯m©m¸D×<RÑ<RÓ.SÐ-TÐUÔVØ�	‰	×Ñ˜s˜e 8¨B¯K©K¸×8NÑ8NÓ,OÐ+PÐQÔRØ�	‰	×ÑØˆe�< §¡°×0FÑ0FÓ GÐHÐIô	
ð 	�	‰	×Ñ˜s˜e 9¨S¨E°¸C¸5ÀÐHÔIØ�	‰	×Ñ˜s˜e >°#°°lÀ3À%ÀyÐQÔRØ�	‰	×ÑØˆeÐ*¨3¨%Ð/HÈÈÈ\ÐZ]ÐY^Ð^jÐkô	
ð ˆe�:˜c˜U )¨C¨5°
Ð;Øˆe�8˜C˜5 ¨¨¨Y°s°e¸9ÀSÀEÈÐVYÐUZÐZdÐeØˆe�;Ðð
ˆð
 	�	‰	×Ñ˜z˜l¨-¸¿	¹	À&Ó8IÐ7JÈ"ÐMÔNàˆÜ�q“ò 	GˆAØ�O‰O˜DŸL™L¨×)=Ñ)=Ó>Ô?Ø�I‰I×Ñ 8¨B¡< .°°J°<¸qÀÀÀ1Ð EÕFð	Gô �X‹Ðùò7 Os   ÃM+c           
     ó6  — | j                   sJ ‚t        |«      t        |«      k(  sJ ‚g }t        t        j                     «       }|D ]·  }t        |t        «      r|j                  €J ‚t        |j                  «      t        | j                  «      z  r|j                  |«       n?|j                  | j                  |› g |j                  ¢g | j                  ¢d d ‘«      «       |j                  |j                  «       Œ¹ | j                  | j                  |«      «      }|j                  r+t        |j                  «      t        | j                  «      z  sJ ‚t        ||«      D ��cg c]  \  }}dt        |«      › d|› d�‘Œ }	}}| j!                  | j#                  | j$                  d   j&                  «      «      }
|j(                  › d�}|› d�}| j*                  j-                  |› d|
› d	�«       t        | j                  «      dk(  sJ d
«       ‚g | j                  ¢\  }|t/        |«      i}|t/        |«      dz
  i}t        |«      dk(  r(d„ }|j1                  |«      g}|j1                  |«      g}nqd„ }t3        t        |«      «      D �cg c]  }|j1                  |«      d|› d�z   ‘Œ }}t3        t        |«      «      D �cg c]  }|j1                  |«      d|› d�z   ‘Œ }}| j*                  j-                  |› d ||	«      › �«       t5        j6                  t9        t;        «       «      «      5   |||«      }d d d «       | j*                  j-                  |j1                  |«      › d |«      › �«       t        |«      dk(  r|fS |D �cg c]"  }| j                  | j                  |«      «      ‘Œ$ }}t=        |«      D ])  \  }}| j*                  j-                  |› d|› d|› d�«       Œ+ t?        |«      S c c}}w c c}w c c}w # 1 sw Y   ŒÎxY wc c}w )Nr%   rZ   r[   r8   r®  Ú_rdomz.xz = hl.RDom([hl.Range(1, z)])z&multi-dimensional scan not implementedc                ó   — | d   S rÍ  rc   r×   s    rD   Úmaybe_tuplez&HalideKernel.scan.<locals>.maybe_tuple$  s   € Ø˜‘t�rF   c                ó,   — ddj                  | «      › d�S )Nz
hl.Tuple([r[   rT  )r³  r×   s    rD   rÑ  z&HalideKernel.scan.<locals>.maybe_tuple+  s   € Ø# D§I¡I¨a£L >°Ð4Ð4rF   r±  r²  rU  ) rì  rk   r   r†   rÕ  r9   r£  r¥  ró  r*  r}  rª  rš  r«  r‡  rÇ   r�  rŽ  r"  r+  rÛ   r–  rü  r!   rÄ  ru  r$   r·  r   rÉ   rV  r²  )rL   ÚdtypesrÀ  Úvalues_origr©  Úall_used_dimsrÔ   r¼  r¿   Úinitialr
  Úscan_domÚscanÚscan_varÚscan_renames_curÚscan_renames_prirÑ  Ú	read_leftÚ
read_rightrH  rÁ  r­  Úunpack_varsrD  s                           rD   rØ  zHalideKernel.scanõ  s+  € ð ×$Ò$Ð$Ð$Ü�6‹{œc +Ó.Ò.Ð.Ð.Ø*,ˆÜ"¤5§<¡<Ñ0Ó2ˆà ò 
	2ˆEÜ˜eÔ%6Ô7¸E¿O¹OÐ<WÐWÐWÜ˜%Ÿ/™/Ó*¬Z¸×8NÑ8NÓ-OÒOØ—‘˜eÕ$à—‘Ø—L‘LØ ˜'Ð$U e§o¡oÐ$UÐ7P¸×9OÑ9OÐ7PÐQSÐRSÐ7TÐ$Uóôð
 × Ñ  §¡Õ1ð
	2ð —\‘\ $×"5Ñ"5°mÓ"DÓEˆ
Ø×#Ò#¬
°:×3GÑ3GÓ(HÌ:Ø×"Ñ"óL
ò )
ð 	
ð 
ô
 !$ F¨FÓ 3÷
á��uð ” uÓ-Ð.¨b°°°qÒ9ð
ˆñ 
ð
 —‘˜D×0Ñ0°×1AÑ1AÀ"Ñ1E×1KÑ1KÓLÓMˆØ —o‘oÐ& eÐ,ˆØ�˜2ˆˆØ�	‰	×Ñ˜x˜jÐ(@ÀÀÈÐLÔMä�4×)Ñ)Ó*¨aÒ/ð 	
Ø4ó	
Ð/ð 0˜×.Ñ.Ð/‰ˆØ$Ô&8¸Ó&>Ð?ÐØ$Ô&8¸Ó&>ÀÑ&BÐCÐäˆv‹;˜!Òòð $×,Ñ,Ð-=Ó>Ð?ˆIØ$×-Ñ-Ð.>Ó?Ð@‰Jò5ô
 œs 6›{Ó+öàð ×#Ñ#Ð$4Ó5¸!¸A¸3¸a¸Ó@ðˆIð ô œs 6›{Ó+öàð ×#Ñ#Ð$4Ó5¸!¸A¸3¸a¸Ó@ðˆJð ð
 	�	‰	×Ñ˜z˜l¨#©k¸'Ó.BÐ-CÐDÔEô ×Ñœ¬Ó/@ÓAÓBñ 	<Ù$ Y°
Ó;ˆK÷	<à�	‰	×ÑØ×"Ñ"Ð#3Ó4Ð5°S¹À[Ó9QÐ8RÐSô	
ô ˆv‹;˜!ÒØ�=Ð àQWÖXÈA�t—|‘| D×$7Ñ$7¸Ó$FÕGÐXˆÐXÜ˜kÓ*ò 	<‰DˆAˆqØ�I‰I×Ñ 1 # S¨¨°A°a°S¸Ð :Õ;ð	<ä�[Ó!Ð!ùók
ùò:ùò÷	<ð 	<üò Ys$   Å)O:ÊP ËPÌ3
P
Î'PÐ
Prz  c               óˆ   — | j                   j                  | j                  ||¬«      }t        |t        «      sJ ‚||_        |S ry  )rž  Úgenerater–  r9   r£  r¥  )rL   rÐ   r¥  r{  r�  s        rD   r}  zHalideKernel.genfuncH  s@   € ð �h‰h×Ñ §	¡	¨4¸ÐÓ?ˆÜ˜#Ô0Ô1Ð1Ð1Ø!ˆŒØˆ
rF   c                ól   — | j                   j                  «       }t        |t        «      sJ ‚||_        |S r¼   )rž  Únewvarr9   r£  r¥  )rL   r¥  r�  s      rD   rš  zHalideKernel.newfuncP  s/   € Ø�h‰h�o‰oÓˆÜ˜#Ô0Ô1Ð1Ð1Ø!ˆŒØˆ
rF   c                óx   — t         j                  j                  |«      j                  «       j	                  «       S )a  
        We map all tensors to 1D buffers in Halide since Halide has trouble representing some strides that PyTorch
        supports.  If there are gaps in the underlying layout the numel we pass to Halide includes the gaps while
        PyTorch's numel excludes them.
        )r$   rÎ  Ú
get_bufferÚ
get_layoutÚstorage_sizer   s     rD   Úhalide_buffer_numelz HalideKernel.halide_buffer_numelV  s+   € ô �w‰w×!Ñ! $Ó'×2Ñ2Ó4×AÑAÓCÐCrF   c                óŒ  ‡— d„ }g }| j                   j                  «       \  }}}}t        t        ||«      |¬«      D ]„  \  }Š|j	                  |‰f«       t        ‰t        «      sŒ*‰j                  dk(  r‰j                  �J ‚|j                  ˆfd„| j                  j                  ‰j                  d«      D «       «       Œ† |S )zX
        Halide requires scalar inputs before outputs, so need to reorder args.
        c                ón   — | \  }}t        |t        «      ryd|j                  v ryd|j                  v sJ ‚y)Nr%   Úout_ptrr   Úin_ptrr   )r9   r-   rÛ   )Ú	arg_tupleÚ	_call_strr®  s      rD   Ú	arg_orderz.HalideKernel.halide_argdefs.<locals>.arg_orderc  s<   € Ø&‰NˆI�sÜ˜#œwÔ'ØØ˜cŸh™hÑ&Øà 3§8¡8Ñ+Ð+Ð+ØrF   r  r   c           	   3  ó’   •K  — | ]>  }d t        |‰j                  ‰j                  ‰j                  ‰j                  ¬«      f–— Œ@ y ­w)N)Úalias_of)r.   Úbufferr¿   r:  rÛ   )r¾  Úaliasr®  s     €rD   r¿  z.HalideKernel.halide_argdefs.<locals>.<genexpr>s  sG   øè ø€ ò ð ð Ü!Ø!ØŸJ™JØŸI™IØŸJ™JØ%(§X¡Xôô	ñùs   ƒAArc   )rl   Úpython_argdefsrn  r‡  r*  r9   r.   r:  rð  r0  rö  rÂ  rÛ   )rL   rî  r™  r­  rŠ   r‹   Úcall_strr®  s          @rD   Úhalide_argdefszHalideKernel.halide_argdefs^  s¶   ø€ ò
	ð =?ˆØ—Y‘Y×-Ñ-Ó/‰
ˆˆ1ˆa�Ü#¤C¨¨1£I°9Ô=ò 	‰MˆH�cØ�M‰M˜8 S˜/Ô*Ü˜#œyÕ)Ø—z‘z Q’¨3¯<©<Ð+?Ð?Ð?Ø—‘ó ð "&×!4Ñ!4×!8Ñ!8¸¿¹À2Ó!Fôõ ð		ð" ˆrF   c                óò  — g }| j                  «       D �]4  \  }}t        |t        «      r	d}d}d}d}ná| j                  |j                     D �cg c]&  }t        | j                  |j                  «      «      ‘Œ( }}| j                  |j                     D �cg c]&  }t        | j                  |j                  «      «      ‘Œ( }}t        |«      t        |«      k(  sJ ‚t        | j                  |j                     «      }t        |j                     › d�}|j                  t        ||j                  ||||j                  ¬«      «       �Œ7 t         j"                  j%                  «       }	|	j&                  dk(  rDt(        j*                  j,                  g}
t(        j*                  j.                  }dt1        «       i}d}ný|	j&                  dk(  sJ d«       ‚|	j2                  d	k  sJ d
«       ‚t(        j*                  j4                  g}
t(        j*                  j6                  }t8        j:                  j=                  |	«      }d|
d	   vrAdD ]<  \  }}|j>                  |k\  sŒ|j@                  |k\  sŒ&|
j                  d|› |› �«        n |
j                  d«       d|jB                  i}tE        d	|	j2                  «      }|
j                  d«       |
j                  d«       t(        j*                  jF                  s|
j                  d«       t(        j*                  jH                  r|
j                  d«       d| jJ                  v r|
j                  d«       tM        |djO                  |
«      |||¬«      S c c}w c c}w )z)Compute metadata required by codecache.pyNÚlongr¯  )ÚshaperË  r:  rð  ÚcpuÚparallelismÚcudazonly cpu/cuda supportedr   zonly default device supportedÚcuda_capability))é   é   )rý  r   )é   é   )rÿ  r   )rþ  r%   Úcuda_capability_Úuser_contextÚstrict_floatÚ
no_runtimeÚ
no_assertsÚdebugÚ64Úlarge_buffersú-)ÚtargetÚ	schedulerÚscheduler_flagsÚcuda_device)(rõ  r9   r-   rï  rÛ   r0   rŽ  rˆ  rË  rk   rð  r/   r¿   r*  r   rð  r$   rÎ  Úget_current_device_or_throwÚtyper   r¡  Ú
cpu_targetÚscheduler_cpur    r€  Ú
gpu_targetÚscheduler_cudar;   rû  Úget_device_propertiesÚmajorÚminorÚmulti_processor_countr?   Úassertsr  r]   r   r³  )rL   Úargtypesr­  r®  rø  rË  r:  r¿   r«   Úcurrent_devicer
  Úschdulerr  r  Ú
capabilityr  r  s                    rD   Úhalide_kernel_metazHalideKernel.halide_kernel_meta‚  s  € àˆØ×)Ñ)Ó+ó 	‰FˆAˆsÜ˜#œwÔ'Ø�Ø�Ø�Ø‘ð "×3Ñ3°C·H±HÑ=öàô ˜$×.Ñ.¨q¯v©vÓ6Õ7ð�ð ð "×3Ñ3°C·H±HÑ=öàô ˜$×.Ñ.¨q¯x©xÓ8Õ9ð�ð ô ˜5“z¤S¨£[Ò0Ð0Ð0Ü˜t×2Ñ2°3·8±8Ñ<Ó=�Ü'¨¯	©	Ñ2Ð3°1Ð5�Ø�O‰OÜØØ—H‘HØØ!Ø!Ø Ÿ\™\ôö	ð%	ô: Ÿ™×<Ñ<Ó>ˆØ×Ñ %Ò'Ü—m‘m×.Ñ.Ð/ˆFÜ—}‘}×2Ñ2ˆHàÔ3Ó5ðˆOð ‰Kà!×&Ñ&¨&Ò0ÐKÐ2KÓKÐ0Ø!×'Ñ'¨1Ò,ÐMÐ.MÓMÐ,Ü—m‘m×.Ñ.Ð/ˆFÜ—}‘}×3Ñ3ˆHÜŸ™×9Ñ9¸.ÓIˆJØ ¨¨q©	Ñ1Ø$Lò ‘L�E˜5Ø!×'Ñ'¨5Ó0°Z×5EÑ5EÈÓ5NØŸ™Ð(8¸¸À¸wÐ&GÔHÙðð �M‰M˜.Ô)à˜z×?Ñ?ðˆOô
 ˜a ×!5Ñ!5Ó6ˆKð 	�‰�nÔ%ð 	�‰�lÔ#ä�}‰}×$Ò$Ø�M‰M˜,Ô'ä�=‰=×ÒØ�M‰M˜'Ô"à�4×#Ñ#Ñ#à�M‰M˜/Ô*äØØ—8‘8˜FÓ#ØØ+Ø#ô
ð 	
ùòCùòs   Á+M/Â+M4c                ó†
  ‡ — ‰ j                   j                  rt        d«      ‚‰ j                  «       }t	        «       }|j                  dd¬«       |j                  «        ‰ j                  «       D ]Ä  \  }}t        |t        «      r,|j                  |j                  › d‰ j                  › d�«       ŒB|j                  sJ |«       ‚d|j                  v rdnd	}t        |j                  «      }t!        ‰ j"                  |j                     «      }|j                  |j                  › d
|› d|› d|› d�«       ŒÆ |j                  d«       |j                  «        ‰ j                  «       D ]/  \  }}|j                  |j                  › d|j                  › �«       Œ1 ‰ j                   j%                  «       D ]  \  }	}
|j                  |	› d
|
› �«       Œ |j                  ‰ j&                  «       ˆ fd„}‰ j(                  j*                  D ]C  }t        |t,        «      r t.        j0                  j3                  ||«      }|j                  |«       ŒE |j                  d«       |j                  d«       ‰ j                  «       D �]Ç  \  }}t        |t        «      rWt4        j6                  j8                  j;                  |j<                  d¬«      }|j                  |j                  › d|› d�«       Œn‰ j"                  |j                     }g }t?        |«      D �]   \  }}‰ jA                  t4        j6                  j8                  j;                  |jB                  d¬«      |«      }|jE                  d|› d�«       d|j                  vsŒp|j                  |j                  › d|› d�«       	 |j                  |j                  › d|› dtG        |jH                  «      › d�«       	 |j                  |j                  › d|› dtG        |jB                  «      › d�«       �Œ |j                  |j                  › ddjM                  |«      › d�«       �ŒÊ |jO                  d«       |j                  djQ                  «       «       |jR                  rk|j                  dtU        jV                  |jR                  «      ›d|jX                  ›d |jR                  ›d|jZ                  ›d!�	d¬«       |j]                  «       S |j                  d"|jX                  ›d#�d¬«       |j]                  «       S # tJ        $ r Y �ŒRw xY w# tJ        $ r Y �Œ+w xY w)$z3Called at the end to generate a final kernel stringÚinplace_bufferszÒ
            import halide as hl
            from torch._inductor.runtime import halide_helpers
            from math import inf, nan

            @hl.generator(name="kernel")
            class Kernel:
        T©Ústripz = hl.InputScalar(r8   Úoutzhl.OutputBufferzhl.InputBufferrU  r±   r[   z&
            def generate(g):
        z = g.c                ó¬   •— t        t        ‰j                  j                  | j	                  d«         «      }|j
                  €J |«       ‚t        |«      S )Nr%   )r   r£  rž  rŸ  rM  r¥  r†  )rP  r�  rL   s     €rD   Úupdate_indexz1HalideKernel.codegen_kernel.<locals>.update_indexü  sE   ø€ ÜÔ(¨$¯(©(×*>Ñ*>¸q¿w¹wÀq»zÑ*JÓKˆCØ—=‘=Ð,Ð1¨cÓ1Ð,Ü�s“8ˆOrF   r�  zassert g.using_autoscheduler()r%   rÿ  z.set_estimate(rS  z.dim(z).set_min(0)z).set_stride(z).set_extent(z.set_estimates([rT  r   zN
            if __name__ == "__main__":
                hl.main()
            z:
                else:
                    hl.load_plugin(z))
                    target = hl.Target(z=)
                    autoscheduler = hl.AutoschedulerParams(a�  )
                    with hl.GeneratorContext(target, autoscheduler):
                        gen = Kernel()
                        pipeline = gen._build_pipeline()
                        # gen.compile_to_callable() does not run the autoscheduler
                        pipeline.apply_autoscheduler(target, autoscheduler)
                        kernel = pipeline.compile_to_callable([
                                gen._get_input_parameter(a.name)._to_argument()
                                for a in gen._get_arginfos()
                                if a.dir == hl.ArgInfoDirection.Input
                            ], target)
                zR
                  else:
                      with hl.GeneratorContext(hl.Target(zX)):
                          kernel = Kernel().compile_to_callable()
                  )/rl   r  rH   r  r)   ÚspliceÚ	do_indentrõ  r9   r-   rü  rÛ   r]   rñ  rÀ   r¿   rk   rï  Úaliasesr5  r–  Ú_linesr†  r£  rÇ  rl  r$   rÎ  rÏ  rÞ  r_   rV  Ú_autoscheduler_workaroundsrˆ  r*  r:   rË  rß  r³  Údo_unindentÚrstripr  r   Úfind_libautoscheduler
  r  Úgetvalue)rL   rÛ   ÚmetaÚcoder­  r®  ÚargclsÚargtypeÚndimrˆ  r‰  r$  rÐ   Úhintrµ  Úrange_hintsrH  Údims   `                 rD   Úcodegen_kernelzHalideKernel.codegen_kernelÕ  sÙ  ø€ à�9‰9×$Ò$ÜÐ/Ó0Ð0Ø×&Ñ&Ó(ˆÜÓˆØ�‰ðð ð 	ô 
	
ð 	�‰ÔØ×)Ñ)Ó+ò 	L‰FˆAˆsÜ˜#œwÔ'Ø—‘ #§(¡( Ð+=¸d×>NÑ>NÐ=OÈqÐQÕRà—z’zÐ& 3Ó&�zØ.3°s·x±xÑ.?Ñ*ÐEU�Ü% c§i¡iÓ0�Ü˜4×1Ñ1°#·(±(Ñ;Ó<�Ø—‘ #§(¡( ¨3¨v¨h°a¸°yÀÀ4À&ÈÐJÕKð	Lð 	�‰ðô	
ð
 	�‰ÔØ×)Ñ)Ó+ò 	9‰FˆAˆsØ�N‰N˜cŸh™h˜Z u¨S¯X©X¨JÐ7Õ8ð	9àŸ	™	×)Ñ)Ó+ò 	-‰HˆC�Ø�N‰N˜c˜U # c UÐ+Õ,ð	-à�‰�D×&Ñ&Ô'ô	ð
 —I‘I×$Ñ$ò 	!ˆDÜ˜$¤Ô$ä(×5Ñ5×9Ñ9¸,ÈÓM�Ø�N‰N˜4Õ ð		!ð
 	�‰�rÔØ�‰Ð7Ô8à×)Ñ)Ó+ó 	X‰FˆAˆsô ˜#œwÔ'Ü—w‘w×'Ñ'×1Ñ1°#·(±(ÀQÐ1ÓG�Ø—‘ #§(¡( ¨>¸$¸¸qÐAÕBà×-Ñ-¨c¯h©hÑ7�Ø �Ü'¨›oó !‘F�A�sØ×:Ñ:ÜŸ™×(Ñ(×2Ñ2°3·8±8ÀaÐ2ÓHÈ$ó�Dð  ×&Ñ&¨°d°V¸1Ð'=Ô>Ø C§H¡HÒ,ØŸ™¨#¯(©(¨°5¸¸¸<Ð'HÔIð!Ø ŸN™NØ#&§8¡8 *¨E°!°°MÄ#ÀcÇjÁjÃ/ÐARÐRSÐ Tôð
!Ø ŸN™NØ#&§8¡8 *¨E°!°°MÄ#ÀcÇhÁhÃ-ÀÐPQÐ Röð!ð& —‘ #§(¡( Ð+;¸D¿I¹IÀkÓ<RÐ;SÐSUÐVÖWð;	Xð> 	×Ñ˜ÔØ�‰ð÷ ‘“ô		
ð �>Š>Ø�K‰Kð$ä$3×$HÑ$HÈÏÉÓ$XÐ#[ð \(Ø(,¯© ð 7<Ø<@¿N¹NÐ;MÈRÐPT×PdÑPdÐOgð hð	ð  ð# ô ð8 �}‰}‹Ðð �K‰Kð:à:>¿+¹+¸ð Iðð
 ð ô ð �}‰}‹Ðøô]  )ò !Ú ð!ûô  )ò !Ú ð!ús$   Î&7T#Ï7T3Ô#	T0Ô/T0Ô3	U Ô?U c                óÊ   — t        |«      dk(  rTt        j                  j                  dk(  r7t        j
                  j                  «       j                  dk(  rt        d| «      } | S )Nr%   ÚAnderson2021rû  r   )	rk   r   r¡  r  r$   rÎ  r  r  r?   )r¶   rµ  s     rD   r)  z'HalideKernel._autoscheduler_workaroundsN  sN   € ô �‹I˜ŠNÜ—‘×,Ñ,°Ò>Ü—‘×3Ñ3Ó5×:Ñ:¸fÒDô �A�q“	ˆAØˆrF   c                óž  — t         j                  j                  }| j                  «       D ��cg c]  \  }}|j                  �Œ|› ‘Œ }}}t         j                  j                  «       }|j                  dk(  r;|j                  |j                  t         j                  «      }|j                  |«       |j                  |||d¬«       yc c}}w )zCodegen a call to this kernelNrû  F)ÚdeviceÚtriton)r$   rÎ  Úwrapper_coderõ  rð  r  r  Úwrite_get_raw_streamr€  r*  Úgenerate_kernel_call)	rL   rÛ   r  Úwrapperr¶   r®  Ú	call_argsr  Ústream_names	            rD   Úcall_kernelzHalideKernel.call_kernelY  s­   € ä—'‘'×&Ñ&ˆØ*.×*=Ñ*=Ó*?×X¡  3À3Ç<Á<ÑCW˜�s‘VÐXˆ	ÑXÜŸ™×<Ñ<Ó>ˆØ×Ñ &Ò(Ø!×6Ñ6°~×7KÑ7KÌQÏWÉWÓUˆKØ×Ñ˜[Ô)Ø×$Ñ$ØØØ!Øð	 	%õ 	
ùó Ys   ®C	ÁC	c                 ó   — yrÙ  rc   )rL   r‰  s     rD   Úgenerate_assertzHalideKernel.generate_asserth  s   € ØrF   c                 ó   — y r¼   rc   )rL   r_   rˆ  ÚlowerÚuppers        rD   Úcheck_boundszHalideKernel.check_boundsk  s   € ð 	rF   )rø  zdict[str, sympy.Expr]rP   rQ   )r¿   rž  rP   r†  )NN)r8  zSequence[sympy.Expr])r€  rÊ  )r�  r†  r€  rÊ  rh  rË   rÙ  )rÛ   r†  r€  rÊ  )rÛ   r†  r¼   )
rÛ   r†  r€  rÊ  rÔ   r'   r¥  r6   rP   rQ   )
r¿   rž  rÌ   rž  r¸  r5   rÔ   ú+Union[CSEVariable, tuple[CSEVariable, ...]]rP   rI  )rÓ  ztuple[torch.dtype, ...]rÀ  zUCallable[[tuple[CSEVariable, ...], tuple[CSEVariable, ...]], tuple[CSEVariable, ...]]rÔ  útuple[CSEVariable, ...]rP   rJ  )rP   r£  )rP   r   )r_   rÊ  rˆ  rÊ  rF  rË   rG  rË   )*rR   rS   rT   rÉ   Ú	overridesÚtexprr�  rÚ  rK   rú  rý  rF  rQ  r6  r|  r\  r…  r|  ry  r  r«  r•  rF  rÖ  r¨  rÃ  r³  rØ  r   Úunknownr}  rš  rç  rõ  r  r6  r¸   r)  rB  rD  rH  rU   rV   s   @rD   ræ  ræ  š  s¦  ø… Ø€IØ).€EÐ&Ó.ð+à%ð+ð 
õ	+ó6"ó6õiòVò*BðNàõNò%óf:òPò(
ó.ò.
óó%HóN?ð SWð6Øð6Ø *ð6Ø3>ð6ØFOð6à	ó6ð:;àð;ð ð;ð &ð	;ð
 ;ð;ð 
5ó;òz$ðLQ"à'ðQ"ð
ðQ"ð -ðQ"ð 
!óQ"ðh *=¨×)<Ñ)<Ó)>ñà	óóóDò"óHQ
ófwðr ñó ðô
òðØðØ&0ðØ9=ðØFJ÷rF   ræ  c                  ó(   — e Zd ZeZedd„«       Zd„ Zy)ÚHalideSchedulingc                óè   — t        t        j                  t        j                  t        j                  g«      }t
        j                  j                  r|j                  t        j                  «       |S r¼   )
r   r&   ÚTUPLE_REDUCTIONÚPREFER_STORE_LOOP_ORDERÚREDUCE_TO_SINGLE_ELEMENTr   r¡  Úscan_kernelsr  ÚSCAN)rÓ   r:  r™  s      rD   Úget_backend_featuresz%HalideScheduling.get_backend_featurest  sR   € ää×.Ñ.Ü×6Ñ6Ü×7Ñ7ðó
ˆô �=‰=×%Ò%Ø�J‰J”~×*Ñ*Ô+ØˆrF   c                ó  — t         j                  j                  }||j                  v r|j                  |   }|S d|j	                  «       › �}||j                  |<   |j                  d«       t        «       }|j                  d|j                  «       ›d�«       |j                  |d¬«       |j                  d«       t        ||«      \  }}|› d|› �}	|j                  ||j                  «       |	«       t        d	«      rt        |d
|«       |S )z6Codegen kernel definition to go in output wrapper codeÚhalide_kernel_zEfrom torch._inductor.runtime.hints import HalideMeta, HalideInputSpeczasync_compile.halide(z, '''Tr   z''')ú
Úkernel_metadatar�  )r$   rÎ  r<  Úsrc_to_kernelÚnext_kernel_suffixÚadd_import_oncer)   rü  r  r%  r   Údefine_kernelr-  r   r   )
rL   Úsrc_codeÚnode_scheduler\   r?  Úkernel_nameÚcompile_wrapperÚoriginsÚdetailed_originsÚmetadata_comments
             rD   r^  zHalideScheduling.define_kernel�  s"  € ä—'‘'×&Ñ&ˆØ�w×,Ñ,Ñ,Ø!×/Ñ/°Ñ9ˆKð. Ðð+ +¨7×+EÑ+EÓ+GÐ*HÐIˆKØ.9ˆG×!Ñ! (Ñ+Ø×#Ñ#ØWôô -Ó.ˆOØ×%Ñ%Ø'¨×(AÑ(AÓ(CÐ'FÀeÐLôð ×"Ñ" 8°4Ð"Ô8Ø×%Ñ% fÔ-ä(;¸MÈ7Ó(SÑ%ˆGÐ%Ø") ¨"Ð-=Ð,>Ð?ÐØ×!Ñ!Ø˜_×5Ñ5Ó7Ð9Iôô 'Ð'8Ô9Ü# K°°XÔ>àÐrF   N)r:  ztorch.devicerP   zOrderedSet[BackendFeature])rR   rS   rT   ræ  Úkernel_typer   rV  r^  rc   rF   rD   rO  rO  q  s   „ Ø€Kàò
ó ð
órF   rO  )rÚ
__future__r   Údataclassesr  r§  ÚloggingrÅ  Úcollectionsr   Úmathr   Útypingr   r   r   r	   r
   r   r†   r;   Útorch._loggingÚ_prims_commonr   Úutils._ordered_setr   Úutils._sympy.functionsr   r   Úutils._sympy.symbolr   r   Úutils._sympy.value_rangesr   r�  r   r   Ú	codecacher   r   Úmetricsr   r   Úops_handlerr   Úruntime.hintsr   r   Úutilsr   r   r    r!   r"   Úvirtualizedr#   rE  r$   Úcommonr&   r'   r(   r)   r*   r+   r,   r-   r.   Úcppr/   Ú	cpp_utilsr0   Úsimdr1   r2   r3   Úcollections.abcr4   r5   r6   Ú	getLoggerrR   rW  rE   ÚRuntimeErrorrH   rX   r   rL  ÚpexprrË   rÅ   rÄ   rÆ   Úfloat64rk  Úint16rÃ   r=   Úuint8Úuint16Úuint32Úuint64r½   rÀ   rÇ   rÉ   Ú_initialize_pointwise_overridesr£  Ú	dataclassrÉ  rá  rj  ræ  rO  rc   rF   rD   ú<module>r‰     sÐ  ðå "ã Û Û Û Û 	Ý #Ý ß F× Fã ã Û å -Ý ,ß ?ß 7Ý 4ß Ý 'Ý )ß BÝ )ß 7÷õ ÷ )÷
÷ 
õ 
õ Ý ß ;Ñ ;ñ Ý(ç6à€g×Ñ˜Ó!€ò
ôF�,ô Fô
zQ�Mô zQñz 	‹×Ñ€Ù‹×Ñ€ð 
‡J�J�Ø	‡N�N�OØ	‡M�M�>Ø	‡M�M�>Ø	‡M�M�>Ø	‡J�J�Ø	‡K�K�Ø	‡K�K�Ø	‡K�K�Ø	‡K�K�Ø	‡L�L�-Ø	‡L�L�-Ø	‡L�L�-ð€ò"òôC+�kô C+ðL
 × /Ñ /°Ô 9ô$P˜ô $PðN ×Ñ÷+ð +ó ð+ò>
òô T�:ô Tôn+�~õ +rF   