Ë
    [^(hgp  ã                   óÈ  — U d dl Z d dlmZmZmZ ddlmZ d dlmZm	Z	m
Z
mZmZ d dlmZ d dlmZ d dlmZ d d	lmZ d d
lmZ d dlmZ d dlZddgZ ed«      Z ed«      Ze j:                  j<                  Zd„ Zi Z e!eef   e"d<   d„ Z#d:deeeef   geeef   f   fd„Z$ e$ejJ                  «      ddœde&fd„«       Z' e$ejP                  «      d;de&fd„«       Z) e$ejT                  «      d;de&fd„«       Z+ e$ejX                  «      d;de&fd„«       Z- e$ej\                  «      	 	 	 	 	 d<de&fd„«       Z/	 d:de0e&   de0e&   de0e&   de1de&f
d„Z2 e$ejf                  ejh                  g«      ddœde&fd „«       Z5 e$ejl                  «      de&fd!„«       Z7d"„ Z8 e$ejr                  ejt                  ejv                  g«      ddœde&fd#„«       Z<d$„ Z=dd%œdee>e>e&d&f   e>e&d&f   e>e&d&f   e	e>e&d&f      f      fd'„Z?dd%œdee>e>e&d&f   e>e&d&f   e>e&d&f   e	e>e&d&f      f      fd(„Z@ e$ej‚                  d)¬*«      ddœde&fd+„«       ZB e$ej†                  d)¬*«      de&fd,„«       ZDd-„ ZE e$ejŒ                  ejŽ                  ej�                  g«      ddœde&fd.„«       ZI e$ej”                  d)¬*«      de&fd/„«       ZK e$ej˜                  d)¬*«      de&fd0„«       ZMi ejJ                  e'“ejP                  e)“ejT                  e+“ejX                  e-“ej\                  e/“ejf                  e5“ejh                  e5“ejl                  e7“ejr                  e<“ejt                  e<“ejv                  e<“ejŒ                  eI“ejŽ                  eI“ej�                  eI“ej‚                  eB“ej†                  eD“ej”                  eK“ej˜                  eMi¥Z d1„ ZNg d2¢ZOd3„ ZPd4„ ZQd5„ ZRd6„ ZS G d7„ d«      ZT G d8„ d9e«      ZUy)=é    N)Útree_mapÚtree_flattenÚtree_unflattené   )ÚModuleTracker)ÚAnyÚOptionalÚUnionÚTypeVarÚCallable)ÚIterator)Ú	ParamSpec)Údefaultdict)ÚTorchDispatchMode©Úprod©ÚwrapsÚFlopCounterModeÚregister_flop_formulaÚ_TÚ_Pc                 óR   — t        | t        j                  «      r| j                  S | S ©N)Ú
isinstanceÚtorchÚTensorÚshape)Úis    úV/var/www/skyplay_api_hub/venv/lib/python3.12/site-packages/torch/utils/flop_counter.pyÚ	get_shaper!      s   € Ü�!”U—\‘\Ô"Ø�w‰wˆØ€Hó    Úflop_registryc                 ó4   ‡ — t        ‰ «      d dœˆ fd„
«       }|S )N)Úout_valc                 óF   •— t        t        ||| f«      \  }}} ‰|d|i|¤ŽS )NÚ	out_shape)r   r!   )r%   ÚargsÚkwargsr'   Úfs       €r    Únfzshape_wrapper.<locals>.nf   s2   ø€ ä"*¬9°t¸VÀWÐ6MÓ"NÑˆˆf�iÙ�$Ð6 )Ð6¨vÑ6Ð6r"   r   ©r*   r+   s   ` r    Úshape_wrapperr-      s#   ø€ Ü
ˆ1ƒXØõ 7ó ð7ð €Ir"   Úreturnc                 ód   ‡ ‡— dt         t        t        f   dt         t        t        f   fˆˆ fd„}|S )NÚflop_formular.   c                 ó‚   •‡ — ‰st        ‰ «      Š ˆ fd„}t        j                  j                  j	                  |‰«       ‰ S )Nc                 óÀ   •— t        | t        j                  j                  «      st	        d| › dt        | «      › �«      ‚| t        v rt        d| › �«      ‚‰t        | <   y )Nzlregister_flop_formula(targets): expected each target to be OpOverloadPacket (i.e. torch.ops.mylib.foo), got z which is of type zduplicate registrations for )r   r   Ú_opsÚOpOverloadPacketÚ
ValueErrorÚtyper#   ÚRuntimeError)Útargetr0   s    €r    Úregisterz=register_flop_formula.<locals>.register_fun.<locals>.register(   si   ø€ Ü˜f¤e§j¡j×&AÑ&AÔBÜ ðHà�hÐ0´°f³°ð@óAð Að œÑ&Ü"Ð%AÀ&ÀÐ#JÓKÐKØ$0ŒM˜&Ò!r"   )r-   r   ÚutilsÚ_pytreeÚ	tree_map_)r0   r9   Úget_rawÚtargetss   ` €€r    Úregister_funz+register_flop_formula.<locals>.register_fun$   s7   ù€ ÙÜ(¨Ó6ˆLô	1ô 	�‰×Ñ×%Ñ% h°Ô8àÐr"   )r   r   r   )r>   r=   r?   s   `` r    r   r   #   s0   ù€ ð¤8¬B´¨FÑ#3ð ¼ÄÄRÀÑ8Hö ð& Ðr"   )r'   c                ó:   — | \  }}|\  }}||k(  sJ ‚||z  dz  |z  S )zCount flops for matmul.é   © )	Úa_shapeÚb_shaper'   r(   r)   ÚmÚkÚk2Úns	            r    Úmm_floprI   9   s3   € ð
 �D€A€qØ�E€BˆØ�Š7€Nˆ7àˆq‰5�1‰9�q‰=Ðr"   c                 ó   — t        ||«      S )zCount flops for addmm.©rI   ©Ú
self_shaperC   rD   r'   r)   s        r    Ú
addmm_floprN   D   s   € ô �7˜GÓ$Ð$r"   c                 óV   — | \  }}}|\  }}}	||k(  sJ ‚||k(  sJ ‚||z  |	z  dz  |z  }
|
S )z"Count flops for the bmm operation.rA   rB   )rC   rD   r'   r)   ÚbrE   rF   Úb2rG   rH   Úflops              r    Úbmm_floprS   I   sK   € ð
 �G€A€qˆ!Ø�I€BˆˆAØ�Š7€Nˆ7Ø�Š7€Nˆ7àˆq‰5�1‰9�q‰=˜1Ñ€DØ€Kr"   c                 ó   — t        ||«      S )z&Count flops for the baddbmm operation.©rS   rL   s        r    Úbaddbmm_floprV   V   s   € ô
 �G˜WÓ%Ð%r"   c	                 ó   — t        | |«      S )zCount flops for _scaled_mm.rK   )
rC   rD   Úscale_a_shapeÚscale_b_shapeÚ
bias_shapeÚscale_result_shapeÚ	out_dtypeÚuse_fast_accumr'   r)   s
             r    Ú_scaled_mm_flopr^   ]   s   € ô �7˜GÓ$Ð$r"   Úx_shapeÚw_shaper'   Ú
transposedc                 ót   — | d   }|r| n|dd }|^}}}	 t        |«      t        |«      z  |z  |z  |z  dz  }	|	S )a  Count flops for convolution.

    Note only multiplication is
    counted. Computation for bias are ignored.
    Flops for a transposed convolution are calculated as
    flops = (x_shape[2:] * prod(w_shape) * batch_size).
    Args:
        x_shape (list(int)): The input shape before convolution.
        w_shape (list(int)): The filter shape.
        out_shape (list(int)): The output shape after convolution.
        transposed (bool): is the convolution transposed
    Returns:
        int: the number of flops
    r   rA   Nr   )
r_   r`   r'   ra   Ú
batch_sizeÚ
conv_shapeÚc_outÚc_inÚfilter_sizerR   s
             r    Úconv_flop_countrh   n   s]   € ð* ˜‘€JÙ'‘'¨Y¸¸Ð;€JØ 'Ð€Eˆ4�+ðô �
Óœd ;Ó/Ñ/°*Ñ<¸uÑDÀtÑKÈaÑO€DØ€Kr"   c                ó    — t        | |||¬«      S )zCount flops for convolution.©ra   )rh   )
r_   r`   Ú_biasÚ_strideÚ_paddingÚ	_dilationra   r'   r(   r)   s
             r    Ú	conv_flopro   •   s   € ô ˜7 G¨YÀ:ÔNÐNr"   c                 ó  — d„ }d}	 |
d   r t        |d   «      }|t        | ||| «      z  }|
d   rZt        |d   «      }|r&|t         || «       ||«       ||«      d¬«      z  }|S |t         ||«       || «       ||«      d¬«      z  }|S )Nc                 ó4   — | d   | d   gt        | dd  «      z   S )Nr   r   rA   )Úlist)r   s    r    Útzconv_backward_flop.<locals>.tª   s$   € Ø�a‘˜% ™(Ð#¤d¨5°°¨9£oÑ5Ð5r"   r   r   Frj   )r!   rh   )Úgrad_out_shaper_   r`   rk   rl   rm   rn   ra   Ú_output_paddingÚ_groupsÚoutput_maskr'   rs   Ú
flop_countÚgrad_input_shapeÚgrad_weight_shapes                   r    Úconv_backward_flopr{   ›   s¸   € ò6à€JðDðL �1‚~Ü$ Y¨q¡\Ó2ÐØ”o n°gÐ?OÐU_ÐQ_Ó`Ñ`ˆ
à�1‚~Ü% i°¡lÓ3ÐÙàœ/©!¨NÓ*;¹Q¸w»ZÉÐK\ÓI]ÐjoÔpÑpˆJð
 Ðð œ/©!¨G«*±a¸Ó6GÉÐK\ÓI]ÐjoÔpÑpˆJàÐr"   c                 óþ   — | \  }}}}|\  }}}	}
|\  }}}}||cxk(  r|k(  r"n J ‚||cxk(  r|k(  rn J ‚||
k(  r
|	|k(  r||
k(  sJ ‚d}|t        ||z  ||f||z  ||	f«      z  }|t        ||z  ||	f||z  |	|f«      z  }|S )z^
    Count flops for self-attention.

    NB: We can assume that value_shape == key_shape
    r   rU   )Úquery_shapeÚ	key_shapeÚvalue_shaperP   ÚhÚs_qÚd_qÚ_b2Ú_h2Ús_kÚ_d2Ú_b3Ú_h3Ú_s3Úd_vÚtotal_flopss                   r    Úsdpa_flop_countrŒ     sÂ   € ð !�N€A€qˆ#ˆsØ"Ñ€Cˆˆc�3Ø$Ñ€Cˆˆc�3Ø�Œ?�sŒ?Ð[Ð[˜q Cœ¨3œÐ[Ð[°3¸#²:À#ÈÂ*ÐQTÐX[ÒQ[Ð[Ð[Ø€Kà”8˜Q ™U C¨Ð-°°A±°s¸CÐ/@ÓAÑA€Kà”8˜Q ™U C¨Ð-°°A±°s¸CÐ/@ÓAÑA€KØÐr"   c                ó   — t        | ||«      S )úCount flops for self-attention.©rŒ   )r}   r~   r   r'   r(   r)   s         r    Ú	sdpa_flopr�     s   € ô ˜;¨	°;Ó?Ð?r"   c                 óÔ   — ddl m} ddlm} t	        | ||f«      s7| j
                  j                  dk7  r| j                  «       j                  «       S |g| j                  d«      dz
  z  S )zŸ
    If the offsets tensor is fake, then we don't know the actual lengths.
    In that case, we can just assume the worst case; each batch has max length.
    r   )Ú
FakeTensor)ÚFunctionalTensorÚmetar   )
Útorch._subclasses.fake_tensorr’   Ú#torch._subclasses.functional_tensorr“   r   Údevicer6   ÚdiffÚtolistÚsize)ÚoffsetsÚmax_lenr’   r“   s       r    Ú_offsets_to_lengthsr�     s[   € õ
 9ÝDÜ�g 
Ð,<Ð=Ô>À7Ç>Á>×CVÑCVÐZ`ÒC`Ø�|‰|‹~×$Ñ$Ó&Ð&Øˆ9˜Ÿ™ Q›¨!Ñ+Ñ,Ð,r"   )Úgrad_out.c              #   óZ  K  — |�ñt        |j                  «      dk(  sJ ‚t        |j                  «      dk(  sJ ‚|�|j                  | j                  k(  sJ ‚| j                  \  }}	}
|j                  \  }}}|j                  \  }}}|€J ‚|€J ‚|j                  |j                  k(  sJ ‚t        ||«      }t        ||«      }t        ||«      D ]%  \  }}d|	||
f}d|||f}d|||f}|�|nd}||||f–— Œ' y| j                  |j                  |j                  |�|j                  ndf–— y­w)a;  
    Given inputs to a flash_attention_(forward|backward) kernel, this will handle behavior for
    NestedTensor inputs by effectively unbinding the NestedTensor and yielding the shapes for
    each batch element.

    In the case that this isn't a NestedTensor kernel, then it just yields the original shapes.
    Né   r   ©Úlenr   r�   Úzip)ÚqueryÚkeyÚvaluerž   Ú	cum_seq_qÚ	cum_seq_kÚmax_qÚmax_kÚ_Úh_qr‚   Úh_kÚd_kÚh_vrŠ   Úseq_q_lengthsÚseq_k_lengthsÚ	seq_q_lenÚ	seq_k_lenÚnew_query_shapeÚnew_key_shapeÚnew_value_shapeÚnew_grad_out_shapes                          r    Ú%_unpack_flash_attention_nested_shapesr¸   *  s[  è ø€ ð$ Ðô �3—9‘9‹~ Ò"Ð"Ð"Ü�5—;‘;Ó 1Ò$Ð$Ð$ØÐ 8§>¡>°U·[±[Ò#@Ð@Ð@Ø—k‘k‰ˆˆ3�Ø—i‘i‰ˆˆ3�Ø—k‘k‰ˆˆ3�ØÐ$Ð$Ð$ØÐ$Ð$Ð$Ø�‰ )§/¡/Ò1Ð1Ð1Ü+¨I°uÓ=ˆÜ+¨I°uÓ=ˆÜ&)¨-¸Ó&Gò 	VÑ"ˆY˜	Ø  # y°#Ð6ˆOØ  Y°Ð4ˆMØ  # y°#Ð6ˆOØ4<Ð4H¡ÈdÐØ! =°/ÐCUÐUÓUð	Vð 	à
�+‰+�s—y‘y %§+¡+ÀÐAU¨x¯~ª~Ð[_Ð
_Ó_ùs   ‚D)D+c              #   ó`  K  — |�ôt        |j                  «      dk(  sJ ‚t        |j                  «      dk(  sJ ‚|�|j                  | j                  k(  sJ ‚| j                  \  }}}	}
|j                  \  }}}}|j                  \  }}}}|€J ‚|€J ‚|j                  |j                  k(  sJ ‚t        ||«      }t        ||«      }t        ||«      D ]%  \  }}d|	||
f}d|||f}d|||f}|�|nd}||||f–— Œ' y| j                  |j                  |j                  |�|j                  ndf–— y­w)a?  
    Given inputs to a efficient_attention_(forward|backward) kernel, this will handle behavior for
    NestedTensor inputs by effectively unbinding the NestedTensor and yielding the shapes for
    each batch element.

    In the case that this isn't a NestedTensor kernel, then it just yields the original shapes.
    Né   r   r¡   )r¤   r¥   r¦   rž   Úcu_seqlens_qÚcu_seqlens_kÚmax_seqlen_qÚmax_seqlen_kr«   r¬   r‚   r­   r®   r¯   rŠ   Ú	seqlens_qÚ	seqlens_kÚlen_qÚlen_kr´   rµ   r¶   r·   s                          r    Ú)_unpack_efficient_attention_nested_shapesrÃ   X  sd  è ø€ ð$ Ðô �3—9‘9‹~ Ò"Ð"Ð"Ü�5—;‘;Ó 1Ò$Ð$Ð$ØÐ 8§>¡>°U·[±[Ò#@Ð@Ð@ØŸ™‰ˆˆ1ˆc�3ØŸ™‰ˆˆ1ˆc�3ØŸ™‰ˆˆ1ˆc�3ØÐ'Ð'Ð'ØÐ'Ð'Ð'Ø×!Ñ! \×%7Ñ%7Ò7Ð7Ð7Ü'¨°lÓCˆ	Ü'¨°lÓCˆ	Ü 	¨9Ó5ò 	V‰LˆE�5Ø  # u¨cÐ2ˆOØ  U¨CÐ0ˆMØ  # u¨cÐ2ˆOØ4<Ð4H¡ÈdÐØ! =°/ÐCUÐUÓUð	Vð 	à
�+‰+�s—y‘y %§+¡+ÀÐAU¨x¯~ª~Ð[_Ð
_Ó_ùs   ‚D,D.T)r=   c          	      óJ   — t        | ||||||¬«      }
t        d„ |
D «       «      S )rŽ   )r¤   r¥   r¦   r§   r¨   r©   rª   c              3   ó@   K  — | ]  \  }}}}t        |||«      –— Œ y ­wr   r�   ©Ú.0r}   r~   r   r«   s        r    ú	<genexpr>z0_flash_attention_forward_flop.<locals>.<genexpr>¢  ó)   è ø€ ò á2ˆK˜ K°ô 	˜ Y°×<ñùó   ‚©r¸   Úsum)r¤   r¥   r¦   r§   r¨   r©   rª   r'   r(   r)   Úsizess              r    Ú_flash_attention_forward_floprÎ   ˆ  s?   € ô" 2ØØØØØØØô€Eô ñ à6;ôó ð r"   c           	      óJ   — t        | ||||||¬«      }
t        d„ |
D «       «      S )rŽ   )r¤   r¥   r¦   r»   r¼   r½   r¾   c              3   ó@   K  — | ]  \  }}}}t        |||«      –— Œ y ­wr   r�   rÆ   s        r    rÈ   z4_efficient_attention_forward_flop.<locals>.<genexpr>Â  rÉ   rÊ   ©rÃ   rÌ   )r¤   r¥   r¦   Úbiasr»   r¼   r½   r¾   r(   r)   rÍ   s              r    Ú!_efficient_attention_forward_floprÓ   ¨  s?   € ô" 6ØØØØ!Ø!Ø!Ø!ô€Eô ñ à6;ôó ð r"   c                 óØ  — d}|\  }}}}|\  }	}
}}|\  }}}}| \  }}}}||	cxk(  r|cxk(  r|k(  rn J ‚||
cxk(  r|cxk(  r|k(  r	n J ‚||k(  sJ ‚||k(  r
||k(  r||k(  sJ ‚d}|t        ||z  ||f||z  ||f«      z  }|t        ||z  ||f||z  ||f«      z  }|t        ||z  ||f||z  ||f«      z  }|t        ||z  ||f||z  ||f«      z  }|t        ||z  ||f||z  ||f«      z  }|S )Nr   rU   )rt   r}   r~   r   r‹   rP   r€   r�   r‚   rƒ   r„   r…   r†   r‡   rˆ   r‰   rŠ   Ú_b4Ú_h4Ú_s4Ú_d4s                        r    Úsdpa_backward_flop_countrÙ   È  sf  € Ø€KØ �N€A€qˆ#ˆsØ"Ñ€Cˆˆc�3Ø$Ñ€Cˆˆc�3Ø'Ñ€Cˆˆc�3Ø�Ô!�sÔ!˜cÔ!ÐKÐK a¨3Ô&<°#Ô&<¸Ô&<ÐKÐKÀÈÂÐKÐKØ�#Š:˜# š*¨°ªÐ3Ð3Ø€Kð ”8˜Q ™U C¨Ð-°°A±°s¸CÐ/@ÓAÑA€Kð ”8˜Q ™U C¨Ð-°°A±°s¸CÐ/@ÓAÑA€Kà”8˜Q ™U C¨Ð-°°A±°s¸CÐ/@ÓAÑA€Kð ”8˜Q ™U C¨Ð-°°A±°s¸CÐ/@ÓAÑA€Kà”8˜Q ™U C¨Ð-°°A±°s¸CÐ/@ÓAÑA€KØÐr"   c                ó   — t        | |||«      S )z(Count flops for self-attention backward.©rÙ   )rt   r}   r~   r   r'   r(   r)   s          r    Úsdpa_backward_floprÜ   ã  s   € ô
 $ N°KÀÈKÓXÐXr"   c
           
      óL   — t        |||| ||||	¬«      }t        d„ |D «       «      S )N)r¤   r¥   r¦   rž   r§   r¨   r©   rª   c              3   óB   K  — | ]  \  }}}}t        ||||«      –— Œ y ­wr   rÛ   ©rÇ   r}   r~   r   rt   s        r    rÈ   z1_flash_attention_backward_flop.<locals>.<genexpr>  ó+   è ø€ ò á?ˆK˜ K°ô 	! °¸iÈ×Uñùó   ‚rË   )rž   r¤   r¥   r¦   ÚoutÚ	logsumexpr§   r¨   r©   rª   r(   r)   Úshapess                r    Ú_flash_attention_backward_floprå   ê  sB   € ô" 3ØØØØØØØØô	€Fô ñ àCIôó ð r"   c
           
      óL   — t        |||| ||||	¬«      }t        d„ |D «       «      S )N)r¤   r¥   r¦   rž   r»   r¼   r½   r¾   c              3   óB   K  — | ]  \  }}}}t        ||||«      –— Œ y ­wr   rÛ   rß   s        r    rÈ   z5_efficient_attention_backward_flop.<locals>.<genexpr>&  rà   rá   rÑ   )rž   r¤   r¥   r¦   rÒ   râ   r»   r¼   r½   r¾   r(   r)   rä   s                r    Ú"_efficient_attention_backward_floprè     sB   € ô" 7ØØØØØ!Ø!Ø!Ø!ô	€Fô ñ àCIôó ð r"   c                 ó,   — t        | t        «      s| fS | S r   )r   Útuple)Úxs    r    Únormalize_tuplerì   A  s   € Ü�aœÔØˆtˆØ€Hr"   )Ú ÚKÚMÚBÚTc                 ó�   — t        dt        t        t        «      dz
  t        t	        | «      «      dz
  dz  «      «      }t        |   S )Nr   r   rA   r    )ÚmaxÚminr¢   ÚsuffixesÚstr)ÚnumberÚindexs     r    Úget_suffix_strrù   J  s=   € ô �”3”sœ8“} qÑ(¬3¬s°6«{Ó+;¸aÑ+?ÀAÑ*EÓFÓG€EÜ�E‰?Ðr"   c                 óX   — t         j                  |«      }| d|z  z  d›}|t         |   z   S )Niè  z.3f)rõ   rø   )r÷   Úsuffixrø   r¦   s       r    Úconvert_num_with_suffixrü   Q  s2   € Ü�N‰N˜6Ó"€Eà˜ ™Ñ% cÐ*€Eà”8˜E‘?Ñ"Ð"r"   c                 ó   — |dk(  ry| |z  d›S )Nr   ú0%z.2%rB   )ÚnumÚdenoms     r    Úconvert_to_percent_strr  X  s   € Ø�‚zØØ�E‰k˜#ÐÐr"   c                 ó.   ‡ — t        ‰ «      ˆ fd„«       }|S )Nc                 óB   •— t        | «      \  }} ‰|Ž }t        ||«      S r   )r   r   )r(   Ú	flat_argsÚspecrâ   r*   s       €r    r+   z)_pytreeify_preserve_structure.<locals>.nf^  s'   ø€ ä& tÓ,‰ˆ	�4Ù�ˆmˆÜ˜c 4Ó(Ð(r"   r   r,   s   ` r    Ú_pytreeify_preserve_structurer  ]  s    ø€ Ü
ˆ1ƒXó)ó ð)ð
 €Ir"   c                   óú   ‡ — e Zd ZdZ	 	 	 	 ddeeej                  j                  e	ej                  j                     f      de
dedeeeef      fˆ fd„Zde
fd„Zdeeeee
f   f   fd	„Zdd
„Zd„ Zd„ Zd„ Zˆ xZS )r   aþ  
    ``FlopCounterMode`` is a context manager that counts the number of flops within its context.

    It does this using a ``TorchDispatchMode``.

    It also supports hierarchical output by passing a module (or list of
    modules) to FlopCounterMode on construction. If you do not need hierarchical
    output, you do not need to use it with a module.

    Example usage

    .. code-block:: python

        mod = ...
        with FlopCounterMode(mod) as flop_counter:
            mod.sum().backward()

    ÚmodsÚdepthÚdisplayÚcustom_mappingc                 ód  •— t         ‰| �  «        t        d„ «      | _        || _        || _        d | _        |€i }|�t        j                  dd¬«       i t        ¥|j                  «       D ��ci c]   \  }}|t        |dd«      r|n
t        |«      “Œ" c}}¥| _	        t        «       | _        y c c}}w )Nc                  ó    — t        t        «      S r   )r   ÚintrB   r"   r    ú<lambda>z*FlopCounterMode.__init__.<locals>.<lambda>‚  s   € Ì+ÔVYÓJZ€ r"   z<mods argument is not needed anymore, you can stop passing itrA   )Ú
stacklevelÚ_get_rawF)ÚsuperÚ__init__r   Úflop_countsr	  r
  ÚmodeÚwarningsÚwarnr#   ÚitemsÚgetattrr-   r   Úmod_tracker)Úselfr  r	  r
  r  rF   ÚvÚ	__class__s          €r    r  zFlopCounterMode.__init__{  s°   ø€ ô 	‰ÑÔÜ6AÑBZÓ6[ˆÔØˆŒ
ØˆŒØ04ˆŒ	ØÐ!ØˆNØÐÜ�M‰MÐXÐefÕgð
Üð
àWe×WkÑWkÓWm×nÉtÈqÐRSˆq”w˜q *¨eÔ4‘!¼-ÈÓ:JÑJÓnð
ˆÔô )›?ˆÕùó os   Á-%B,r.   c                 óN   — t        | j                  d   j                  «       «      S )NÚGlobal)rÌ   r  Úvalues©r  s    r    Úget_total_flopszFlopCounterMode.get_total_flops�  s!   € Ü�4×#Ñ# HÑ-×4Ñ4Ó6Ó7Ð7r"   c                 ó|   — | j                   j                  «       D ��ci c]  \  }}|t        |«      “Œ c}}S c c}}w )a  Return the flop counts as a dictionary of dictionaries.

        The outer
        dictionary is keyed by module name, and the inner dictionary is keyed by
        operation name.

        Returns:
            Dict[str, Dict[Any, int]]: The flop counts as a dictionary.
        )r  r  Údict)r  rF   r  s      r    Úget_flop_countszFlopCounterMode.get_flop_counts“  s3   € ð (,×'7Ñ'7×'=Ñ'=Ó'?×@™t˜q !�”4˜“7‘
Ó@Ð@ùÓ@s   ž8c                 ó  ‡ ‡
‡‡— |€‰ j                   }|€d}dd l}d|_        g d¢}g }‰ j                  «       Š
t	        ‰
«      ŠdŠˆ
ˆˆˆ fd„}t        ‰ j                  j                  «       «      D ]?  }|dk(  rŒ	|j                  d«      d	z   }||kD  rŒ# |||d	z
  «      }|j                  |«       ŒA d‰ j                  v r ‰s|D ]  }	d
|	d   z   |	d<   Œ  |dd«      |z   }t        |«      dk(  rg d¢g}|j                  ||d¬«      S )Ni?B r   T)ÚModuleÚFLOPz% TotalFc           	      ó€  •— t        ‰
j                  |    j                  «       «      }‰	|‰k\  z  Š	d|z  }g }|j                  || z   t	        |‰«      t        |‰«      g«       ‰
j                  |    j                  «       D ]<  \  }}|j                  |dz   t        |«      z   t	        |‰«      t        |‰«      g«       Œ> |S )Nú z - )rÌ   r  r   Úappendrü   r  r  rö   )Úmod_namer	  r‹   Úpaddingr   rF   r  Úglobal_flopsÚglobal_suffixÚis_global_subsumedr  s          €€€€r    Úprocess_modz.FlopCounterMode.get_table.<locals>.process_mod­  sÓ   ø€ ô ˜d×.Ñ.¨xÑ8×?Ñ?ÓAÓBˆKà +°Ñ"=Ñ=Ðà˜E‘kˆGØˆFØ�M‰MØ˜(Ñ"Ü'¨°]ÓCÜ& {°LÓAðô ð
 ×(Ñ(¨Ñ2×8Ñ8Ó:ò ‘��1Ø—‘Ø˜e‘O¤c¨!£fÑ,Ü+¨A¨}Ó=Ü*¨1¨lÓ;ðõ ðð ˆMr"   r  ú.r   r*  )r  Ú0rþ   )ÚleftÚrightr5  )ÚheadersÚcolalign)r	  ÚtabulateÚPRESERVE_WHITESPACEr"  rù   Úsortedr  ÚkeysÚcountÚextendr¢   )r  r	  r8  Úheaderr   r1  ÚmodÚ	mod_depthÚ
cur_valuesr¦   r.  r/  r0  s   `         @@@r    Ú	get_tablezFlopCounterMode.get_tableŸ  s.  û€ Øˆ=Ø—J‘JˆEØˆ=ØˆEãØ'+ˆÔ$Ú.ˆØˆØ×+Ñ+Ó-ˆÜ& |Ó4ˆØ"Ð÷	ô, ˜$×*Ñ*×/Ñ/Ó1Ó2ò 	&ˆCØ�hŠØØŸ	™	 #›¨Ñ*ˆIØ˜5Ò Øá$ S¨)°a©-Ó8ˆJØ�M‰M˜*Õ%ð	&ð �t×'Ñ'Ñ'Ñ0BØò *�Ø  q¡™>��a’ð*ñ ! ¨1Ó-°Ñ6ˆFäˆv‹;˜!ÒÚ+Ð,ˆFà× Ñ  °ÐB\Ð Ó]Ð]r"   c                 óÂ   — | j                   j                  «        | j                  j                  «        t	        | «      | _        | j
                  j                  «        | S r   )r  Úclearr  Ú	__enter__Ú_FlopCounterModer  r!  s    r    rE  zFlopCounterMode.__enter__Ü  sG   € Ø×Ñ×ÑÔ Ø×Ñ×"Ñ"Ô$Ü$ TÓ*ˆŒ	Ø�	‰	×ÑÔØˆr"   c                 óö   — | j                   €J ‚ | j                   j                  |Ž }d | _         | j                  j                  «        | j                  r$t	        | j                  | j                  «      «       |S r   )r  Ú__exit__r  r
  ÚprintrB  r	  )r  r(   rP   s      r    rH  zFlopCounterMode.__exit__ã  sb   € Ø�y‰yÐ$Ð$Ð$ØˆD�I‰I×Ñ Ð%ˆØˆŒ	Ø×Ñ×!Ñ!Ô#Ø�<Š<Ü�$—.‘. §¡Ó,Ô-Øˆr"   c                 óÔ   — || j                   v rY| j                   |   } ||i |¤d|i¤Ž}t        | j                  j                  «      D ]  }| j                  |   |xx   |z  cc<   Œ |S )Nr%   )r#   Úsetr  Úparentsr  )r  Úfunc_packetrâ   r(   r)   Úflop_count_funcrx   Úpars           r    Ú_count_flopszFlopCounterMode._count_flopsì  sx   € Ø˜$×,Ñ,Ñ,Ø"×0Ñ0°Ñ=ˆOÙ(¨$ÐF°&ÑFÀ#ÒFˆJÜ˜4×+Ñ+×3Ñ3Ó4ò A�Ø× Ñ  Ñ% kÓ2°jÑ@Ô2ðAð ˆ
r"   )NrA   TNr   )Ú__name__Ú
__module__Ú__qualname__Ú__doc__r	   r
   r   Únnr'  rr   r  Úboolr$  r   r  r"  rö   r%  rB  rE  rH  rP  Ú__classcell__)r  s   @r    r   r   g  s¶   ø„ ñð* MQØØ Ø7;ñ+à˜5 §¡§¡°$°u·x±x·±Ñ2GÐ!GÑHÑIð+ð ð+ð ð	+ð
 % T¨#¨s¨(¡^Ñ4õ+ð*8 ó 8ð
A  c¨4°°S°©>Ð&9Ñ!:ó 
Aó:^òzòör"   c                   ó    — e Zd Zdefd„Zdd„Zy)rF  Úcounterc                 ó   — || _         y r   )rY  )r  rY  s     r    r  z_FlopCounterMode.__init__÷  s	   € Øˆ�r"   Nc                 óx  — |r|ni }|t         j                  j                  j                  j                  t         j                  j                  j                  j
                  t         j                  j                  j                  j                  t         j                  j                  j                  j                  t         j                  j                  j                  j                  t         j                  j                  j                  j                  t         j                  j                  j                  j                  t         j                  j                  j                  j                  t         j                  j                  j                  j                  t         j                  j                  j                  j                  t         j                  j                  j                  j                  t         j                  j                  j                  j                  t         j                  j                  j                   j                  t         j                  j"                  j$                  j                  hv rt&        S || j(                  j*                  vra|t         j                  j"                  j,                  j                  ur1| 5   |j.                  |i |¤Ž}|t&        ur|cd d d «       S 	 d d d «        ||i |¤Ž}| j(                  j1                  |j2                  |||«      S # 1 sw Y   Œ9xY wr   )r   ÚopsÚatenÚis_contiguousÚdefaultÚmemory_formatÚis_strides_like_formatÚis_non_overlapping_and_denserš   Úsym_sizeÚstrideÚ
sym_strideÚstorage_offsetÚsym_storage_offsetÚnumelÚ	sym_numelÚdimÚprimÚlayoutÚNotImplementedrY  r#   r—   Ú	decomposerP  Ú_overloadpacket)r  ÚfuncÚtypesr(   r)   Úrrâ   s          r    Ú__torch_dispatch__z#_FlopCounterMode.__torch_dispatch__ú  s	  € Ù!‘ rˆð ”E—I‘I—N‘N×0Ñ0×8Ñ8Ü—I‘I—N‘N×0Ñ0×>Ñ>Ü—I‘I—N‘N×9Ñ9×AÑAÜ—I‘I—N‘N×?Ñ?×GÑGÜ—I‘I—N‘N×'Ñ'×/Ñ/Ü—I‘I—N‘N×+Ñ+×3Ñ3Ü—I‘I—N‘N×)Ñ)×1Ñ1Ü—I‘I—N‘N×-Ñ-×5Ñ5Ü—I‘I—N‘N×1Ñ1×9Ñ9Ü—I‘I—N‘N×5Ñ5×=Ñ=Ü—I‘I—N‘N×(Ñ(×0Ñ0Ü—I‘I—N‘N×,Ñ,×4Ñ4Ü—I‘I—N‘N×&Ñ&×.Ñ.Ü—I‘I—N‘N×)Ñ)×1Ñ1ð3ñ 3ô "Ð!ð �t—|‘|×1Ñ1Ñ1°dÄ%Ç)Á)Ç.Á.×BWÑBW×B_ÑB_Ñ6_Øñ Ø"�D—N‘N DÐ3¨FÑ3�ØœNÑ*Ø÷ñ à*÷ñ �DÐ#˜FÑ#ˆØ�|‰|×(Ñ(¨×)=Ñ)=¸sÀDÈ&ÓQÐQ÷ð ús   ËL0Ì0L9)rB   N)rQ  rR  rS  r   r  rs  rB   r"   r    rF  rF  ö  s   „ ð ó ôRr"   rF  )Fr   )NNNFN)Vr   Útorch.utils._pytreer   r   r   Úmodule_trackerr   Útypingr   r	   r
   r   r   Úcollections.abcr   Útyping_extensionsr   Úcollectionsr   Útorch.utils._python_dispatchr   Úmathr   Ú	functoolsr   r  Ú__all__r   r   r\  r]  r!   r#   r$  Ú__annotations__r-   r   Úmmr  rI   ÚaddmmrN   ÚbmmrS   ÚbaddbmmrV   Ú
_scaled_mmr^   rr   rV  rh   ÚconvolutionÚ_convolutionro   Úconvolution_backwardr{   rŒ   Ú'_scaled_dot_product_efficient_attentionÚ#_scaled_dot_product_flash_attentionÚ#_scaled_dot_product_cudnn_attentionr�   r�   rê   r¸   rÃ   Ú_flash_attention_forwardrÎ   Ú_efficient_attention_forwardrÓ   rÙ   Ú0_scaled_dot_product_efficient_attention_backwardÚ,_scaled_dot_product_flash_attention_backwardÚ,_scaled_dot_product_cudnn_attention_backwardrÜ   Ú_flash_attention_backwardrå   Ú_efficient_attention_backwardrè   rì   rõ   rù   rü   r  r  r   rF  rB   r"   r    ú<module>r‘     s¡  ðä ß FÑ FÝ )ß :Õ :Ý $Ý 'Ý #Ý :Ý Ý Û àÐ5Ð
6€áˆTƒ]€Ùˆtƒ_€à‡y�y‡~�~€òð
 !#€ˆt�C˜�H‰~Ó "òñ°X¸xÈÈBÈÑ?OÐ>PÐRZÐ[]Ð_aÐ[aÑRbÐ>bÑ5có ñ, �t—w‘wÓØ/3ò À#ò ó  ðñ �t—z‘zÓ"ñ%È#ò %ó #ð%ñ �t—x‘xÓ ñ
¸Cò 
ó !ð
ñ �t—|‘|Ó$ñ&ÈCò &ó %ð&ñ �t—‘Ó'ð ØØØØñ%ð 	ò%ó (ð%ð( ñ	%Ø�#‰Yð%à�#‰Yð%ð �C‰yð%ð ð	%ð
 	ó%ñN ˜×(Ñ(¨$×*;Ñ*;Ð<Ó=Øbfò OÐuxò Oó >ðOñ
 �t×0Ñ0Ó1ðeð òeó 2ðeòNñ$ ˜×DÑDØ×@Ñ@Ø×@Ñ@ðBó Cð EIò @ÐWZò @óCð@ò	-ð" ò+`ð ˆe�E˜#˜s˜(‘O U¨3°¨8¡_°e¸CÀ¸H±oÀxÐPUÐVYÐ[^ÐV^ÑP_ÑG`Ð`ÑaÑbó+`ðf ò-`ð ˆe�E˜#˜s˜(‘O U¨3°¨8¡_°e¸CÀ¸H±oÀxÐPUÐVYÐ[^ÐV^ÑP_ÑG`Ð`ÑaÑbó-`ñ` �t×4Ñ4¸dÔCð òð 	òó Dðñ> �t×8Ñ8À$ÔGðð 	òó Hðò>ñ6 ˜×MÑMØ×IÑIØ×IÑIðKó Lð ^bò YÐpsò YóLðYñ �t×5Ñ5¸tÔDðð 	òó Eðñ@ �t×9Ñ9À4ÔHðð 	òó Iðð@Ø‡G�GˆWðà‡J�J�
ðð 	‡H�Hˆhðð 	‡L�L�,ð	ð
 	‡O�O�_ðð 	×Ñ�iðð 	×Ñ�yðð 	×ÑÐ1ðð 	×0Ñ0°)ðð 	×,Ñ,¨iðð 	×,Ñ,¨iðð 	×9Ñ9Ð;Mðð 	×5Ñ5Ð7Iðð 	×5Ñ5Ð7Iðð 	×!Ñ!Ð#@ðð  	×%Ñ%Ð'Hð!ð" 	×"Ñ"Ð$Bð#ð$ 	×&Ñ&Ð(Jñ%€ò*ò $€òò#ò ò
÷Lñ Lô^"RÐ(õ "Rr"   