Ë
    S^(h-ª  ã                  ó$  — d Z ddlmZ ddlZddlmZ ddlmZm	Z	m
Z
 ddlZddlmZ ddlmZmZmZmZmZmZmZ dd	lmZmZ dd
lmZmZmZmZmZ ddl m!Z!  ejD                  e#«      Z$dZ%e G d„ de«      «       Z& G d„ dejN                  jP                  «      Z) G d„ dejN                  jP                  «      Z* G d„ dejN                  jP                  «      Z+ G d„ dejN                  jP                  «      Z, G d„ dejN                  jP                  «      Z- G d„ dejN                  jP                  «      Z. G d„ dejN                  jP                  «      Z/ G d„ dejN                  jP                  «      Z0 G d „ d!ejN                  jP                  «      Z1 G d"„ d#ejN                  jP                  «      Z2 G d$„ d%ejN                  jP                  «      Z3 G d&„ d'ejN                  jP                  «      Z4 G d(„ d)ejN                  jP                  «      Z5 G d*„ d+ejN                  jP                  «      Z6e G d,„ d-ejN                  jP                  «      «       Z7 G d.„ d/e«      Z8d0Z9d1Z: ed2e9«       G d3„ d4e8«      «       Z; ed5e9«       G d6„ d7e8e«      «       Z<g d8¢Z=y)9zTF 2.0 Cvt model.é    )ÚannotationsN)Ú	dataclass)ÚOptionalÚTupleÚUnioné   )Ú&TFImageClassifierOutputWithNoAttention)ÚTFModelInputTypeÚTFPreTrainedModelÚTFSequenceClassificationLossÚget_initializerÚkerasÚkeras_serializableÚunpack_inputs)Ú
shape_listÚstable_softmax)ÚModelOutputÚadd_start_docstringsÚ%add_start_docstrings_to_model_forwardÚloggingÚreplace_return_docstringsé   )Ú	CvtConfigr   c                  ó<   — e Zd ZU dZdZded<   dZded<   dZded<   y)ÚTFBaseModelOutputWithCLSTokena2  
    Base class for model's outputs.

    Args:
        last_hidden_state (`tf.Tensor` of shape `(batch_size, sequence_length, hidden_size)`):
            Sequence of hidden-states at the output of the last layer of the model.
        cls_token_value (`tf.Tensor` of shape `(batch_size, 1, hidden_size)`):
            Classification token at the output of the last layer of the model.
        hidden_states (`tuple(tf.Tensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
            Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of shape
            `(batch_size, sequence_length, hidden_size)`. Hidden-states of the model at the output of each layer plus
            the initial embedding outputs.
    NzOptional[tf.Tensor]Úlast_hidden_stateÚcls_token_valuezTuple[tf.Tensor, ...] | NoneÚhidden_states)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   Ú__annotations__r   r   © ó    úe/var/www/skyplay_api_hub/venv/lib/python3.12/site-packages/transformers/models/cvt/modeling_tf_cvt.pyr   r   4   s+   … ñð .2ÐÐ*Ó1Ø+/€OÐ(Ó/Ø26€MÐ/Ô6r%   r   c                  ó.   ‡ — e Zd ZdZdˆ fd„Zddd„Zˆ xZS )ÚTFCvtDropPathz£Drop paths (Stochastic Depth) per sample (when applied in main path of residual blocks).
    References:
        (1) github.com:rwightman/pytorch-image-models
    c                ó2   •— t        ‰| �  di |¤Ž || _        y )Nr$   )ÚsuperÚ__init__Ú	drop_prob)Úselfr,   ÚkwargsÚ	__class__s      €r&   r+   zTFCvtDropPath.__init__O   s   ø€ Ü‰ÑÑ"˜6Ò"Ø"ˆ�r%   c                ó\  — | j                   dk(  s|s|S d| j                   z
  }t        j                  |«      d   fdt        t        j                  |«      «      dz
  z  z   }|t        j                  j                  |dd| j                  ¬«      z   }t        j                  |«      }||z  |z  S )Nç        r   r   )r   )Údtype)r,   ÚtfÚshapeÚlenÚrandomÚuniformÚcompute_dtypeÚfloor)r-   ÚxÚtrainingÚ	keep_probr4   Úrandom_tensors         r&   ÚcallzTFCvtDropPath.callS   s˜   € Ø�>‰>˜SÒ ©ØˆHØ˜Ÿ™Ñ&ˆ	Ü—‘˜!“˜Q‘Ð! D¬C´·±¸³Ó,<¸qÑ,@Ñ$AÑAˆØ!¤B§I¡I×$5Ñ$5°e¸QÀÈ×I[ÑI[Ð$5Ó$\Ñ\ˆÜŸ™ Ó/ˆØ�I‘ Ñ.Ð.r%   )r,   Úfloat©N)r:   ú	tf.Tensor)r   r    r!   r"   r+   r>   Ú__classcell__©r/   s   @r&   r(   r(   I   s   ø„ ñõ
#÷/ð /r%   r(   c                  óR   ‡ — e Zd ZdZ	 	 	 	 	 	 	 	 	 	 	 	 	 	 dˆ fd„Zddd„Zdd„Zˆ xZS )	ÚTFCvtEmbeddingsz-Construct the Convolutional Token Embeddings.c           	     óš   •— t        ‰	| �  di |¤Ž t        ||||||d¬«      | _        t        j
                  j                  |«      | _        y )NÚconvolution_embeddings)Ú
patch_sizeÚnum_channelsÚ	embed_dimÚstrideÚpaddingÚnamer$   )r*   r+   ÚTFCvtConvEmbeddingsrG   r   ÚlayersÚDropoutÚdropout)
r-   ÚconfigrH   rI   rJ   rK   rL   Údropout_rater.   r/   s
            €r&   r+   zTFCvtEmbeddings.__init__`   sO   ø€ ô 	‰ÑÑ"˜6Ò"Ü&9ØØ!Ø%ØØØØ)ô'
ˆÔ#ô —|‘|×+Ñ+¨LÓ9ˆ�r%   c                óN   — | j                  |«      }| j                  ||¬«      }|S ©N©r;   )rG   rQ   )r-   Úpixel_valuesr;   Úhidden_states       r&   r>   zTFCvtEmbeddings.callw   s*   € Ø×2Ñ2°<Ó@ˆØ—|‘| L¸8�|ÓDˆØÐr%   c                óú   — | j                   ry d| _         t        | dd «      �Nt        j                  | j                  j
                  «      5  | j                  j                  d «       d d d «       y y # 1 sw Y   y xY w)NTrG   )ÚbuiltÚgetattrr3   Ú
name_scoperG   rM   Úbuild©r-   Úinput_shapes     r&   r]   zTFCvtEmbeddings.build|   óo   € Ø�:Š:ØØˆŒ
Ü�4Ð1°4Ó8ÐDÜ—‘˜t×:Ñ:×?Ñ?Ó@ñ 8Ø×+Ñ+×1Ñ1°$Ô7÷8ð 8ð E÷8ð 8úó   ÁA1Á1A:)rR   r   rH   ÚintrI   rb   rJ   rb   rK   rb   rL   rb   rS   r?   ©F)rW   rA   r;   ÚboolÚreturnrA   r@   ©r   r    r!   r"   r+   r>   r]   rB   rC   s   @r&   rE   rE   ]   sY   ø„ Ù7ð:àð:ð ð:ð ð	:ð
 ð:ð ð:ð ð:ð õ:ô.÷
8r%   rE   c                  óL   ‡ — e Zd ZdZ	 	 	 	 	 	 	 	 	 	 	 	 dˆ fd„Zdd„Zdd„Zˆ xZS )rN   zcImage to Convolution Embeddings. This convolutional operation aims to model local spatial contexts.c           
     ó°  •— t        ‰| �  d	i |¤Ž t        j                  j	                  |¬«      | _        t        |t        j                  j                  «      r|n||f| _
        t        j                  j                  |||ddt        |j                  «      d¬«      | _        t        j                  j                  dd¬«      | _        || _        || _        y )
N©rL   ÚvalidÚchannels_lastÚ
projection)ÚfiltersÚkernel_sizeÚstridesrL   Údata_formatÚkernel_initializerrM   çñhãˆµøä>Únormalization©ÚepsilonrM   r$   )r*   r+   r   rO   ÚZeroPadding2DrL   Ú
isinstanceÚcollectionsÚabcÚIterablerH   ÚConv2Dr   Úinitializer_rangerl   ÚLayerNormalizationrs   rI   rJ   )	r-   rR   rH   rI   rJ   rK   rL   r.   r/   s	           €r&   r+   zTFCvtConvEmbeddings.__init__ˆ   sº   ø€ ô 	‰ÑÑ"˜6Ò"Ü—|‘|×1Ñ1¸'Ð1ÓBˆŒÜ(2°:¼{¿¹×?WÑ?WÔ(X™*Ð_iÐkuÐ^vˆŒÜŸ,™,×-Ñ-ØØ"ØØØ'Ü.¨v×/GÑ/GÓHØð .ó 
ˆŒô #Ÿ\™\×<Ñ<ÀTÐP_Ð<Ó`ˆÔØ(ˆÔØ"ˆ�r%   c                ó&  — t        |t        «      r|d   }| j                  | j                  |«      «      }t	        |«      \  }}}}||z  }t        j                  ||||f¬«      }| j                  |«      }t        j                  |||||f¬«      }|S )NrW   ©r4   )rw   Údictrl   rL   r   r3   Úreshapers   )r-   rW   Ú
batch_sizeÚheightÚwidthrI   Úhidden_sizes          r&   r>   zTFCvtConvEmbeddings.call£   s•   € Ü�l¤DÔ)Ø'¨Ñ7ˆLà—‘ t§|¡|°LÓ'AÓBˆô 3=¸\Ó2JÑ/ˆ
�F˜E <Ø˜u‘nˆÜ—z‘z ,°zÀ;ÐP\Ð6]Ô^ˆØ×)Ñ)¨,Ó7ˆô —z‘z ,°zÀ6È5ÐR^Ð6_Ô`ˆØÐr%   c                óü  — | j                   ry d| _         t        | dd «      �[t        j                  | j                  j
                  «      5  | j                  j                  d d d | j                  g«       d d d «       t        | dd «      �[t        j                  | j                  j
                  «      5  | j                  j                  d d | j                  g«       d d d «       y y # 1 sw Y   ŒrxY w# 1 sw Y   y xY w)NTrl   rs   )
rZ   r[   r3   r\   rl   rM   r]   rI   rs   rJ   r^   s     r&   r]   zTFCvtConvEmbeddings.build³   sÜ   € Ø�:Š:ØØˆŒ
Ü�4˜ tÓ,Ð8Ü—‘˜tŸ™×3Ñ3Ó4ñ MØ—‘×%Ñ% t¨T°4¸×9JÑ9JÐ&KÔL÷Mä�4˜¨$Ó/Ð;Ü—‘˜t×1Ñ1×6Ñ6Ó7ñ GØ×"Ñ"×(Ñ(¨$°°d·n±nÐ)EÔF÷Gð Gð <÷Mð Mú÷Gð Gús   Á*C&Â3)C2Ã&C/Ã2C;)rR   r   rH   rb   rI   rb   rJ   rb   rK   rb   rL   rb   )rW   rA   re   rA   r@   rf   rC   s   @r&   rN   rN   …   sP   ø„ Ùmð#àð#ð ð#ð ð	#ð
 ð#ð ð#ð õ#ó6÷ 	Gr%   rN   c                  ó6   ‡ — e Zd ZdZdˆ fd„Zddd„Zdd„Zˆ xZS )	Ú TFCvtSelfAttentionConvProjectionzConvolutional projection layer.c           
     óH  •— t        ‰| �  d
i |¤Ž t        j                  j	                  |¬«      | _        t        j                  j                  ||t        |j                  «      d|dd|¬«      | _	        t        j                  j                  ddd¬	«      | _        || _        y )Nri   rj   FÚconvolution)rm   rn   rq   rL   ro   Úuse_biasrM   Úgroupsrr   gÍÌÌÌÌÌì?rs   )ru   ÚmomentumrM   r$   )r*   r+   r   rO   rv   rL   r{   r   r|   rŠ   ÚBatchNormalizationrs   rJ   )r-   rR   rJ   rn   rK   rL   r.   r/   s          €r&   r+   z)TFCvtSelfAttentionConvProjection.__init__Â   s”   ø€ Ü‰ÑÑ"˜6Ò"Ü—|‘|×1Ñ1¸'Ð1ÓBˆŒÜ Ÿ<™<×.Ñ.ØØ#Ü.¨v×/GÑ/GÓHØØØØØð /ó 	
ˆÔô #Ÿ\™\×<Ñ<ÀTÐTWÐ^mÐ<ÓnˆÔØ"ˆ�r%   c                ól   — | j                  | j                  |«      «      }| j                  ||¬«      }|S rU   )rŠ   rL   rs   ©r-   rX   r;   s      r&   r>   z%TFCvtSelfAttentionConvProjection.callÓ   s6   € Ø×'Ñ'¨¯©°\Ó(BÓCˆØ×)Ñ)¨,ÀÐ)ÓJˆØÐr%   c                óþ  — | j                   ry d| _         t        | dd «      �[t        j                  | j                  j
                  «      5  | j                  j                  d d d | j                  g«       d d d «       t        | dd «      �\t        j                  | j                  j
                  «      5  | j                  j                  d d d | j                  g«       d d d «       y y # 1 sw Y   ŒsxY w# 1 sw Y   y xY w)NTrŠ   rs   )	rZ   r[   r3   r\   rŠ   rM   r]   rJ   rs   r^   s     r&   r]   z&TFCvtSelfAttentionConvProjection.buildØ   sà   € Ø�:Š:ØØˆŒ
Ü�4˜¨Ó-Ð9Ü—‘˜t×/Ñ/×4Ñ4Ó5ñ KØ× Ñ ×&Ñ&¨¨d°D¸$¿.¹.Ð'IÔJ÷Kä�4˜¨$Ó/Ð;Ü—‘˜t×1Ñ1×6Ñ6Ó7ñ MØ×"Ñ"×(Ñ(¨$°°d¸D¿N¹NÐ)KÔL÷Mð Mð <÷Kð Kú÷Mð Mús   Á*C'Â3*C3Ã'C0Ã3C<)
rR   r   rJ   rb   rn   rb   rK   rb   rL   rb   rc   ©rX   rA   r;   rd   re   rA   r@   rf   rC   s   @r&   rˆ   rˆ   ¿   s   ø„ Ù)õ#ô"÷
	Mr%   rˆ   c                  ó   — e Zd ZdZdd„Zy)Ú"TFCvtSelfAttentionLinearProjectionz7Linear projection layer used to flatten tokens into 1D.c                ód   — t        |«      \  }}}}||z  }t        j                  ||||f¬«      }|S )Nr   )r   r3   r�   )r-   rX   r‚   rƒ   r„   rI   r…   s          r&   r>   z'TFCvtSelfAttentionLinearProjection.callç   s<   € ä2<¸\Ó2JÑ/ˆ
�F˜E <Ø˜u‘nˆÜ—z‘z ,°zÀ;ÐP\Ð6]Ô^ˆØÐr%   N©rX   rA   re   rA   )r   r    r!   r"   r>   r$   r%   r&   r”   r”   ä   s
   „ ÙAôr%   r”   c                  óP   ‡ — e Zd ZdZ	 d	 	 	 	 	 	 	 	 	 	 	 dˆ fd„Zddd„Zd	d„Zˆ xZS )
ÚTFCvtSelfAttentionProjectionz'Convolutional Projection for Attention.c                óx   •— t        ‰| �  di |¤Ž |dk(  rt        |||||d¬«      | _        t	        «       | _        y )NÚdw_bnÚconvolution_projection©rM   r$   )r*   r+   rˆ   r›   r”   Úlinear_projection)	r-   rR   rJ   rn   rK   rL   Úprojection_methodr.   r/   s	           €r&   r+   z%TFCvtSelfAttentionProjection.__init__ò   sF   ø€ ô 	‰ÑÑ"˜6Ò"Ø Ò'Ü*JØ˜	 ;°¸ÐF^ô+ˆDÔ'ô "DÓ!EˆÕr%   c                óN   — | j                  ||¬«      }| j                  |«      }|S rU   )r›   r�   r�   s      r&   r>   z!TFCvtSelfAttentionProjection.call  s-   € Ø×2Ñ2°<È(Ð2ÓSˆØ×-Ñ-¨lÓ;ˆØÐr%   c                óú   — | j                   ry d| _         t        | dd «      �Nt        j                  | j                  j
                  «      5  | j                  j                  d «       d d d «       y y # 1 sw Y   y xY w)NTr›   )rZ   r[   r3   r\   r›   rM   r]   r^   s     r&   r]   z"TFCvtSelfAttentionProjection.build  r`   ra   )rš   )rR   r   rJ   rb   rn   rb   rK   rb   rL   rb   rž   Ústrrc   r’   r@   rf   rC   s   @r&   r˜   r˜   ï   s[   ø„ Ù1ð ")ðFàðFð ðFð ð	Fð
 ðFð ðFð õFô"÷
8r%   r˜   c                  óp   ‡ — e Zd ZdZ	 d	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dˆ fd„Zdd„Zd	d
d„Zdd„Zˆ xZS )ÚTFCvtSelfAttentionz�
    Self-attention layer. A depth-wise separable convolution operation (Convolutional Projection), is applied for
    query, key, and value embeddings.
    c           	     óª  •— t        ‰| �  di |¤Ž |dz  | _        || _        || _        || _        t        ||||||	dk(  rdn|	d¬«      | _        t        ||||||	d¬«      | _        t        ||||||	d¬«      | _	        t        j                  j                  |t        |j                  «      |
dd	¬
«      | _        t        j                  j                  |t        |j                  «      |
dd¬
«      | _        t        j                  j                  |t        |j                  «      |
dd¬
«      | _        t        j                  j%                  |«      | _        y )Ng      à¿ÚavgÚlinearÚconvolution_projection_query)rž   rM   Úconvolution_projection_keyÚconvolution_projection_valueÚzerosÚprojection_query©Úunitsrq   r‹   Úbias_initializerrM   Úprojection_keyÚprojection_valuer$   )r*   r+   ÚscaleÚwith_cls_tokenrJ   Ú	num_headsr˜   r§   r¨   r©   r   rO   ÚDenser   r|   r«   r¯   r°   rP   rQ   )r-   rR   r³   rJ   rn   Ústride_qÚ	stride_kvÚ	padding_qÚ
padding_kvÚqkv_projection_methodÚqkv_biasÚattention_drop_rater²   r.   r/   s                 €r&   r+   zTFCvtSelfAttention.__init__  sm  ø€ ô  	‰ÑÑ"˜6Ò"Ø ‘_ˆŒ
Ø,ˆÔØ"ˆŒØ"ˆŒä,HØØØØØØ*?À5Ò*H™hÐNcØ/ô-
ˆÔ)ô +GØØØØØØ3Ø-ô+
ˆÔ'ô -IØØØØØØ3Ø/ô-
ˆÔ)ô !&§¡× 2Ñ 2ØÜ.¨v×/GÑ/GÓHØØ$Ø#ð !3ó !
ˆÔô $Ÿl™l×0Ñ0ØÜ.¨v×/GÑ/GÓHØØ$Ø!ð 1ó 
ˆÔô !&§¡× 2Ñ 2ØÜ.¨v×/GÑ/GÓHØØ$Ø#ð !3ó !
ˆÔô —|‘|×+Ñ+Ð,?Ó@ˆ�r%   c                óÎ   — t        |«      \  }}}| j                  | j                  z  }t        j                  |||| j                  |f¬«      }t        j
                  |d¬«      }|S )Nr   ©r   é   r   r   ©Úperm)r   rJ   r³   r3   r�   Ú	transpose)r-   rX   r‚   r…   Ú_Úhead_dims         r&   Ú"rearrange_for_multi_head_attentionz5TFCvtSelfAttention.rearrange_for_multi_head_attention`  s\   € Ü%/°Ó%=Ñ"ˆ
�K Ø—>‘> T§^¡^Ñ3ˆÜ—z‘z ,°zÀ;ÐPT×P^ÑP^Ð`hÐ6iÔjˆÜ—|‘| L°|ÔDˆØÐr%   c                ó  — | j                   rt        j                  |d||z  gd«      \  }}t        |«      \  }}}t        j                  |||||f¬«      }| j                  ||¬«      }	| j                  ||¬«      }
| j                  ||¬«      }| j                   rKt        j                  |
fd¬«      }
t        j                  ||	fd¬«      }	t        j                  ||fd¬«      }| j                  | j                  z  }| j                  | j                  |
«      «      }
| j                  | j                  |	«      «      }	| j                  | j                  |«      «      }t        j                  |
|	d¬«      | j                   z  }t#        |d¬«      }| j%                  ||¬«      }t        j                  ||«      }t        |«      \  }}}}t        j&                  |d	¬
«      }t        j                  |||| j                  |z  f«      }|S )Nr   r   rV   ©ÚaxisT)Útranspose_béÿÿÿÿ)ÚlogitsrÇ   r½   r¿   )r²   r3   Úsplitr   r�   r¨   r§   r©   ÚconcatrJ   r³   rÄ   r«   r¯   r°   Úmatmulr±   r   rQ   rÁ   )r-   rX   rƒ   r„   r;   Ú	cls_tokenr‚   r…   rI   ÚkeyÚqueryÚvaluerÃ   Úattention_scoreÚattention_probsÚcontextrÂ   s                    r&   r>   zTFCvtSelfAttention.callg  sÕ  € Ø×ÒÜ&(§h¡h¨|¸aÀÈ%ÁÐ=PÐRSÓ&TÑ#ˆI�|ô 1;¸<Ó0HÑ-ˆ
�K Ü—z‘z ,°zÀ6È5ÐR^Ð6_Ô`ˆà×-Ñ-¨lÀXÐ-ÓNˆØ×1Ñ1°,ÈÐ1ÓRˆØ×1Ñ1°,ÈÐ1ÓRˆà×ÒÜ—I‘I˜y¨%Ð0°qÔ9ˆEÜ—)‘)˜Y¨Ð,°1Ô5ˆCÜ—I‘I˜y¨%Ð0°qÔ9ˆEà—>‘> T§^¡^Ñ3ˆà×7Ñ7¸×8MÑ8MÈeÓ8TÓUˆØ×5Ñ5°d×6IÑ6IÈ#Ó6NÓOˆØ×7Ñ7¸×8MÑ8MÈeÓ8TÓUˆäŸ)™) E¨3¸DÔAÀDÇJÁJÑNˆÜ(°ÀbÔIˆØŸ,™, À˜,ÓJˆä—)‘)˜O¨UÓ3ˆä)¨'Ó2Ñˆˆ1ˆk˜1Ü—,‘,˜w¨\Ô:ˆÜ—*‘*˜W z°;ÀÇÁÐQYÑ@YÐ&ZÓ[ˆØˆr%   c                óL  — | j                   ry d| _         t        | dd «      �Mt        j                  | j                  j
                  «      5  | j                  j                  d «       d d d «       t        | dd «      �Mt        j                  | j                  j
                  «      5  | j                  j                  d «       d d d «       t        | dd «      �Mt        j                  | j                  j
                  «      5  | j                  j                  d «       d d d «       t        | dd «      �Zt        j                  | j                  j
                  «      5  | j                  j                  d d | j                  g«       d d d «       t        | dd «      �Zt        j                  | j                  j
                  «      5  | j                  j                  d d | j                  g«       d d d «       t        | dd «      �[t        j                  | j                  j
                  «      5  | j                  j                  d d | j                  g«       d d d «       y y # 1 sw Y   �ŒõxY w# 1 sw Y   �Œ¨xY w# 1 sw Y   �Œ[xY w# 1 sw Y   �ŒxY w# 1 sw Y   Œ¦xY w# 1 sw Y   y xY w)NTr§   r¨   r©   r«   r¯   r°   )rZ   r[   r3   r\   r§   rM   r]   r¨   r©   r«   rJ   r¯   r°   r^   s     r&   r]   zTFCvtSelfAttention.build‰  s1  € Ø�:Š:ØØˆŒ
Ü�4Ð7¸Ó>ÐJÜ—‘˜t×@Ñ@×EÑEÓFñ >Ø×1Ñ1×7Ñ7¸Ô=÷>ä�4Ð5°tÓ<ÐHÜ—‘˜t×>Ñ>×CÑCÓDñ <Ø×/Ñ/×5Ñ5°dÔ;÷<ä�4Ð7¸Ó>ÐJÜ—‘˜t×@Ñ@×EÑEÓFñ >Ø×1Ñ1×7Ñ7¸Ô=÷>ä�4Ð+¨TÓ2Ð>Ü—‘˜t×4Ñ4×9Ñ9Ó:ñ JØ×%Ñ%×+Ñ+¨T°4¸¿¹Ð,HÔI÷Jä�4Ð)¨4Ó0Ð<Ü—‘˜t×2Ñ2×7Ñ7Ó8ñ HØ×#Ñ#×)Ñ)¨4°°t·~±~Ð*FÔG÷Hä�4Ð+¨TÓ2Ð>Ü—‘˜t×4Ñ4×9Ñ9Ó:ñ JØ×%Ñ%×+Ñ+¨T°4¸¿¹Ð,HÔI÷Jð Jð ?÷>ñ >ú÷<ñ <ú÷>ñ >ú÷Jñ Jú÷Hð Hú÷Jð JúsH   ÁIÂ%I'Ã?I4Å)JÇ )JÈ')JÉI$É'I1É4I>ÊJÊJÊJ#©T)rR   r   r³   rb   rJ   rb   rn   rb   rµ   rb   r¶   rb   r·   rb   r¸   rb   r¹   r¡   rº   rd   r»   r?   r²   rd   r–   rc   ©
rX   rA   rƒ   rb   r„   rb   r;   rd   re   rA   r@   )	r   r    r!   r"   r+   rÄ   r>   r]   rB   rC   s   @r&   r£   r£     s´   ø„ ñð$  $ðGAàðGAð ðGAð ð	GAð
 ðGAð ðGAð ðGAð ðGAð ðGAð  #ðGAð ðGAð #ðGAð õGAóRô ÷DJr%   r£   c                  ó6   ‡ — e Zd ZdZdˆ fd„Zddd„Zdd„Zˆ xZS )	ÚTFCvtSelfOutputzOutput of the Attention layer .c                óî   •— t        ‰| �  di |¤Ž t        j                  j	                  |t        |j                  «      d¬«      | _        t        j                  j                  |«      | _	        || _
        y ©NÚdense)r­   rq   rM   r$   )r*   r+   r   rO   r´   r   r|   rÜ   rP   rQ   rJ   )r-   rR   rJ   Ú	drop_rater.   r/   s        €r&   r+   zTFCvtSelfOutput.__init__¤  s`   ø€ Ü‰ÑÑ"˜6Ò"Ü—\‘\×'Ñ'Ø´À×@XÑ@XÓ0YÐ`gð (ó 
ˆŒ
ô —|‘|×+Ñ+¨IÓ6ˆŒØ"ˆ�r%   c                óP   — | j                  |¬«      }| j                  ||¬«      }|S ©N)Úinputs)rà   r;   ©rÜ   rQ   r�   s      r&   r>   zTFCvtSelfOutput.call¬  s*   € Ø—z‘z¨�zÓ6ˆØ—|‘|¨<À(�|ÓKˆØÐr%   c                ó  — | j                   ry d| _         t        | dd «      �[t        j                  | j                  j
                  «      5  | j                  j                  d d | j                  g«       d d d «       y y # 1 sw Y   y xY w©NTrÜ   ©rZ   r[   r3   r\   rÜ   rM   r]   rJ   r^   s     r&   r]   zTFCvtSelfOutput.build±  ór   € Ø�:Š:ØØˆŒ
Ü�4˜ $Ó'Ð3Ü—‘˜tŸz™zŸ™Ó/ñ ?Ø—
‘
× Ñ  $¨¨d¯n©nÐ!=Ô>÷?ð ?ð 4÷?ð ?úó   Á)A>Á>B)rR   r   rJ   rb   rÝ   r?   rc   r’   r@   rf   rC   s   @r&   rÙ   rÙ   ¡  s   ø„ Ù)õ#ô÷
?r%   rÙ   c                  ór   ‡ — e Zd ZdZ	 d	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dˆ fd„Zd„ Zdd	d„Zd
d„Zˆ xZS )ÚTFCvtAttentionzDAttention layer. First chunk of the convolutional transformer block.c                ó†   •— t        ‰| �  di |¤Ž t        |||||||||	|
||d¬«      | _        t	        |||d¬«      | _        y )NÚ	attentionrœ   Úoutputr$   )r*   r+   r£   rê   rÙ   Údense_output)r-   rR   r³   rJ   rn   rµ   r¶   r·   r¸   r¹   rº   r»   rÝ   r²   r.   r/   s                  €r&   r+   zTFCvtAttention.__init__½  s]   ø€ ô" 	‰ÑÑ"˜6Ò"Ü+ØØØØØØØØØ!ØØØØô
ˆŒô ,¨F°I¸yÈxÔXˆÕr%   c                ó   — t         ‚r@   )ÚNotImplementedError)r-   Úheadss     r&   Úprune_headszTFCvtAttention.prune_headsà  s   € Ü!Ð!r%   c                óV   — | j                  ||||¬«      }| j                  ||¬«      }|S rU   )rê   rì   )r-   rX   rƒ   r„   r;   Úself_outputÚattention_outputs          r&   r>   zTFCvtAttention.callã  s4   € Ø—n‘n \°6¸5È8�nÓTˆØ×,Ñ,¨[À8Ð,ÓLÐØÐr%   c                óÆ  — | j                   ry d| _         t        | dd «      �Mt        j                  | j                  j
                  «      5  | j                  j                  d «       d d d «       t        | dd «      �Nt        j                  | j                  j
                  «      5  | j                  j                  d «       d d d «       y y # 1 sw Y   ŒexY w# 1 sw Y   y xY w)NTrê   rì   )rZ   r[   r3   r\   rê   rM   r]   rì   r^   s     r&   r]   zTFCvtAttention.buildè  s¹   € Ø�:Š:ØØˆŒ
Ü�4˜ dÓ+Ð7Ü—‘˜tŸ~™~×2Ñ2Ó3ñ +Ø—‘×$Ñ$ TÔ*÷+ä�4˜¨Ó.Ð:Ü—‘˜t×0Ñ0×5Ñ5Ó6ñ .Ø×!Ñ!×'Ñ'¨Ô-÷.ð .ð ;÷+ð +ú÷.ð .ús   ÁCÂ%CÃCÃC rÖ   )rR   r   r³   rb   rJ   rb   rn   rb   rµ   rb   r¶   rb   r·   rb   r¸   rb   r¹   r¡   rº   rd   r»   r?   rÝ   r?   r²   rd   rc   )rX   rA   rƒ   rb   r„   rb   r;   rd   r@   )	r   r    r!   r"   r+   rð   r>   r]   rB   rC   s   @r&   rè   rè   º  s®   ø„ ÙNð   $ð!Yàð!Yð ð!Yð ð	!Yð
 ð!Yð ð!Yð ð!Yð ð!Yð ð!Yð  #ð!Yð ð!Yð #ð!Yð ð!Yð õ!YòF"ô ÷
	.r%   rè   c                  ó4   ‡ — e Zd ZdZdˆ fd„Zdd„Zdd„Zˆ xZS )ÚTFCvtIntermediatezNIntermediate dense layer. Second chunk of the convolutional transformer block.c                óÀ   •— t        ‰| �  di |¤Ž t        j                  j	                  t        ||z  «      t        |j                  «      dd¬«      | _        || _	        y )NÚgelurÜ   )r­   rq   Ú
activationrM   r$   )
r*   r+   r   rO   r´   rb   r   r|   rÜ   rJ   )r-   rR   rJ   Ú	mlp_ratior.   r/   s        €r&   r+   zTFCvtIntermediate.__init__÷  sX   ø€ Ü‰ÑÑ"˜6Ò"Ü—\‘\×'Ñ'Ü�i )Ñ+Ó,Ü.¨v×/GÑ/GÓHØØð	 (ó 
ˆŒ
ð #ˆ�r%   c                ó(   — | j                  |«      }|S r@   )rÜ   )r-   rX   s     r&   r>   zTFCvtIntermediate.call  s   € Ø—z‘z ,Ó/ˆØÐr%   c                ó  — | j                   ry d| _         t        | dd «      �[t        j                  | j                  j
                  «      5  | j                  j                  d d | j                  g«       d d d «       y y # 1 sw Y   y xY wrã   rä   r^   s     r&   r]   zTFCvtIntermediate.build  rå   ræ   )rR   r   rJ   rb   rú   rb   r–   r@   rf   rC   s   @r&   rö   rö   ô  s   ø„ ÙXõ#ó÷?r%   rö   c                  ó6   ‡ — e Zd ZdZdˆ fd„Zddd„Zdd„Zˆ xZS )	ÚTFCvtOutputzu
    Output of the Convolutional Transformer Block (last chunk). It consists of a MLP and a residual connection.
    c                óü   •— t        ‰| �  di |¤Ž t        j                  j	                  |t        |j                  «      d¬«      | _        t        j                  j                  |«      | _	        || _
        || _        y rÛ   )r*   r+   r   rO   r´   r   r|   rÜ   rP   rQ   rJ   rú   )r-   rR   rJ   rú   rÝ   r.   r/   s         €r&   r+   zTFCvtOutput.__init__  sg   ø€ Ü‰ÑÑ"˜6Ò"Ü—\‘\×'Ñ'Ø´À×@XÑ@XÓ0YÐ`gð (ó 
ˆŒ
ô —|‘|×+Ñ+¨IÓ6ˆŒØ"ˆŒØ"ˆ�r%   c                óZ   — | j                  |¬«      }| j                  ||¬«      }||z   }|S rß   rá   )r-   rX   Úinput_tensorr;   s       r&   r>   zTFCvtOutput.call  s4   € Ø—z‘z¨�zÓ6ˆØ—|‘|¨<À(�|ÓKˆØ# lÑ2ˆØÐr%   c           	     ó@  — | j                   ry d| _         t        | dd «      �qt        j                  | j                  j
                  «      5  | j                  j                  d d t        | j                  | j                  z  «      g«       d d d «       y y # 1 sw Y   y xY wrã   )
rZ   r[   r3   r\   rÜ   rM   r]   rb   rJ   rú   r^   s     r&   r]   zTFCvtOutput.build"  s…   € Ø�:Š:ØØˆŒ
Ü�4˜ $Ó'Ð3Ü—‘˜tŸz™zŸ™Ó/ñ UØ—
‘
× Ñ  $¨¬c°$·.±.À4Ç>Á>Ñ2QÓ.RÐ!SÔT÷Uð Uð 4÷Uð Uús   Á?BÂB)rR   r   rJ   rb   rú   rb   rÝ   rb   rc   )rX   rA   r  rA   r;   rd   re   rA   r@   rf   rC   s   @r&   rþ   rþ     s   ø„ ñõ#ô÷Ur%   rþ   c                  ót   ‡ — e Zd ZdZ	 d	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dˆ fd„Zddd„Zd	d„Zˆ xZS )
Ú
TFCvtLayera&  
    Convolutional Transformer Block composed by attention layers, normalization and multi-layer perceptrons (mlps). It
    consists of 3 chunks : an attention layer, an intermediate dense layer and an output layer. This corresponds to the
    `Block` class in the original implementation.
    c                óÈ  •— t        ‰| �  di |¤Ž t        |||||||||	|
|||d¬«      | _        t	        |||d¬«      | _        t        ||||d¬«      | _        |dkD  rt        |d¬«      n t        j                  j                  dd¬«      | _        t        j                  j                  dd	¬
«      | _        t        j                  j                  dd¬
«      | _        || _        y )Nrê   rœ   Úintermediaterë   r1   Ú	drop_pathr¦   rr   Úlayernorm_beforert   Úlayernorm_afterr$   )r*   r+   rè   rê   rö   r  rþ   rì   r(   r   rO   Ú
Activationr  r}   r  r	  rJ   )r-   rR   r³   rJ   rn   rµ   r¶   r·   r¸   r¹   rº   r»   rÝ   rú   Údrop_path_rater²   r.   r/   s                    €r&   r+   zTFCvtLayer.__init__2  sî   ø€ ô& 	‰ÑÑ"˜6Ò"Ü'ØØØØØØØØØ!ØØØØØô
ˆŒô  .¨f°iÀÐQ_Ô`ˆÔÜ'¨°	¸9ÀiÐV^Ô_ˆÔð  Ò#ô ˜.¨{Õ;ä—‘×(Ñ(¨¸Ð(ÓDð 	Œô !&§¡× ?Ñ ?ÈÐSeÐ ?Ó fˆÔÜ$Ÿ|™|×>Ñ>ÀtÐRcÐ>ÓdˆÔØ"ˆ�r%   c                ó  — | j                  | j                  |«      |||¬«      }| j                  ||¬«      }||z   }| j                  |«      }| j	                  |«      }| j                  ||«      }| j                  ||¬«      }|S rU   )rê   r  r  r	  r  rì   )r-   rX   rƒ   r„   r;   ró   Úlayer_outputs          r&   r>   zTFCvtLayer.callc  s”   € àŸ>™>¨$×*?Ñ*?ÀÓ*MÈvÐW\Ðgo˜>ÓpÐØŸ>™>Ð*:ÀX˜>ÓNÐð (¨,Ñ6ˆð ×+Ñ+¨LÓ9ˆØ×(Ñ(¨Ó6ˆð ×(Ñ(¨°|ÓDˆØ—~‘~ l¸X�~ÓFˆØÐr%   c                ó2  — | j                   ry d| _         t        | dd «      �Mt        j                  | j                  j
                  «      5  | j                  j                  d «       d d d «       t        | dd «      �Mt        j                  | j                  j
                  «      5  | j                  j                  d «       d d d «       t        | dd «      �Mt        j                  | j                  j
                  «      5  | j                  j                  d «       d d d «       t        | dd «      �Mt        j                  | j                  j
                  «      5  | j                  j                  d «       d d d «       t        | dd «      �Zt        j                  | j                  j
                  «      5  | j                  j                  d d | j                  g«       d d d «       t        | dd «      �[t        j                  | j                  j
                  «      5  | j                  j                  d d | j                  g«       d d d «       y y # 1 sw Y   �ŒèxY w# 1 sw Y   �Œ›xY w# 1 sw Y   �ŒNxY w# 1 sw Y   �ŒxY w# 1 sw Y   Œ¦xY w# 1 sw Y   y xY w)NTrê   r  rì   r  r  r	  )rZ   r[   r3   r\   rê   rM   r]   r  rì   r  r  rJ   r	  r^   s     r&   r]   zTFCvtLayer.buildt  s  € Ø�:Š:ØØˆŒ
Ü�4˜ dÓ+Ð7Ü—‘˜tŸ~™~×2Ñ2Ó3ñ +Ø—‘×$Ñ$ TÔ*÷+ä�4˜¨Ó.Ð:Ü—‘˜t×0Ñ0×5Ñ5Ó6ñ .Ø×!Ñ!×'Ñ'¨Ô-÷.ä�4˜¨Ó.Ð:Ü—‘˜t×0Ñ0×5Ñ5Ó6ñ .Ø×!Ñ!×'Ñ'¨Ô-÷.ä�4˜ dÓ+Ð7Ü—‘˜tŸ~™~×2Ñ2Ó3ñ +Ø—‘×$Ñ$ TÔ*÷+ä�4Ð+¨TÓ2Ð>Ü—‘˜t×4Ñ4×9Ñ9Ó:ñ JØ×%Ñ%×+Ñ+¨T°4¸¿¹Ð,HÔI÷Jä�4Ð*¨DÓ1Ð=Ü—‘˜t×3Ñ3×8Ñ8Ó9ñ IØ×$Ñ$×*Ñ*¨D°$¸¿¹Ð+GÔH÷Ið Ið >÷+ñ +ú÷.ñ .ú÷.ñ .ú÷+ñ +ú÷Jð Jú÷Ið IúsH   ÁIÂ%IÃ?I'ÅI4Æ3)JÈ)JÉIÉI$É'I1É4I>ÊJ
ÊJrÖ   )rR   r   r³   rb   rJ   rb   rn   rb   rµ   rb   r¶   rb   r·   rb   r¸   rb   r¹   r¡   rº   rd   r»   r?   rÝ   r?   rú   r?   r  r?   r²   rd   rc   r×   r@   rf   rC   s   @r&   r  r  +  s²   ø„ ñð,  $ð!/#àð/#ð ð/#ð ð	/#ð
 ð/#ð ð/#ð ð/#ð ð/#ð ð/#ð  #ð/#ð ð/#ð #ð/#ð ð/#ð ð/#ð ð/#ð  õ!/#ôb÷"Ir%   r  c                  ó6   ‡ — e Zd ZdZdˆ fd„Zddd„Zdd„Zˆ xZS )	Ú
TFCvtStageaK  
    Cvt stage (encoder block). Each stage has 2 parts :
    - (1) A Convolutional Token Embedding layer
    - (2) A Convolutional Transformer Block (layer).
    The classification token is added only in the last stage.

    Args:
        config ([`CvtConfig`]): Model configuration class.
        stage (`int`): Stage number.
    c                ó°  •— t        ‰| �  di |¤Ž || _        || _        | j                  j                  | j                     rQ| j                  dd| j                  j                  d   ft        | j                  j                  «      dd¬«      | _        t        | j                  |j                  | j                     | j                  dk(  r|j                  n|j                  | j                  dz
     |j                  | j                     |j                  | j                     |j                  | j                     |j                  | j                     d¬«      | _        t!        j"                  d	|j$                  | j                     |j&                  |   «      }|D �cg c]   }|j)                  «       j+                  «       ‘Œ" }}t-        |j&                  | j                     «      D �cg c�]X  }t/        |f|j0                  | j                     |j                  | j                     |j2                  | j                     |j4                  | j                     |j6                  | j                     |j8                  | j                     |j:                  | j                     |j<                  | j                     |j>                  | j                     |j@                  | j                     |j                  | j                     |jB                  | j                     || j                     |j                  | j                     d
|› �dœŽ‘�Œ[ c}| _"        y c c}w c c}w )Nr   rÉ   Tzcvt.encoder.stages.2.cls_token)r4   ÚinitializerÚ	trainablerM   r   Ú	embedding)rH   rI   rK   rJ   rL   rS   rM   r1   zlayers.)r³   rJ   rn   rµ   r¶   r·   r¸   r¹   rº   r»   rÝ   rú   r  r²   rM   r$   )#r*   r+   rR   ÚstagerÎ   Ú
add_weightrJ   r   r|   rE   Úpatch_sizesrI   Úpatch_strideÚpatch_paddingrÝ   r  r3   Úlinspacer  ÚdepthÚnumpyÚitemÚranger  r³   Ú
kernel_qkvrµ   r¶   r·   r¸   r¹   rº   r»   rú   rO   )r-   rR   r  r.   Údrop_path_ratesr:   Újr/   s          €r&   r+   zTFCvtStage.__init__˜  s¬  ø€ Ü‰ÑÑ"˜6Ò"ØˆŒØˆŒ
Ø�;‰;× Ñ  §¡Ò,Ø!Ÿ_™_Ø˜!˜TŸ[™[×2Ñ2°2Ñ6Ð7Ü+¨D¯K©K×,IÑ,IÓJØØ5ð	 -ó ˆDŒNô )Ø�K‰KØ×)Ñ)¨$¯*©*Ñ5Ø04·
±
¸a²˜×,Ò,ÀV×EUÑEUÐVZ×V`ÑV`ÐcdÑVdÑEeØ×&Ñ& t§z¡zÑ2Ø×&Ñ& t§z¡zÑ2Ø×(Ñ(¨¯©Ñ4Ø×)Ñ)¨$¯*©*Ñ5Øô	
ˆŒô Ÿ+™+ c¨6×+@Ñ+@ÀÇÁÑ+LÈfÏlÉlÐ[`ÑNaÓbˆØ5DÖE°˜1Ÿ7™7›9Ÿ>™>Õ+ÐEˆÐEô( ˜6Ÿ<™<¨¯
©
Ñ3Ó4÷'
ð& ô% Øðà ×*Ñ*¨4¯:©:Ñ6Ø ×*Ñ*¨4¯:©:Ñ6Ø"×-Ñ-¨d¯j©jÑ9ØŸ™¨¯©Ñ4Ø ×*Ñ*¨4¯:©:Ñ6Ø ×*Ñ*¨4¯:©:Ñ6Ø!×,Ñ,¨T¯Z©ZÑ8Ø&,×&BÑ&BÀ4Ç:Á:Ñ&NØŸ™¨¯©Ñ4Ø$*×$>Ñ$>¸t¿z¹zÑ$JØ ×*Ñ*¨4¯:©:Ñ6Ø ×*Ñ*¨4¯:©:Ñ6Ø.¨t¯z©zÑ:Ø%×/Ñ/°·
±
Ñ;Ø˜q˜c�]õ!ò
ˆ�ùò Fùò
s   Æ%MÇ'EMc                óD  — d }| j                  ||«      }t        |«      \  }}}}||z  }t        j                  ||||f¬«      }| j                  j
                  | j                     r;t        j                  | j
                  |d¬«      }t        j                  ||fd¬«      }| j                  D ]  }	 |	||||¬«      }
|
}Œ | j                  j
                  | j                     rt        j                  |d||z  gd«      \  }}t        j                  |||||f¬«      }||fS )Nr   r   )ÚrepeatsrÇ   r   rÆ   rV   )r  r   r3   r�   rR   rÎ   r  ÚrepeatrÌ   rO   rË   )r-   rX   r;   rÎ   r‚   rƒ   r„   rI   r…   ÚlayerÚlayer_outputss              r&   r>   zTFCvtStage.callÇ  s  € Øˆ	Ø—~‘~ l°HÓ=ˆô 3=¸\Ó2JÑ/ˆ
�F˜E <Ø˜u‘nˆÜ—z‘z ,°zÀ;ÐP\Ð6]Ô^ˆà�;‰;× Ñ  §¡Ò,ÜŸ	™	 $§.¡.¸*È1ÔMˆIÜŸ9™9 i°Ð%>ÀQÔGˆLà—[‘[ò 	)ˆEÙ! ,°¸ÈÔQˆMØ(‰Lð	)ð �;‰;× Ñ  §¡Ò,Ü&(§h¡h¨|¸aÀÈ%ÁÐ=PÐRSÓ&TÑ#ˆI�|ô —z‘z ,°zÀ6È5ÐR^Ð6_Ô`ˆØ˜YÐ&Ð&r%   c                óÀ  — | j                   ry d| _         t        | dd «      �Mt        j                  | j                  j
                  «      5  | j                  j                  d «       d d d «       t        | dd «      �K| j                  D ];  }t        j                  |j
                  «      5  |j                  d «       d d d «       Œ= y y # 1 sw Y   ŒbxY w# 1 sw Y   ŒUxY w)NTr  rO   )rZ   r[   r3   r\   r  rM   r]   rO   ©r-   r_   r%  s      r&   r]   zTFCvtStage.buildß  s¼   € Ø�:Š:ØØˆŒ
Ü�4˜ dÓ+Ð7Ü—‘˜tŸ~™~×2Ñ2Ó3ñ +Ø—‘×$Ñ$ TÔ*÷+ä�4˜ 4Ó(Ð4ØŸ™ò &�Ü—]‘] 5§:¡:Ó.ñ &Ø—K‘K Ô%÷&ð &ñ&ð 5÷+ð +ú÷&ð &ús   ÁCÂ*CÃCÃC	)rR   r   r  rb   rc   )rX   rA   r;   rd   r@   rf   rC   s   @r&   r  r  Œ  s   ø„ ñ	õ-
ô^'÷0
&r%   r  c                  óR   ‡ — e Zd ZdZeZdˆ fd„Z	 	 	 d	 	 	 	 	 	 	 	 	 dd„Zdd„Zˆ xZ	S )	ÚTFCvtEncoderzâ
    Convolutional Vision Transformer encoder. CVT has 3 stages of encoder blocks with their respective number of layers
    (depth) being 1, 2 and 10.

    Args:
        config ([`CvtConfig`]): Model configuration class.
    c           	     ó¼   •— t        ‰| �  di |¤Ž || _        t        t	        |j
                  «      «      D �cg c]  }t        ||d|› �¬«      ‘Œ c}| _        y c c}w )Nzstages.rœ   r$   )r*   r+   rR   r  r5   r  r  Ústages)r-   rR   r.   Ú	stage_idxr/   s       €r&   r+   zTFCvtEncoder.__init__÷  sX   ø€ Ü‰ÑÑ"˜6Ò"ØˆŒäW\Ô]`Ðag×amÑamÓ]nÓWoö
ØJSŒJ�v˜y°¸¸Ð/DÖEò
ˆ�ùò 
s   ¸Ac           	     óŒ  — |rdnd }|}t        j                  |d¬«      }d }t        | j                  «      D ]  \  }}	 |	||¬«      \  }}|sŒ||fz   }Œ t        j                  |d¬«      }|r.t	        |D �
cg c]  }
t        j                  |
d¬«      ‘Œ c}
«      }|st	        d„ |||fD «       «      S t        |||¬«      S c c}
w )Nr$   )r   r¾   r   r   r¿   rV   )r   r   r   r¾   c              3  ó&   K  — | ]	  }|€Œ|–— Œ y ­wr@   r$   )Ú.0Úvs     r&   ú	<genexpr>z$TFCvtEncoder.call.<locals>.<genexpr>  s   è ø€ Òb˜qÐTUÑTaœÑbùs   ‚Š©r   r   r   )r3   rÁ   Ú	enumerater,  Útupler   )r-   rW   Úoutput_hidden_statesÚreturn_dictr;   Úall_hidden_statesrX   rÎ   rÂ   Ústage_moduleÚhss              r&   r>   zTFCvtEncoder.callþ  sÙ   € ñ #7™B¸DÐØ#ˆô —|‘| L°|ÔDˆàˆ	Ü!*¨4¯;©;Ó!7ò 	HÑˆA�Ù&2°<È(Ô&SÑ#ˆL˜)Ú#Ø$5¸¸Ñ$GÑ!ð	Hô —|‘| L°|ÔDˆÙÜ %ÐUfÖ&gÈr¤r§|¡|°B¸\Ö'JÒ&gÓ hÐáÜÑb \°9Ð>OÐ$PÔbÓbÐbä,Ø*Ø%Ø+ô
ð 	
ùò 'hs   Á7Cc                óô   — | j                   ry d| _         t        | dd «      �K| j                  D ];  }t        j                  |j
                  «      5  |j                  d «       d d d «       Œ= y y # 1 sw Y   ŒIxY w)NTr,  )rZ   r[   r,  r3   r\   rM   r]   r(  s      r&   r]   zTFCvtEncoder.build  sp   € Ø�:Š:ØØˆŒ
Ü�4˜ 4Ó(Ð4ØŸ™ò &�Ü—]‘] 5§:¡:Ó.ñ &Ø—K‘K Ô%÷&ð &ñ&ð 5÷&ð &ús   ÁA.Á.A7	©rR   r   )FTF)
rW   r
   r6  úOptional[bool]r7  r=  r;   r=  re   ú6Union[TFBaseModelOutputWithCLSToken, Tuple[tf.Tensor]]r@   )
r   r    r!   r"   r   Úconfig_classr+   r>   r]   rB   rC   s   @r&   r*  r*  ì  s[   ø„ ñð €Lõ
ð 05Ø&*Ø#(ð
à&ð
ð -ð
ð $ð	
ð
 !ð
ð 
@ó
÷B&r%   r*  c                  ó^   ‡ — e Zd ZdZeZdˆ fd„Ze	 	 	 	 d	 	 	 	 	 	 	 	 	 dd„«       Zdd„Z	ˆ xZ
S )	ÚTFCvtMainLayerzConstruct the Cvt model.c                óV   •— t        ‰| �  di |¤Ž || _        t        |d¬«      | _        y )NÚencoderrœ   r$   )r*   r+   rR   r*  rC  )r-   rR   r.   r/   s      €r&   r+   zTFCvtMainLayer.__init__/  s(   ø€ Ü‰ÑÑ"˜6Ò"ØˆŒÜ# F°Ô;ˆ�r%   c                óª   — |€t        d«      ‚| j                  ||||¬«      }|d   }|s	|f|dd  z   S t        ||j                  |j                  ¬«      S )Nú You have to specify pixel_values©r6  r7  r;   r   r   r3  )Ú
ValueErrorrC  r   r   r   )r-   rW   r6  r7  r;   Úencoder_outputsÚsequence_outputs          r&   r>   zTFCvtMainLayer.call4  s{   € ð ÐÜÐ?Ó@Ð@àŸ,™,ØØ!5Ø#Øð	 'ó 
ˆð *¨!Ñ,ˆáØ#Ð%¨¸¸Ð(;Ñ;Ð;ä,Ø-Ø+×;Ñ;Ø)×7Ñ7ô
ð 	
r%   c                óú   — | j                   ry d| _         t        | dd «      �Nt        j                  | j                  j
                  «      5  | j                  j                  d «       d d d «       y y # 1 sw Y   y xY w)NTrC  )rZ   r[   r3   r\   rC  rM   r]   r^   s     r&   r]   zTFCvtMainLayer.buildQ  si   € Ø�:Š:ØØˆŒ
Ü�4˜ DÓ)Ð5Ü—‘˜tŸ|™|×0Ñ0Ó1ñ )Ø—‘×"Ñ" 4Ô(÷)ð )ð 6÷)ð )úra   r<  ©NNNF)
rW   zTFModelInputType | Noner6  r=  r7  r=  r;   r=  re   r>  r@   )r   r    r!   r"   r   r?  r+   r   r>   r]   rB   rC   s   @r&   rA  rA  )  sh   ø„ á"à€Lõ<ð
 ð 15Ø/3Ø&*Ø#(ð
à-ð
ð -ð
ð $ð	
ð
 !ð
ð 
@ò
ó ð
÷8)r%   rA  c                  ó   — e Zd ZdZeZdZdZy)ÚTFCvtPreTrainedModelz†
    An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained
    models.
    ÚcvtrW   N)r   r    r!   r"   r   r?  Úbase_model_prefixÚmain_input_namer$   r%   r&   rM  rM  Z  s   „ ñð
 €LØÐØ$�Or%   rM  aØ  

    This model inherits from [`TFPreTrainedModel`]. Check the superclass documentation for the generic methods the
    library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
    etc.)

    This model is also a [keras.Model](https://www.tensorflow.org/api_docs/python/tf/keras/Model) subclass. Use it
    as a regular TF 2.0 Keras Model and refer to the TF 2.0 documentation for all matter related to general usage and
    behavior.

    <Tip>

    TF 2.0 models accepts two formats as inputs:

    - having all inputs as keyword arguments (like PyTorch models), or
    - having all inputs as a list, tuple or dict in the first positional arguments.

    This second option is useful when using [`keras.Model.fit`] method which currently requires having all the
    tensors in the first argument of the model call function: `model(inputs)`.

    </Tip>

    Args:
        config ([`CvtConfig`]): Model configuration class with all the parameters of the model.
            Initializing with a config file does not load the weights associated with the model, only the
            configuration. Check out the [`~TFPreTrainedModel.from_pretrained`] method to load the model weights.
al  
    Args:
        pixel_values (`np.ndarray`, `tf.Tensor`, `List[tf.Tensor]` ``Dict[str, tf.Tensor]` or `Dict[str, np.ndarray]` and each example must have the shape `(batch_size, num_channels, height, width)`):
            Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See [`CvtImageProcessor.__call__`]
            for details.

        output_hidden_states (`bool`, *optional*):
            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
            more detail. This argument can be used only in eager mode, in graph mode the value in the config will be
            used instead.
        return_dict (`bool`, *optional*):
            Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. This argument can be used in
            eager mode, in graph mode the value will always be set to True.
        training (`bool`, *optional*, defaults to `False``):
            Whether or not to use the model in training mode (some modules like dropout modules have different
            behaviors between training and evaluation).
z]The bare Cvt Model transformer outputting raw hidden-states without any specific head on top.c                  ó†   ‡ — e Zd Zdˆ fd„Ze ee«       eee	¬«      	 	 	 	 d	 	 	 	 	 	 	 	 	 dd„«       «       «       Z
dd„Zˆ xZS )	Ú
TFCvtModelc                óP   •— t        ‰| �  |g|¢­i |¤Ž t        |d¬«      | _        y )NrN  rœ   )r*   r+   rA  rN  ©r-   rR   rà   r.   r/   s       €r&   r+   zTFCvtModel.__init__™  s(   ø€ Ü‰Ñ˜Ð3 &Ò3¨FÒ3ä! &¨uÔ5ˆ�r%   ©Úoutput_typer?  c                óº   — |€t        d«      ‚| j                  ||||¬«      }|s|d   f|dd z   S t        |j                  |j                  |j
                  ¬«      S )a—  
        Returns:

        Examples:

        ```python
        >>> from transformers import AutoImageProcessor, TFCvtModel
        >>> from PIL import Image
        >>> import requests

        >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
        >>> image = Image.open(requests.get(url, stream=True).raw)

        >>> image_processor = AutoImageProcessor.from_pretrained("microsoft/cvt-13")
        >>> model = TFCvtModel.from_pretrained("microsoft/cvt-13")

        >>> inputs = image_processor(images=image, return_tensors="tf")
        >>> outputs = model(**inputs)
        >>> last_hidden_states = outputs.last_hidden_state
        ```NrE  )rW   r6  r7  r;   r   r   r3  )rG  rN  r   r   r   r   )r-   rW   r6  r7  r;   Úoutputss         r&   r>   zTFCvtModel.callž  sy   € ð> ÐÜÐ?Ó@Ð@à—(‘(Ø%Ø!5Ø#Øð	 ó 
ˆñ Ø˜A‘J�= 7¨1¨2 ;Ñ.Ð.ä,Ø%×7Ñ7Ø#×3Ñ3Ø!×/Ñ/ô
ð 	
r%   c                óú   — | j                   ry d| _         t        | dd «      �Nt        j                  | j                  j
                  «      5  | j                  j                  d «       d d d «       y y # 1 sw Y   y xY w)NTrN  )rZ   r[   r3   r\   rN  rM   r]   r^   s     r&   r]   zTFCvtModel.buildÐ  se   € Ø�:Š:ØØˆŒ
Ü�4˜ Ó%Ð1Ü—‘˜tŸx™xŸ}™}Ó-ñ %Ø—‘—‘˜tÔ$÷%ð %ð 2÷%ð %úra   r<  rK  )
rW   útf.Tensor | Noner6  r=  r7  r=  r;   r=  re   r>  r@   )r   r    r!   r+   r   r   ÚTFCVT_INPUTS_DOCSTRINGr   r   Ú_CONFIG_FOR_DOCr>   r]   rB   rC   s   @r&   rR  rR  ”  s‚   ø„ õ
6ð
 Ù*Ð+AÓBÙÐ+HÐWfÔgð *.Ø/3Ø&*Ø#(ð-
à&ð-
ð -ð-
ð $ð	-
ð
 !ð-
ð 
@ò-
ó hó Có ð-
÷^%r%   rR  z¤
    Cvt Model transformer with an image classification head on top (a linear layer on top of the final hidden state of
    the [CLS] token) e.g. for ImageNet.
    c                  óŒ   ‡ — e Zd Zdˆ fd„Ze ee«       eee	¬«      	 	 	 	 	 d	 	 	 	 	 	 	 	 	 	 	 dd„«       «       «       Z
dd„Zˆ xZS )	ÚTFCvtForImageClassificationc                óX  •— t        ‰| �  |g|¢­i |¤Ž |j                  | _        t        |d¬«      | _        t
        j                  j                  dd¬«      | _        t
        j                  j                  |j                  t        |j                  «      ddd¬	«      | _        || _        y )
NrN  rœ   rr   Ú	layernormrt   Trª   Ú
classifierr¬   )r*   r+   Ú
num_labelsrA  rN  r   rO   r}   r`  r´   r   r|   ra  rR   rT  s       €r&   r+   z$TFCvtForImageClassification.__init__á  s–   ø€ Ü‰Ñ˜Ð3 &Ò3¨FÒ3à ×+Ñ+ˆŒÜ! &¨uÔ5ˆŒäŸ™×8Ñ8ÀÈKÐ8ÓXˆŒô  Ÿ,™,×,Ñ,Ø×#Ñ#Ü.¨v×/GÑ/GÓHØØ$Øð -ó 
ˆŒð ˆ�r%   rU  c                ó  — | j                  ||||¬«      }|d   }|d   }| j                  j                  d   r| j                  |«      }nUt	        |«      \  }	}
}}t        j                  ||	|
||z  f¬«      }t        j                  |d¬«      }| j                  |«      }t        j                  |d¬«      }| j                  |«      }|€d	n| j                  ||¬
«      }|s|f|dd	 z   }|�|f|z   S |S t        |||j                  ¬«      S )a+  
        labels (`tf.Tensor` or `np.ndarray` of shape `(batch_size,)`, *optional*):
            Labels for computing the image classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).

        Returns:

        Examples:

        ```python
        >>> from transformers import AutoImageProcessor, TFCvtForImageClassification
        >>> import tensorflow as tf
        >>> from PIL import Image
        >>> import requests

        >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
        >>> image = Image.open(requests.get(url, stream=True).raw)

        >>> image_processor = AutoImageProcessor.from_pretrained("microsoft/cvt-13")
        >>> model = TFCvtForImageClassification.from_pretrained("microsoft/cvt-13")

        >>> inputs = image_processor(images=image, return_tensors="tf")
        >>> outputs = model(**inputs)
        >>> logits = outputs.logits
        >>> # model predicts one of the 1000 ImageNet classes
        >>> predicted_class_idx = tf.math.argmax(logits, axis=-1)[0]
        >>> print("Predicted class:", model.config.id2label[int(predicted_class_idx)])
        ```rF  r   r   rÉ   r   )r   r¾   r   r¿   rÆ   N)ÚlabelsrÊ   r¾   )ÚlossrÊ   r   )rN  rR   rÎ   r`  r   r3   r�   rÁ   Úreduce_meanra  Úhf_compute_lossr	   r   )r-   rW   rd  r6  r7  r;   rX  rI  rÎ   r‚   rI   rƒ   r„   Úsequence_output_meanrÊ   re  rë   s                    r&   r>   z TFCvtForImageClassification.calló  s(  € ðR —(‘(ØØ!5Ø#Øð	 ó 
ˆð " !™*ˆØ˜A‘Jˆ	Ø�;‰;× Ñ  Ò$Ø"Ÿn™n¨YÓ7‰Oô 7AÀÓ6QÑ3ˆJ˜ f¨eÜ Ÿj™j¨ÀÈ\Ð[aÐdiÑ[iÐ@jÔkˆOÜ Ÿl™l¨?ÀÔKˆOØ"Ÿn™n¨_Ó=ˆOä!Ÿ~™~¨oÀAÔFÐØ—‘Ð!5Ó6ˆØ�~‰t¨4×+?Ñ+?ÀvÐV\Ð+?Ó+]ˆáØ�Y ¨¨ Ñ,ˆFØ)-Ð)9�T�G˜fÑ$ÐE¸vÐEä5¸4ÈÐ^e×^sÑ^sÔtÐtr%   c                ó*  — | j                   ry d| _         t        | dd «      �Mt        j                  | j                  j
                  «      5  | j                  j                  d «       d d d «       t        | dd «      �gt        j                  | j                  j
                  «      5  | j                  j                  d d | j                  j                  d   g«       d d d «       t        | dd «      �t        | j                  d«      rht        j                  | j                  j
                  «      5  | j                  j                  d d | j                  j                  d   g«       d d d «       y y y # 1 sw Y   �ŒxY w# 1 sw Y   Œ£xY w# 1 sw Y   y xY w)NTrN  r`  rÉ   ra  rM   )rZ   r[   r3   r\   rN  rM   r]   r`  rR   rJ   Úhasattrra  r^   s     r&   r]   z!TFCvtForImageClassification.build8  sE  € Ø�:Š:ØØˆŒ
Ü�4˜ Ó%Ð1Ü—‘˜tŸx™xŸ}™}Ó-ñ %Ø—‘—‘˜tÔ$÷%ä�4˜ dÓ+Ð7Ü—‘˜tŸ~™~×2Ñ2Ó3ñ NØ—‘×$Ñ$ d¨D°$·+±+×2GÑ2GÈÑ2KÐ%LÔM÷Nä�4˜ tÓ,Ð8Ü�t—‘¨Ô/Ü—]‘] 4§?¡?×#7Ñ#7Ó8ñ SØ—O‘O×)Ñ)¨4°°t·{±{×7LÑ7LÈRÑ7PÐ*QÔR÷Sð Sð 0ð 9÷%ñ %ú÷Nð Nú÷Sð Sús$   ÁE0Â%6E=Ä/6F	Å0E:Å=FÆ	Fr<  )NNNNF)rW   rZ  rd  rZ  r6  r=  r7  r=  r;   r=  re   z?Union[TFImageClassifierOutputWithNoAttention, Tuple[tf.Tensor]]r@   )r   r    r!   r+   r   r   r[  r   r	   r\  r>   r]   rB   rC   s   @r&   r^  r^  Ù  s    ø„ õð$ Ù*Ð+AÓBÙÐ+QÐ`oÔpð *.Ø#'Ø/3Ø&*Ø#(ð@uà&ð@uð !ð@uð -ð	@uð
 $ð@uð !ð@uð 
Iò@uó qó Có ð@u÷DSr%   r^  )r^  rR  rM  )>r"   Ú
__future__r   Úcollections.abcrx   Údataclassesr   Útypingr   r   r   Ú
tensorflowr3   Úmodeling_tf_outputsr	   Úmodeling_tf_utilsr
   r   r   r   r   r   r   Útf_utilsr   r   Úutilsr   r   r   r   r   Úconfiguration_cvtr   Ú
get_loggerr   Úloggerr\  r   rO   ÚLayerr(   rE   rN   rˆ   r”   r˜   r£   rÙ   rè   rö   rþ   r  r  r*  rA  rM  ÚTFCVT_START_DOCSTRINGr[  rR  r^  Ú__all__r$   r%   r&   ú<module>rz     sm  ðñ å "ã Ý !ß )Ñ )ã å I÷÷ ñ ÷ 3÷õ õ )ð 
ˆ×	Ñ	˜HÓ	%€ð €ð ô7 Kó 7ó ð7ô(/�E—L‘L×&Ñ&ô /ô(%8�e—l‘l×(Ñ(ô %8ôP7G˜%Ÿ,™,×,Ñ,ô 7Gôt"M u§|¡|×'9Ñ'9ô "MôJ¨¯©×);Ñ);ô ô8 5§<¡<×#5Ñ#5ô 8ôDMJ˜Ÿ™×+Ñ+ô MJô`?�e—l‘l×(Ñ(ô ?ô27.�U—\‘\×'Ñ'ô 7.ôt?˜Ÿ™×*Ñ*ô ?ô4U�%—,‘,×$Ñ$ô Uô:^I�—‘×#Ñ#ô ^IôB]&�—‘×#Ñ#ô ]&ô@:&�5—<‘<×%Ñ%ô :&ðz ô-)�U—\‘\×'Ñ'ó -)ó ð-)ô`%Ð,ô %ðÐ ð8Ð ñ& ØcØóô>%Ð%ó >%ó	ð>%ñB ðð óôeSÐ"6Ð8Tó eSóðeSòP P�r%   