Ë
    T^(h€s  ã                   ó‚  — d Z ddlZddlmZ ddlmZmZmZ ddlZddl	Zddlm
Z
 ddlmZmZmZ ddlmZmZmZmZ dd	lmZ dd
lmZmZmZmZ ddlmZ  ej:                  e«      ZdZ dZ!g d¢Z"dZ#dZ$e G d„ de«      «       Z% G d„ de
jL                  «      Z' G d„ de
jL                  «      Z( G d„ de
jL                  «      Z) G d„ de
jL                  «      Z* G d„ de
jL                  «      Z+ G d„ de
jL                  «      Z, G d„ d e
jL                  «      Z- G d!„ d"e
jL                  «      Z. G d#„ d$e
jL                  «      Z/ G d%„ d&e
jL                  «      Z0 G d'„ d(e
jL                  «      Z1 G d)„ d*e«      Z2d+Z3d,Z4 ed-e3«       G d.„ d/e2«      «       Z5 ed0e3«       G d1„ d2e2«      «       Z6 ed3e3«       G d4„ d5e2«      «       Z7g d6¢Z8y)7zPyTorch LeViT model.é    N)Ú	dataclass)ÚOptionalÚTupleÚUnion)Únn)ÚBCEWithLogitsLossÚCrossEntropyLossÚMSELossé   )ÚBaseModelOutputWithNoAttentionÚ(BaseModelOutputWithPoolingAndNoAttentionÚ$ImageClassifierOutputWithNoAttentionÚModelOutput)ÚPreTrainedModel)Úadd_code_sample_docstringsÚadd_start_docstringsÚ%add_start_docstrings_to_model_forwardÚloggingé   )ÚLevitConfigr   zfacebook/levit-128S)r   é   i€  ztabby, tabby catc                   ó¸   — e Zd ZU dZdZeej                     ed<   dZ	eej                     ed<   dZ
eej                     ed<   dZeeej                        ed<   y)Ú,LevitForImageClassificationWithTeacherOutputa•  
    Output type of [`LevitForImageClassificationWithTeacher`].

    Args:
        logits (`torch.FloatTensor` of shape `(batch_size, config.num_labels)`):
            Prediction scores as the average of the `cls_logits` and `distillation_logits`.
        cls_logits (`torch.FloatTensor` of shape `(batch_size, config.num_labels)`):
            Prediction scores of the classification head (i.e. the linear layer on top of the final hidden state of the
            class token).
        distillation_logits (`torch.FloatTensor` of shape `(batch_size, config.num_labels)`):
            Prediction scores of the distillation head (i.e. the linear layer on top of the final hidden state of the
            distillation token).
        hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
            Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
            shape `(batch_size, sequence_length, hidden_size)`. Hidden-states of the model at the output of each layer
            plus the initial embedding outputs.
    NÚlogitsÚ
cls_logitsÚdistillation_logitsÚhidden_states)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r   ÚtorchÚFloatTensorÚ__annotations__r   r   r   r   © ó    úf/var/www/skyplay_api_hub/venv/lib/python3.12/site-packages/transformers/models/levit/modeling_levit.pyr   r   3   sc   … ñð$ +/€FˆH�U×&Ñ&Ñ'Ó.Ø.2€J�˜×*Ñ*Ñ+Ó2Ø7;Ð˜ %×"3Ñ"3Ñ4Ó;Ø8<€M�8˜E %×"3Ñ"3Ñ4Ñ5Ô<r&   r   c                   ó,   ‡ — e Zd ZdZ	 dˆ fd„	Zd„ Zˆ xZS )ÚLevitConvEmbeddingsz[
    LeViT Conv Embeddings with Batch Norm, used in the initial patch embedding layer.
    c	           
      óš   •— t         ‰	| �  «        t        j                  |||||||d¬«      | _        t        j
                  |«      | _        y )NF)ÚdilationÚgroupsÚbias)ÚsuperÚ__init__r   ÚConv2dÚconvolutionÚBatchNorm2dÚ
batch_norm)
ÚselfÚin_channelsÚout_channelsÚkernel_sizeÚstrideÚpaddingr+   r,   Úbn_weight_initÚ	__class__s
            €r'   r/   zLevitConvEmbeddings.__init__R   sF   ø€ ô 	‰ÑÔÜŸ9™9Ø˜ {°F¸GÈhÐ_eÐlqô
ˆÔô Ÿ.™.¨Ó6ˆ�r&   c                 óJ   — | j                  |«      }| j                  |«      }|S ©N)r1   r3   )r4   Ú
embeddingss     r'   ÚforwardzLevitConvEmbeddings.forward[   s&   € Ø×%Ñ% jÓ1ˆ
Ø—_‘_ ZÓ0ˆ
ØÐr&   )r   r   r   ©r   r   r    r!   r/   r?   Ú__classcell__©r;   s   @r'   r)   r)   M   s   ø„ ñð
 mnõ7ör&   r)   c                   ó(   ‡ — e Zd ZdZˆ fd„Zd„ Zˆ xZS )ÚLevitPatchEmbeddingsz�
    LeViT patch embeddings, for final embeddings to be passed to transformer blocks. It consists of multiple
    `LevitConvEmbeddings`.
    c                 óX  •— t         ‰| �  «        t        |j                  |j                  d   dz  |j
                  |j                  |j                  «      | _        t        j                  «       | _        t        |j                  d   dz  |j                  d   dz  |j
                  |j                  |j                  «      | _        t        j                  «       | _        t        |j                  d   dz  |j                  d   dz  |j
                  |j                  |j                  «      | _        t        j                  «       | _        t        |j                  d   dz  |j                  d   |j
                  |j                  |j                  «      | _        |j                  | _        y )Nr   é   é   é   )r.   r/   r)   Únum_channelsÚhidden_sizesr7   r8   r9   Úembedding_layer_1r   Ú	HardswishÚactivation_layer_1Úembedding_layer_2Úactivation_layer_2Úembedding_layer_3Úactivation_layer_3Úembedding_layer_4©r4   Úconfigr;   s     €r'   r/   zLevitPatchEmbeddings.__init__g   so  ø€ Ü‰ÑÔÜ!4Ø×Ñ ×!4Ñ!4°QÑ!7¸1Ñ!<¸f×>PÑ>PÐRX×R_ÑR_Ðag×aoÑaoó"
ˆÔô #%§,¡,£.ˆÔä!4Ø×Ñ Ñ" aÑ'¨×)<Ñ)<¸QÑ)?À1Ñ)DÀf×FXÑFXÐZ`×ZgÑZgÐio×iwÑiwó"
ˆÔô #%§,¡,£.ˆÔä!4Ø×Ñ Ñ" aÑ'¨×)<Ñ)<¸QÑ)?À1Ñ)DÀf×FXÑFXÐZ`×ZgÑZgÐio×iwÑiwó"
ˆÔô #%§,¡,£.ˆÔä!4Ø×Ñ Ñ" aÑ'¨×)<Ñ)<¸QÑ)?À×ASÑASÐU[×UbÑUbÐdj×drÑdró"
ˆÔð #×/Ñ/ˆÕr&   c                 ó„  — |j                   d   }|| j                  k7  rt        d«      ‚| j                  |«      }| j	                  |«      }| j                  |«      }| j                  |«      }| j                  |«      }| j                  |«      }| j                  |«      }|j                  d«      j                  dd«      S )Nr   zeMake sure that the channel dimension of the pixel values match with the one set in the configuration.rH   )ÚshaperI   Ú
ValueErrorrK   rM   rN   rO   rP   rQ   rR   ÚflattenÚ	transpose)r4   Úpixel_valuesrI   r>   s       r'   r?   zLevitPatchEmbeddings.forward}   sÀ   € Ø#×)Ñ)¨!Ñ,ˆØ˜4×,Ñ,Ò,ÜØwóð ð ×+Ñ+¨LÓ9ˆ
Ø×,Ñ,¨ZÓ8ˆ
Ø×+Ñ+¨JÓ7ˆ
Ø×,Ñ,¨ZÓ8ˆ
Ø×+Ñ+¨JÓ7ˆ
Ø×,Ñ,¨ZÓ8ˆ
Ø×+Ñ+¨JÓ7ˆ
Ø×!Ñ! !Ó$×.Ñ.¨q°!Ó4Ð4r&   r@   rB   s   @r'   rD   rD   a   s   ø„ ñô
0ö,5r&   rD   c                   ó&   ‡ — e Zd Zdˆ fd„	Zd„ Zˆ xZS )ÚMLPLayerWithBNc                 ó�   •— t         ‰| �  «        t        j                  ||d¬«      | _        t        j
                  |«      | _        y )NF)Úin_featuresÚout_featuresr-   )r.   r/   r   ÚLinearÚlinearÚBatchNorm1dr3   )r4   Ú	input_dimÚ
output_dimr:   r;   s       €r'   r/   zMLPLayerWithBN.__init__Ž   s3   ø€ Ü‰ÑÔÜ—i‘i¨IÀJÐUZÔ[ˆŒÜŸ.™.¨Ó4ˆ�r&   c                 óˆ   — | j                  |«      }| j                  |j                  dd«      «      j                  |«      }|S )Nr   r   )ra   r3   rX   Ú
reshape_as©r4   Úhidden_states     r'   r?   zMLPLayerWithBN.forward“   s<   € Ø—{‘{ <Ó0ˆØ—‘ |×';Ñ';¸A¸qÓ'AÓB×MÑMÈlÓ[ˆØÐr&   )r   ©r   r   r    r/   r?   rA   rB   s   @r'   r\   r\   �   s   ø„ õ5ö
r&   r\   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚLevitSubsamplec                 ó>   •— t         ‰| �  «        || _        || _        y r=   )r.   r/   r8   Ú
resolution)r4   r8   rm   r;   s      €r'   r/   zLevitSubsample.__init__š   s   ø€ Ü‰ÑÔØˆŒØ$ˆ�r&   c                 óÜ   — |j                   \  }}}|j                  || j                  | j                  |«      d d …d d | j                  …d d | j                  …f   j	                  |d|«      }|S )Néÿÿÿÿ)rV   Úviewrm   r8   Úreshape)r4   rh   Ú
batch_sizeÚ_Úchannelss        r'   r?   zLevitSubsample.forwardŸ   sk   € Ø".×"4Ñ"4Ñˆ
�A�xØ#×(Ñ(¨°T·_±_ÀdÇoÁoÐW_Ó`Ú‰~�$—+‘+ˆ~™~ $§+¡+˜~Ð-ñ
ç
‰'�*˜b (Ó
+ð 	ð Ðr&   ri   rB   s   @r'   rk   rk   ™   s   ø„ ô%ö
r&   rk   c                   ó^   ‡ — e Zd Zˆ fd„Z ej
                  «       dˆ fd„	«       Zd„ Zd„ Zˆ xZ	S )ÚLevitAttentionc                 ó~  •— t         ‰| �  «        || _        |dz  | _        || _        || _        ||z  |z  ||z  dz  z   | _        ||z  |z  | _        t        || j                  «      | _	        t        j                  «       | _        t        | j                  |d¬«      | _        t        t        j                   t#        |«      t#        |«      «      «      }t%        |«      }i g }	}|D ]W  }
|D ]P  }t'        |
d   |d   z
  «      t'        |
d   |d   z
  «      f}||vrt%        |«      ||<   |	j)                  ||   «       ŒR ŒY i | _        t,        j                  j/                  t-        j0                  |t%        |«      «      «      | _        | j5                  dt-        j6                  |	«      j9                  ||«      d¬«       y )	Nç      à¿rH   r   )r:   r   Úattention_bias_idxsF©Ú
persistent)r.   r/   Únum_attention_headsÚscaleÚkey_dimÚattention_ratioÚout_dim_keys_valuesÚout_dim_projectionr\   Úqueries_keys_valuesr   rL   Ú
activationÚ
projectionÚlistÚ	itertoolsÚproductÚrangeÚlenÚabsÚappendÚattention_bias_cacher"   Ú	ParameterÚzerosÚattention_biasesÚregister_bufferÚ
LongTensorrp   )r4   rJ   r~   r|   r   rm   ÚpointsÚ
len_pointsÚattention_offsetsÚindicesÚp1Úp2Úoffsetr;   s                €r'   r/   zLevitAttention.__init__¨   s²  ø€ Ü‰ÑÔØ#6ˆÔ Ø˜d‘]ˆŒ
ØˆŒØ.ˆÔØ#2°WÑ#<Ð?RÑ#RÐU\Ð_rÑUrÐuvÑUvÑ#vˆÔ Ø"1°GÑ";Ð>QÑ"QˆÔä#1°,À×@XÑ@XÓ#YˆÔ ÜŸ,™,›.ˆŒÜ(¨×)@Ñ)@À,Ð_`ÔaˆŒä”i×'Ñ'¬¨jÓ(9¼5ÀÓ;LÓMÓNˆÜ˜“[ˆ
Ø%'¨˜7ÐØò 	:ˆBØò :�Ü˜b ™e b¨¡e™mÓ,¬c°"°Q±%¸"¸Q¹%±-Ó.@ÐA�ØÐ!2Ñ2Ü03Ð4EÓ0FÐ% fÑ-Ø—‘Ð0°Ñ8Õ9ñ	:ð	:ð %'ˆÔ!Ü %§¡× 2Ñ 2´5·;±;Ð?RÔTWÐXiÓTjÓ3kÓ lˆÔØ×ÑØ!¤5×#3Ñ#3°GÓ#<×#AÑ#AÀ*ÈjÓ#YÐfkð 	õ 	
r&   c                 óR   •— t         ‰| �  |«       |r| j                  ri | _        y y y r=   ©r.   ÚtrainrŒ   ©r4   Úmoder;   s     €r'   r›   zLevitAttention.trainÅ   ó)   ø€ ä‰‰�dÔÙ�D×-Ò-Ø(*ˆDÕ%ð .ˆ4r&   c                 óø   — | j                   r| j                  d d …| j                  f   S t        |«      }|| j                  vr*| j                  d d …| j                  f   | j                  |<   | j                  |   S r=   ©Útrainingr�   ry   ÚstrrŒ   ©r4   ÚdeviceÚ
device_keys      r'   Úget_attention_biasesz#LevitAttention.get_attention_biasesË   ót   € Ø�=Š=Ø×(Ñ(ª¨D×,DÑ,DÐ)DÑEÐEä˜V›ˆJØ ×!:Ñ!:Ñ:Ø8<×8MÑ8MÊaÐQU×QiÑQiÐNiÑ8j�×)Ñ)¨*Ñ5Ø×,Ñ,¨ZÑ8Ð8r&   c                 óÂ  — |j                   \  }}}| j                  |«      }|j                  ||| j                  d«      j	                  | j
                  | j
                  | j                  | j
                  z  gd¬«      \  }}}|j                  dddd«      }|j                  dddd«      }|j                  dddd«      }||j                  dd«      z  | j                  z  | j                  |j                  «      z   }	|	j                  d¬«      }	|	|z  j                  dd«      j                  ||| j                  «      }| j                  | j!                  |«      «      }|S ©Nro   r   ©Údimr   rH   r   éþÿÿÿ)rV   r‚   rp   r|   Úsplitr~   r   ÚpermuterY   r}   r¦   r¤   Úsoftmaxrq   r�   r„   rƒ   )
r4   rh   rr   Ú
seq_lengthrs   r‚   ÚqueryÚkeyÚvalueÚ	attentions
             r'   r?   zLevitAttention.forwardÔ   sN  € Ø$0×$6Ñ$6Ñ!ˆ
�J Ø"×6Ñ6°|ÓDÐØ/×4Ñ4°ZÀÈT×MeÑMeÐgiÓj×pÑpØ�\‰\˜4Ÿ<™<¨×)=Ñ)=ÀÇÁÑ)LÐMÐSTð qó 
Ñˆˆs�Eð —‘˜a  A qÓ)ˆØ�k‰k˜!˜Q  1Ó%ˆØ—‘˜a  A qÓ)ˆà˜CŸM™M¨"¨bÓ1Ñ1°D·J±JÑ>À×AZÑAZÐ[g×[nÑ[nÓAoÑoˆ	Ø×%Ñ%¨"Ð%Ó-ˆ	Ø! EÑ)×4Ñ4°Q¸Ó:×BÑBÀ:ÈzÐ[_×[rÑ[rÓsˆØ—‘ t§¡°|Ó'DÓEˆØÐr&   ©T©
r   r   r    r/   r"   Úno_gradr›   r¦   r?   rA   rB   s   @r'   rv   rv   §   s.   ø„ ô
ð: €U‡]�]ƒ_ô+ó ð+ò
9ör&   rv   c                   ó^   ‡ — e Zd Zˆ fd„Z ej
                  «       dˆ fd„	«       Zd„ Zd„ Zˆ xZ	S )ÚLevitAttentionSubsamplec	                 óx  •— t         ‰| �  «        || _        |dz  | _        || _        || _        ||z  |z  ||z  z   | _        ||z  |z  | _        || _        t        || j                  «      | _
        t        ||«      | _        t        |||z  «      | _        t        j                  «       | _        t        | j                  |«      | _        i | _        t'        t)        j*                  t-        |«      t-        |«      «      «      }	t'        t)        j*                  t-        |«      t-        |«      «      «      }
t/        |	«      t/        |
«      }}i g }}|
D ]q  }|	D ]j  }d}t1        |d   |z  |d   z
  |dz
  dz  z   «      t1        |d   |z  |d   z
  |dz
  dz  z   «      f}||vrt/        |«      ||<   |j3                  ||   «       Œl Œs t4        j                  j7                  t5        j8                  |t/        |«      «      «      | _        | j=                  dt5        j>                  |«      jA                  ||«      d¬«       y )Nrx   r   r   rH   ry   Frz   )!r.   r/   r|   r}   r~   r   r€   r�   Úresolution_outr\   Úkeys_valuesrk   Úqueries_subsampleÚqueriesr   rL   rƒ   r„   rŒ   r…   r†   r‡   rˆ   r‰   rŠ   r‹   r"   r�   rŽ   r�   r�   r‘   rp   )r4   rc   rd   r~   r|   r   r8   Úresolution_inr»   r’   Úpoints_r“   Úlen_points_r”   r•   r–   r—   Úsizer˜   r;   s                      €r'   r/   z LevitAttentionSubsample.__init__æ   s1  ø€ ô 	‰ÑÔØ#6ˆÔ Ø˜d‘]ˆŒ
ØˆŒØ.ˆÔØ#2°WÑ#<Ð?RÑ#RÐU\Ð_rÑUrÑ#rˆÔ Ø"1°GÑ";Ð>QÑ"QˆÔØ,ˆÔä)¨)°T×5MÑ5MÓNˆÔÜ!/°¸Ó!FˆÔÜ% i°Ð;NÑ1NÓOˆŒÜŸ,™,›.ˆŒÜ(¨×)@Ñ)@À*ÓMˆŒà$&ˆÔ!ä”i×'Ñ'¬¨mÓ(<¼eÀMÓ>RÓSÓTˆÜ”y×(Ñ(¬¨~Ó)>ÄÀnÓ@UÓVÓWˆÜ"% f£+¬s°7«|�Kˆ
Ø%'¨˜7ÐØò 	:ˆBØò :�Ø�Ü˜b ™e f™n¨r°!©uÑ4¸¸q¹ÀA±~ÑEÓFÌÈBÈqÉEÐTZÉNÐ]_Ð`aÑ]bÑLbÐfjÐmnÑfnÐrsÑesÑLsÓHtÐu�ØÐ!2Ñ2Ü03Ð4EÓ0FÐ% fÑ-Ø—‘Ð0°Ñ8Õ9ñ:ð	:ô !&§¡× 2Ñ 2´5·;±;Ð?RÔTWÐXiÓTjÓ3kÓ lˆÔØ×ÑØ!¤5×#3Ñ#3°GÓ#<×#AÑ#AÀ+ÈzÓ#ZÐglð 	õ 	
r&   c                 óR   •— t         ‰| �  |«       |r| j                  ri | _        y y y r=   rš   rœ   s     €r'   r›   zLevitAttentionSubsample.train  rž   r&   c                 óø   — | j                   r| j                  d d …| j                  f   S t        |«      }|| j                  vr*| j                  d d …| j                  f   | j                  |<   | j                  |   S r=   r    r£   s      r'   r¦   z,LevitAttentionSubsample.get_attention_biases  r§   r&   c                 óL  — |j                   \  }}}| j                  |«      j                  ||| j                  d«      j	                  | j
                  | j                  | j
                  z  gd¬«      \  }}|j                  dddd«      }|j                  dddd«      }| j                  | j                  |«      «      }|j                  || j                  dz  | j                  | j
                  «      j                  dddd«      }||j                  dd«      z  | j                  z  | j                  |j                  «      z   }|j                  d¬«      }||z  j                  dd«      j!                  |d| j"                  «      }| j%                  | j'                  |«      «      }|S r©   )rV   r¼   rp   r|   r­   r~   r   r®   r¾   r½   r»   rY   r}   r¦   r¤   r¯   rq   r�   r„   rƒ   )	r4   rh   rr   r°   rs   r²   r³   r±   r´   s	            r'   r?   zLevitAttentionSubsample.forward"  s~  € Ø$0×$6Ñ$6Ñ!ˆ
�J à×Ñ˜\Ó*ß‰T�*˜j¨$×*BÑ*BÀBÓGß‰U�D—L‘L $×"6Ñ"6¸¿¹Ñ"EÐFÈAˆUÓNñ 	ˆˆUð
 �k‰k˜!˜Q  1Ó%ˆØ—‘˜a  A qÓ)ˆà—‘˜T×3Ñ3°LÓAÓBˆØ—
‘
˜: t×':Ñ':¸AÑ'=¸t×?WÑ?WÐY]×YeÑYeÓf×nÑnØˆq�!�Qó
ˆð ˜CŸM™M¨"¨bÓ1Ñ1°D·J±JÑ>À×AZÑAZÐ[g×[nÑ[nÓAoÑoˆ	Ø×%Ñ%¨"Ð%Ó-ˆ	Ø! EÑ)×4Ñ4°Q¸Ó:×BÑBÀ:ÈrÐSW×SjÑSjÓkˆØ—‘ t§¡°|Ó'DÓEˆØÐr&   rµ   r¶   rB   s   @r'   r¹   r¹   å   s/   ø„ ô+
ðZ €U‡]�]ƒ_ô+ó ð+ò
9ör&   r¹   c                   ó(   ‡ — e Zd ZdZˆ fd„Zd„ Zˆ xZS )ÚLevitMLPLayerzE
    MLP Layer with `2X` expansion in contrast to ViT with `4X`.
    c                 ó˜   •— t         ‰| �  «        t        ||«      | _        t	        j
                  «       | _        t        ||«      | _        y r=   )r.   r/   r\   Ú	linear_upr   rL   rƒ   Úlinear_down)r4   rc   Ú
hidden_dimr;   s      €r'   r/   zLevitMLPLayer.__init__=  s8   ø€ Ü‰ÑÔÜ'¨	°:Ó>ˆŒÜŸ,™,›.ˆŒÜ)¨*°iÓ@ˆÕr&   c                 ól   — | j                  |«      }| j                  |«      }| j                  |«      }|S r=   )rÉ   rƒ   rÊ   rg   s     r'   r?   zLevitMLPLayer.forwardC  s4   € Ø—~‘~ lÓ3ˆØ—‘ |Ó4ˆØ×'Ñ'¨Ó5ˆØÐr&   r@   rB   s   @r'   rÇ   rÇ   8  s   ø„ ñôAör&   rÇ   c                   ó(   ‡ — e Zd ZdZˆ fd„Zd„ Zˆ xZS )ÚLevitResidualLayerz"
    Residual Block for LeViT
    c                 ó>   •— t         ‰| �  «        || _        || _        y r=   )r.   r/   ÚmoduleÚ	drop_rate)r4   rÐ   rÑ   r;   s      €r'   r/   zLevitResidualLayer.__init__O  s   ø€ Ü‰ÑÔØˆŒØ"ˆ�r&   c                 ó„  — | j                   rŸ| j                  dkD  r�t        j                  |j	                  d«      dd|j
                  ¬«      }|j                  | j                  «      j                  d| j                  z
  «      j                  «       }|| j                  |«      |z  z   }|S || j                  |«      z   }|S )Nr   r   )r¤   )
r¡   rÑ   r"   ÚrandrÂ   r¤   Úge_ÚdivÚdetachrÐ   )r4   rh   Úrnds      r'   r?   zLevitResidualLayer.forwardT  sŸ   € Ø�=Š=˜TŸ^™^¨aÒ/Ü—*‘*˜\×.Ñ.¨qÓ1°1°aÀ×@SÑ@SÔTˆCØ—'‘'˜$Ÿ.™.Ó)×-Ñ-¨a°$·.±.Ñ.@ÓA×HÑHÓJˆCØ'¨$¯+©+°lÓ*CÀcÑ*IÑIˆLØÐà'¨$¯+©+°lÓ*CÑCˆLØÐr&   r@   rB   s   @r'   rÎ   rÎ   J  s   ø„ ñô#ö
 r&   rÎ   c                   ó.   ‡ — e Zd ZdZˆ fd„Zd„ Zd„ Zˆ xZS )Ú
LevitStagezP
    LeViT Stage consisting of `LevitMLPLayer` and `LevitAttention` layers.
    c                 ó
  •— t         ‰| �  «        g | _        || _        |
| _        t        |«      D ]–  }| j                  j                  t        t        |||||
«      | j                  j                  «      «       |dkD  sŒO||z  }| j                  j                  t        t        ||«      | j                  j                  «      «       Œ˜ |	d   dk(  �r| j                  dz
  |	d   z  dz   | _        | j                  j                  t        | j                  j                  ||dz    |	d   |	d   |	d   |	d   |
| j                  dœŽ«       | j                  | _        |	d   dkD  r| j                  j                  |dz      |	d   z  }| j                  j                  t        t        | j                  j                  |dz      |«      | j                  j                  «      «       t        j                  | j                  «      | _        y )	Nr   Ú	Subsampler   é   rH   r   )r~   r|   r   r8   r¿   r»   rG   )r.   r/   ÚlayersrT   r¿   rˆ   r‹   rÎ   rv   Údrop_path_raterÇ   r»   r¹   rJ   r   Ú
ModuleList)r4   rT   ÚidxrJ   r~   Údepthsr|   r   Ú	mlp_ratioÚdown_opsr¿   rs   rË   r;   s                €r'   r/   zLevitStage.__init__d  sÖ  ø€ ô 	‰ÑÔØˆŒØˆŒØ*ˆÔä�v“ò 	ˆAØ�K‰K×ÑÜ"Ü" <°Ð:MÈÐ`mÓnØ—K‘K×.Ñ.óôð ˜1‹}Ø)¨IÑ5�
Ø—‘×"Ñ"Ü&¤}°\À:Ó'NÐPT×P[ÑP[×PjÑPjÓkõð	ð �A‰;˜+Ó%Ø#'×#5Ñ#5¸Ñ#9¸hÀq¹kÑ"IÈAÑ"MˆDÔØ�K‰K×ÑÜ'Ø—[‘[×-Ñ-¨c°C¸!±GÐ<Ø$ Q™KØ(0°©Ø$,¨Q¡KØ# A™;Ø"/Ø#'×#6Ñ#6òô
ð "&×!4Ñ!4ˆDÔØ˜‰{˜QŠØ!Ÿ[™[×5Ñ5°c¸A±gÑ>ÀÈ!ÁÑL�
Ø—‘×"Ñ"Ü&Ü% d§k¡k×&>Ñ&>¸sÀQ¹wÑ&GÈÓTÐVZ×VaÑVa×VpÑVpóôô —m‘m D§K¡KÓ0ˆ�r&   c                 ó   — | j                   S r=   )r¿   )r4   s    r'   Úget_resolutionzLevitStage.get_resolution›  s   € Ø×!Ñ!Ð!r&   c                 ó8   — | j                   D ]
  } ||«      }Œ |S r=   )rÝ   )r4   rh   Úlayers      r'   r?   zLevitStage.forwardž  s%   € Ø—[‘[ò 	/ˆEÙ  Ó.‰Lð	/àÐr&   )r   r   r    r!   r/   rå   r?   rA   rB   s   @r'   rÙ   rÙ   _  s   ø„ ñô51òn"ör&   rÙ   c                   ó*   ‡ — e Zd ZdZˆ fd„Zdd„Zˆ xZS )ÚLevitEncoderzC
    LeViT Encoder consisting of multiple `LevitStage` stages.
    c                 ó¦  •— t         ‰| �  «        || _        | j                  j                  | j                  j                  z  }g | _        | j                  j                  j                  dg«       t        t        |j                  «      «      D ]œ  }t        |||j                  |   |j                  |   |j                  |   |j                  |   |j                  |   |j                   |   |j                  |   |«
      }|j#                  «       }| j
                  j                  |«       Œž t%        j&                  | j
                  «      | _        y )NÚ )r.   r/   rT   Ú
image_sizeÚ
patch_sizeÚstagesrã   r‹   rˆ   r‰   rá   rÙ   rJ   r~   r|   r   râ   rå   r   rß   )r4   rT   rm   Ú	stage_idxÚstager;   s        €r'   r/   zLevitEncoder.__init__©  s  ø€ Ü‰ÑÔØˆŒØ—[‘[×+Ñ+¨t¯{©{×/EÑ/EÑEˆ
ØˆŒØ�‰×Ñ×#Ñ# R DÔ)äœs 6§=¡=Ó1Ó2ò 	&ˆIÜØØØ×#Ñ# IÑ.Ø—‘˜yÑ)Ø—‘˜iÑ(Ø×*Ñ*¨9Ñ5Ø×&Ñ& yÑ1Ø× Ñ  Ñ+Ø—‘ 	Ñ*ØóˆEð ×-Ñ-Ó/ˆJØ�K‰K×Ñ˜uÕ%ð	&ô  —m‘m D§K¡KÓ0ˆ�r&   c                 ó¦   — |rdnd }| j                   D ]  }|r||fz   } ||«      }Œ |r||fz   }|st        d„ ||fD «       «      S t        ||¬«      S )Nr%   c              3   ó&   K  — | ]	  }|€Œ|–— Œ y ­wr=   r%   )Ú.0Úvs     r'   ú	<genexpr>z'LevitEncoder.forward.<locals>.<genexpr>Í  s   è ø€ ÒW˜qÈÉœÑWùs   ‚Š)Úlast_hidden_stater   )rî   Útupler   )r4   rh   Úoutput_hidden_statesÚreturn_dictÚall_hidden_statesrð   s         r'   r?   zLevitEncoder.forwardÂ  ss   € Ù"6™B¸DÐà—[‘[ò 	/ˆEÙ#Ø$5¸¸Ñ$GÐ!Ù  Ó.‰Lð	/ñ
  Ø 1°\°OÑ CÐÙÜÑW \Ð3DÐ$EÔWÓWÐWä-ÀÐ\mÔnÐnr&   )FTr@   rB   s   @r'   ré   ré   ¤  s   ø„ ñô1÷2or&   ré   c                   ó(   ‡ — e Zd ZdZˆ fd„Zd„ Zˆ xZS )ÚLevitClassificationLayerz$
    LeViT Classification Layer
    c                 óŒ   •— t         ‰| �  «        t        j                  |«      | _        t        j
                  ||«      | _        y r=   )r.   r/   r   rb   r3   r`   ra   )r4   rc   rd   r;   s      €r'   r/   z!LevitClassificationLayer.__init__×  s0   ø€ Ü‰ÑÔÜŸ.™.¨Ó3ˆŒÜ—i‘i 	¨:Ó6ˆ�r&   c                 óJ   — | j                  |«      }| j                  |«      }|S r=   )r3   ra   )r4   rh   r   s      r'   r?   z LevitClassificationLayer.forwardÜ  s#   € Ø—‘ |Ó4ˆØ—‘˜\Ó*ˆØˆr&   r@   rB   s   @r'   rü   rü   Ò  s   ø„ ñô7ö
r&   rü   c                   ó(   — e Zd ZdZeZdZdZdgZd„ Z	y)ÚLevitPreTrainedModelz†
    An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained
    models.
    ÚlevitrZ   rÎ   c                 ó  — t        |t        j                  t        j                  f«      rm|j                  j
                  j                  d| j                  j                  ¬«       |j                  �%|j                  j
                  j                  «        yyt        |t        j                  t        j                  f«      rJ|j                  j
                  j                  «        |j                  j
                  j                  d«       yy)zInitialize the weightsg        )ÚmeanÚstdNg      ð?)Ú
isinstancer   r`   r0   ÚweightÚdataÚnormal_rT   Úinitializer_ranger-   Úzero_rb   r2   Úfill_)r4   rÐ   s     r'   Ú_init_weightsz"LevitPreTrainedModel._init_weightsí  s²   € ä�fœrŸy™y¬"¯)©)Ð4Ô5ð �M‰M×Ñ×&Ñ&¨C°T·[±[×5RÑ5RÐ&ÔSØ�{‰{Ð&Ø—‘× Ñ ×&Ñ&Õ(ð 'ä˜¤§¡´·±Ð @ÔAØ�K‰K×Ñ×"Ñ"Ô$Ø�M‰M×Ñ×$Ñ$ SÕ)ð Br&   N)
r   r   r    r!   r   Úconfig_classÚbase_model_prefixÚmain_input_nameÚ_no_split_modulesr  r%   r&   r'   r   r   â  s'   „ ñð
 €LØÐØ$€OØ-Ð.Ðó
*r&   r   aG  
    This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. Use it
    as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and
    behavior.

    Parameters:
        config ([`LevitConfig`]): Model configuration class with all the parameters of the model.
            Initializing with a config file does not load the weights associated with the model, only the
            configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
aC  
    Args:
        pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`):
            Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See
            [`LevitImageProcessor.__call__`] for details.

        output_hidden_states (`bool`, *optional*):
            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
            more detail.
        return_dict (`bool`, *optional*):
            Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
zNThe bare Levit model outputting raw features without any specific head on top.c                   ó¤   ‡ — e Zd Zˆ fd„Z ee«       eeee	de
¬«      	 	 	 d	deej                     dee   dee   deeef   fd„«       «       Zˆ xZS )
Ú
LevitModelc                 ó’   •— t         ‰| �  |«       || _        t        |«      | _        t        |«      | _        | j                  «        y r=   )r.   r/   rT   rD   Úpatch_embeddingsré   ÚencoderÚ	post_initrS   s     €r'   r/   zLevitModel.__init__  s:   ø€ Ü‰Ñ˜Ô ØˆŒÜ 4°VÓ <ˆÔÜ# FÓ+ˆŒà�‰Õr&   Úvision)Ú
checkpointÚoutput_typer  ÚmodalityÚexpected_outputrZ   rø   rù   Úreturnc                 óD  — |�|n| j                   j                  }|�|n| j                   j                  }|€t        d«      ‚| j	                  |«      }| j                  |||¬«      }|d   }|j                  d¬«      }|s
||f|dd  z   S t        |||j                  ¬«      S )Nz You have to specify pixel_values©rø   rù   r   r   rª   )rö   Úpooler_outputr   )	rT   rø   Úuse_return_dictrW   r  r  r  r   r   )r4   rZ   rø   rù   r>   Úencoder_outputsrö   Úpooled_outputs           r'   r?   zLevitModel.forward   sÌ   € ð %9Ð$DÑ È$Ï+É+×JjÑJjð 	ð &1Ð%<‘kÀ$Ç+Á+×B]ÑB]ˆàÐÜÐ?Ó@Ð@à×*Ñ*¨<Ó8ˆ
ØŸ,™,ØØ!5Ø#ð 'ó 
ˆð ,¨AÑ.Ðð *×.Ñ.°1Ð.Ó5ˆáØ% }Ð5¸ÈÈÐ8KÑKÐKä7Ø/Ø'Ø)×7Ñ7ô
ð 	
r&   ©NNN)r   r   r    r/   r   ÚLEVIT_INPUTS_DOCSTRINGr   Ú_CHECKPOINT_FOR_DOCr   Ú_CONFIG_FOR_DOCÚ_EXPECTED_OUTPUT_SHAPEr   r"   r#   Úboolr   r   r?   rA   rB   s   @r'   r  r    sŽ   ø„ ô
ñ +Ð+AÓBÙØ&Ø<Ø$ØØ.ôð 59Ø/3Ø&*ñ	!
à˜u×0Ñ0Ñ1ð!
ð ' t™nð!
ð ˜d‘^ð	!
ð
 
ˆuÐ>Ð>Ñ	?ò!
óó Cô!
r&   r  z…
    Levit Model with an image classification head on top (a linear layer on top of the pooled features), e.g. for
    ImageNet.
    c                   óÂ   ‡ — e Zd Zˆ fd„Z ee«       eeee	e
¬«      	 	 	 	 d	deej                     deej                     dee   dee   deeef   f
d„«       «       Zˆ xZS )
ÚLevitForImageClassificationc                 ó>  •— t         ‰| �  |«       || _        |j                  | _        t	        |«      | _        |j                  dkD  r#t        |j                  d   |j                  «      nt        j                  j                  «       | _        | j                  «        y ©Nr   ro   )r.   r/   rT   Ú
num_labelsr  r  rü   rJ   r"   r   ÚIdentityÚ
classifierr  rS   s     €r'   r/   z$LevitForImageClassification.__init__T  s€   ø€ Ü‰Ñ˜Ô ØˆŒØ ×+Ñ+ˆŒÜ Ó'ˆŒ
ð
 × Ñ  1Ò$ô % V×%8Ñ%8¸Ñ%<¸f×>OÑ>OÔPä—‘×"Ñ"Ó$ð 	Œð 	�‰Õr&   ©r  r  r  r  rZ   Úlabelsrø   rù   r  c                 ó  — |�|n| j                   j                  }| j                  |||¬«      }|d   }|j                  d«      }| j	                  |«      }d}|��‡| j                   j
                  €�| j                  dk(  rd| j                   _        nl| j                  dkD  rL|j                  t        j                  k(  s|j                  t        j                  k(  rd| j                   _        nd| j                   _        | j                   j
                  dk(  rIt        «       }	| j                  dk(  r& |	|j                  «       |j                  «       «      }nŒ |	||«      }n‚| j                   j
                  dk(  r=t        «       }	 |	|j                  d| j                  «      |j                  d«      «      }n,| j                   j
                  dk(  rt        «       }	 |	||«      }|s|f|d	d z   }
|�|f|
z   S |
S t!        |||j"                  ¬
«      S )aŠ  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the image classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        Nr  r   r   Ú
regressionÚsingle_label_classificationÚmulti_label_classificationro   rH   )Úlossr   r   )rT   r   r  r  r/  Úproblem_typer-  Údtyper"   ÚlongÚintr
   Úsqueezer	   rp   r   r   r   )r4   rZ   r1  rø   rù   ÚoutputsÚsequence_outputr   r6  Úloss_fctÚoutputs              r'   r?   z#LevitForImageClassification.forwardd  sÇ  € ð( &1Ð%<‘kÀ$Ç+Á+×B]ÑB]ˆà—*‘*˜\Ð@TÐbm�*Ónˆà! !™*ˆØ)×.Ñ.¨qÓ1ˆØ—‘ Ó1ˆàˆØÑØ�{‰{×'Ñ'Ð/Ø—?‘? aÒ'Ø/;�D—K‘KÕ,Ø—_‘_ qÒ(¨f¯l©l¼e¿j¹jÒ.HÈFÏLÉLÔ\a×\eÑ\eÒLeØ/L�D—K‘KÕ,à/K�D—K‘KÔ,à�{‰{×'Ñ'¨<Ò7Ü"›9�Ø—?‘? aÒ'Ù# F§N¡NÓ$4°f·n±nÓ6FÓG‘Dá# F¨FÓ3‘DØ—‘×)Ñ)Ð-JÒJÜ+Ó-�Ù §¡¨B°·±Ó @À&Ç+Á+ÈbÃ/ÓR‘Ø—‘×)Ñ)Ð-IÒIÜ,Ó.�Ù ¨Ó/�ÙØ�Y ¨¨ Ñ,ˆFØ)-Ð)9�T�G˜fÑ$ÐE¸vÐEä3ØØØ!×/Ñ/ô
ð 	
r&   )NNNN)r   r   r    r/   r   r$  r   Ú_IMAGE_CLASS_CHECKPOINTr   r&  Ú_IMAGE_CLASS_EXPECTED_OUTPUTr   r"   r#   r‘   r(  r   r   r?   rA   rB   s   @r'   r*  r*  L  s£   ø„ ôñ  +Ð+AÓBÙØ*Ø8Ø$Ø4ô	ð 59Ø-1Ø/3Ø&*ñ3
à˜u×0Ñ0Ñ1ð3
ð ˜×)Ñ)Ñ*ð3
ð ' t™nð	3
ð
 ˜d‘^ð3
ð 
ˆuÐ:Ð:Ñ	;ò3
óó Cô3
r&   r*  ap  
    LeViT Model transformer with image classification heads on top (a linear layer on top of the final hidden state and
    a linear layer on top of the final hidden state of the distillation token) e.g. for ImageNet. .. warning::
           This model supports inference-only. Fine-tuning with distillation (i.e. with a teacher) is not yet
           supported.
    c                   ó¢   ‡ — e Zd Zˆ fd„Z ee«       eeee	e
¬«      	 	 	 ddeej                     dee   dee   deeef   fd„«       «       Zˆ xZS )	Ú&LevitForImageClassificationWithTeacherc                 óè  •— t         ‰| �  |«       || _        |j                  | _        t	        |«      | _        |j                  dkD  r#t        |j                  d   |j                  «      nt        j                  j                  «       | _        |j                  dkD  r#t        |j                  d   |j                  «      nt        j                  j                  «       | _        | j                  «        y r,  )r.   r/   rT   r-  r  r  rü   rJ   r"   r   r.  r/  Úclassifier_distillr  rS   s     €r'   r/   z/LevitForImageClassificationWithTeacher.__init__«  sÅ   ø€ Ü‰Ñ˜Ô ØˆŒØ ×+Ñ+ˆŒÜ Ó'ˆŒ
ð
 × Ñ  1Ò$ô % V×%8Ñ%8¸Ñ%<¸f×>OÑ>OÔPä—‘×"Ñ"Ó$ð 	Œð × Ñ  1Ò$ô % V×%8Ñ%8¸Ñ%<¸f×>OÑ>OÔPä—‘×"Ñ"Ó$ð 	Ôð 	�‰Õr&   r0  rZ   rø   rù   r  c                 ó.  — |�|n| j                   j                  }| j                  |||¬«      }|d   }|j                  d«      }| j	                  |«      | j                  |«      }}||z   dz  }|s|||f|dd  z   }	|	S t        ||||j                  ¬«      S )Nr  r   r   rH   )r   r   r   r   )rT   r   r  r  r/  rE  r   r   )
r4   rZ   rø   rù   r<  r=  r   Údistill_logitsr   r?  s
             r'   r?   z.LevitForImageClassificationWithTeacher.forwardÀ  s´   € ð &1Ð%<‘kÀ$Ç+Á+×B]ÑB]ˆà—*‘*˜\Ð@TÐbm�*Ónˆà! !™*ˆØ)×.Ñ.¨qÓ1ˆØ%)§_¡_°_Ó%EÀt×G^ÑG^Ð_nÓGo�Nˆ
Ø˜~Ñ-°Ñ2ˆáØ˜j¨.Ð9¸GÀAÀB¸KÑGˆFØˆMä;ØØ!Ø .Ø!×/Ñ/ô	
ð 	
r&   r#  )r   r   r    r/   r   r$  r   r@  r   r&  rA  r   r"   r#   r(  r   r   r?   rA   rB   s   @r'   rC  rC  ¡  s‹   ø„ ôñ* +Ð+AÓBÙØ*Ø@Ø$Ø4ô	ð 59Ø/3Ø&*ñ	
à˜u×0Ñ0Ñ1ð
ð ' t™nð
ð ˜d‘^ð	
ð
 
ˆuÐBÐBÑ	Cò
óó Cô
r&   rC  )r*  rC  r  r   )9r!   r†   Údataclassesr   Útypingr   r   r   r"   Útorch.utils.checkpointr   Útorch.nnr   r	   r
   Úmodeling_outputsr   r   r   r   Úmodeling_utilsr   Úutilsr   r   r   r   Úconfiguration_levitr   Ú
get_loggerr   Úloggerr&  r%  r'  r@  rA  r   ÚModuler)   rD   r\   rk   rv   r¹   rÇ   rÎ   rÙ   ré   rü   r   ÚLEVIT_START_DOCSTRINGr$  r  r*  rC  Ú__all__r%   r&   r'   ú<module>rU     sð  ðñ ã Ý !ß )Ñ )ã Û Ý ß AÑ A÷ó õ .ß uÓ uÝ ,ð 
ˆ×	Ñ	˜HÓ	%€ð  €ð ,Ð Ú%Ð ð 0Ð Ø1Ð ð ô=°;ó =ó ð=ô2˜"Ÿ)™)ô ô()5˜2Ÿ9™9ô )5ôX	�R—Y‘Yô 	ô�R—Y‘Yô ô;�R—Y‘Yô ;ô|P˜bŸi™iô Pôf�B—I‘Iô ô$ ˜Ÿ™ô  ô*B�—‘ô BôJ+o�2—9‘9ô +oô\˜rŸy™yô ô *˜?ô *ð0	Ð ðÐ ñ ØTØóô2
Ð%ó 2
ó	ð2
ñj ðð óôK
Ð"6ó K
óðK
ñ\ ðð óô5
Ð-Aó 5
óð5
òp�r&   