Ë
    S^(hip  ã            	       ó  — d Z ddlZddlmZ ddlmZmZmZ ddl	Z	ddl
Z	ddl	mZ ddlmZmZmZ ddlmZmZmZ dd	lmZmZ dd
lmZmZmZ ddlmZ ddlmZ  ej>                  e «      Z!dZ"dZ#g d¢Z$dZ%dZ&e G d„ de«      «       Z'd@de	jP                  de)de*de	jP                  fd„Z+ G d„ dejX                  «      Z- G d„ dejX                  «      Z. G d„ dejX                  «      Z/ G d„ d ejX                  «      Z0 G d!„ d"ejX                  «      Z1 G d#„ d$ejX                  «      Z2 G d%„ d&ejX                  «      Z3 G d'„ d(ejX                  «      Z4 G d)„ d*ejX                  «      Z5 G d+„ d,ejX                  «      Z6 G d-„ d.ejX                  «      Z7 G d/„ d0ejX                  «      Z8 G d1„ d2ejX                  «      Z9 G d3„ d4ejX                  «      Z: G d5„ d6e«      Z;d7Z<d8Z= ed9e<«       G d:„ d;e;«      «       Z> ed<e<«       G d=„ d>e;«      «       Z?g d?¢Z@y)AzPyTorch CvT model.é    N)Ú	dataclass)ÚOptionalÚTupleÚUnion)Únn)ÚBCEWithLogitsLossÚCrossEntropyLossÚMSELossé   )Úadd_code_sample_docstringsÚadd_start_docstringsÚ%add_start_docstrings_to_model_forward)Ú$ImageClassifierOutputWithNoAttentionÚModelOutput)ÚPreTrainedModelÚ find_pruneable_heads_and_indicesÚprune_linear_layer)Úloggingé   )Ú	CvtConfigr   zmicrosoft/cvt-13)r   i€  é   r   ztabby, tabby catc                   ó”   — e Zd ZU dZdZeej                     ed<   dZ	eej                     ed<   dZ
eeej                  df      ed<   y)ÚBaseModelOutputWithCLSTokena  
    Base class for model's outputs, with potential hidden states and attentions.

    Args:
        last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
            Sequence of hidden-states at the output of the last layer of the model.
        cls_token_value (`torch.FloatTensor` of shape `(batch_size, 1, hidden_size)`):
            Classification token at the output of the last layer of the model.
        hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
            Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
            shape `(batch_size, sequence_length, hidden_size)`. Hidden-states of the model at the output of each layer
            plus the initial embedding outputs.
    NÚlast_hidden_stateÚcls_token_value.Úhidden_states)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r   ÚtorchÚFloatTensorÚ__annotations__r   r   r   © ó    úb/var/www/skyplay_api_hub/venv/lib/python3.12/site-packages/transformers/models/cvt/modeling_cvt.pyr   r   /   sS   … ñð 6:Ð�x × 1Ñ 1Ñ2Ó9Ø37€O�X˜e×/Ñ/Ñ0Ó7Ø=A€M�8˜E %×"3Ñ"3°SÐ"8Ñ9Ñ:ÔAr%   r   ÚinputÚ	drop_probÚtrainingÚreturnc                 ó  — |dk(  s|s| S d|z
  }| j                   d   fd| j                  dz
  z  z   }|t        j                  || j                  | j
                  ¬«      z   }|j                  «        | j                  |«      |z  }|S )aF  
    Drop paths (Stochastic Depth) per sample (when applied in main path of residual blocks).

    Comment by Ross Wightman: This is the same as the DropConnect impl I created for EfficientNet, etc networks,
    however, the original name is misleading as 'Drop Connect' is a different form of dropout in a separate paper...
    See discussion: https://github.com/tensorflow/tpu/issues/494#issuecomment-532968956 ... I've opted for changing the
    layer and argument names to 'drop path' rather than mix DropConnect as a layer name and use 'survival rate' as the
    argument.
    ç        r   r   )r   )ÚdtypeÚdevice)ÚshapeÚndimr!   Úrandr-   r.   Úfloor_Údiv)r'   r(   r)   Ú	keep_probr/   Úrandom_tensorÚoutputs          r&   Ú	drop_pathr7   E   s�   € ð �CÒ™xØˆØ�I‘€IØ�[‰[˜‰^Ð ¨¯
©
°Q©Ñ 7Ñ7€EØ¤§
¡
¨5¸¿¹ÈEÏLÉLÔ YÑY€MØ×ÑÔØ�Y‰Y�yÓ! MÑ1€FØ€Mr%   c                   óx   ‡ — e Zd ZdZd	dee   ddfˆ fd„Zdej                  dej                  fd„Z	de
fd„Zˆ xZS )
ÚCvtDropPathzXDrop paths (Stochastic Depth) per sample (when applied in main path of residual blocks).Nr(   r*   c                 ó0   •— t         ‰| �  «        || _        y ©N)ÚsuperÚ__init__r(   )Úselfr(   Ú	__class__s     €r&   r=   zCvtDropPath.__init__]   s   ø€ Ü‰ÑÔØ"ˆ�r%   r   c                 óD   — t        || j                  | j                  «      S r;   )r7   r(   r)   )r>   r   s     r&   ÚforwardzCvtDropPath.forwarda   s   € Ü˜¨¯©¸¿¹ÓFÐFr%   c                 ó8   — dj                  | j                  «      S )Nzp={})Úformatr(   )r>   s    r&   Ú
extra_reprzCvtDropPath.extra_reprd   s   € Ø�}‰}˜TŸ^™^Ó,Ð,r%   r;   )r   r   r   r    r   Úfloatr=   r!   ÚTensorrA   ÚstrrD   Ú__classcell__©r?   s   @r&   r9   r9   Z   sG   ø„ Ùbñ# (¨5¡/ð #¸Tõ #ðG U§\¡\ð G°e·l±ló Gð-˜C÷ -r%   r9   c                   ó(   ‡ — e Zd ZdZˆ fd„Zd„ Zˆ xZS )ÚCvtEmbeddingsz'
    Construct the CvT embeddings.
    c                 ó€   •— t         ‰| �  «        t        |||||¬«      | _        t	        j
                  |«      | _        y )N)Ú
patch_sizeÚnum_channelsÚ	embed_dimÚstrideÚpadding)r<   r=   ÚCvtConvEmbeddingsÚconvolution_embeddingsr   ÚDropoutÚdropout)r>   rM   rN   rO   rP   rQ   Údropout_rater?   s          €r&   r=   zCvtEmbeddings.__init__m   s:   ø€ Ü‰ÑÔÜ&7Ø!°È	ÐZ`Ðjqô'
ˆÔ#ô —z‘z ,Ó/ˆ�r%   c                 óJ   — | j                  |«      }| j                  |«      }|S r;   )rS   rU   )r>   Úpixel_valuesÚhidden_states      r&   rA   zCvtEmbeddings.forwardt   s&   € Ø×2Ñ2°<Ó@ˆØ—|‘| LÓ1ˆØÐr%   ©r   r   r   r    r=   rA   rH   rI   s   @r&   rK   rK   h   s   ø„ ñô0ör%   rK   c                   ó(   ‡ — e Zd ZdZˆ fd„Zd„ Zˆ xZS )rR   z"
    Image to Conv Embedding.
    c                 óö   •— t         ‰| �  «        t        |t        j                  j
                  «      r|n||f}|| _        t        j                  |||||¬«      | _	        t        j                  |«      | _        y )N)Úkernel_sizerP   rQ   )r<   r=   Ú
isinstanceÚcollectionsÚabcÚIterablerM   r   ÚConv2dÚ
projectionÚ	LayerNormÚnormalization)r>   rM   rN   rO   rP   rQ   r?   s         €r&   r=   zCvtConvEmbeddings.__init__   sa   ø€ Ü‰ÑÔÜ#-¨j¼+¿/¹/×:RÑ:RÔ#S‘ZÐZdÐfpÐYqˆ
Ø$ˆŒÜŸ)™) L°)ÈÐ\bÐlsÔtˆŒÜŸ\™\¨)Ó4ˆÕr%   c                 ó   — | j                  |«      }|j                  \  }}}}||z  }|j                  |||«      j                  ddd«      }| j                  r| j	                  |«      }|j                  ddd«      j                  ||||«      }|S ©Nr   é   r   )rc   r/   ÚviewÚpermutere   )r>   rX   Ú
batch_sizerN   ÚheightÚwidthÚhidden_sizes          r&   rA   zCvtConvEmbeddings.forward†   s™   € Ø—‘ |Ó4ˆØ2>×2DÑ2DÑ/ˆ
�L &¨%Ø˜u‘nˆà#×(Ñ(¨°\À;ÓO×WÑWÐXYÐ[\Ð^_Ó`ˆØ×ÒØ×-Ñ-¨lÓ;ˆLà#×+Ñ+¨A¨q°!Ó4×9Ñ9¸*ÀlÐTZÐ\aÓbˆØÐr%   rZ   rI   s   @r&   rR   rR   z   s   ø„ ñô5ö
r%   rR   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚCvtSelfAttentionConvProjectionc           	      ó˜   •— t         ‰| �  «        t        j                  |||||d|¬«      | _        t        j
                  |«      | _        y )NF)r]   rQ   rP   ÚbiasÚgroups)r<   r=   r   rb   ÚconvolutionÚBatchNorm2dre   )r>   rO   r]   rQ   rP   r?   s        €r&   r=   z'CvtSelfAttentionConvProjection.__init__”   sG   ø€ Ü‰ÑÔÜŸ9™9ØØØ#ØØØØô
ˆÔô  Ÿ^™^¨IÓ6ˆÕr%   c                 óJ   — | j                  |«      }| j                  |«      }|S r;   )rt   re   ©r>   rY   s     r&   rA   z&CvtSelfAttentionConvProjection.forward¡   s(   € Ø×'Ñ'¨Ó5ˆØ×)Ñ)¨,Ó7ˆØÐr%   ©r   r   r   r=   rA   rH   rI   s   @r&   rp   rp   “   s   ø„ ô7ör%   rp   c                   ó   — e Zd Zd„ Zy)Ú CvtSelfAttentionLinearProjectionc                 óz   — |j                   \  }}}}||z  }|j                  |||«      j                  ddd«      }|S rg   )r/   ri   rj   )r>   rY   rk   rN   rl   rm   rn   s          r&   rA   z(CvtSelfAttentionLinearProjection.forward¨   sK   € Ø2>×2DÑ2DÑ/ˆ
�L &¨%Ø˜u‘nˆà#×(Ñ(¨°\À;ÓO×WÑWÐXYÐ[\Ð^_Ó`ˆØÐr%   N)r   r   r   rA   r$   r%   r&   rz   rz   §   s   „ ór%   rz   c                   ó&   ‡ — e Zd Zdˆ fd„	Zd„ Zˆ xZS )ÚCvtSelfAttentionProjectionc                 óp   •— t         ‰| �  «        |dk(  rt        ||||«      | _        t	        «       | _        y )NÚdw_bn)r<   r=   rp   Úconvolution_projectionrz   Úlinear_projection)r>   rO   r]   rQ   rP   Úprojection_methodr?   s         €r&   r=   z#CvtSelfAttentionProjection.__init__±   s7   ø€ Ü‰ÑÔØ Ò'Ü*HÈÐT_ÐahÐjpÓ*qˆDÔ'Ü!AÓ!CˆÕr%   c                 óJ   — | j                  |«      }| j                  |«      }|S r;   )r€   r�   rw   s     r&   rA   z"CvtSelfAttentionProjection.forward·   s(   € Ø×2Ñ2°<Ó@ˆØ×-Ñ-¨lÓ;ˆØÐr%   )r   rx   rI   s   @r&   r}   r}   °   s   ø„ õDör%   r}   c                   ó.   ‡ — e Zd Z	 dˆ fd„	Zd„ Zd„ Zˆ xZS )ÚCvtSelfAttentionc                 óÎ  •— t         ‰| �  «        |dz  | _        || _        || _        || _        t        |||||dk(  rdn|¬«      | _        t        |||||¬«      | _        t        |||||¬«      | _	        t        j                  |||	¬«      | _        t        j                  |||	¬«      | _        t        j                  |||	¬«      | _        t        j                  |
«      | _        y )Ng      à¿ÚavgÚlinear)r‚   )rr   )r<   r=   ÚscaleÚwith_cls_tokenrO   Ú	num_headsr}   Úconvolution_projection_queryÚconvolution_projection_keyÚconvolution_projection_valuer   ÚLinearÚprojection_queryÚprojection_keyÚprojection_valuerT   rU   )r>   r‹   rO   r]   Ú	padding_qÚ
padding_kvÚstride_qÚ	stride_kvÚqkv_projection_methodÚqkv_biasÚattention_drop_raterŠ   Úkwargsr?   s                €r&   r=   zCvtSelfAttention.__init__¾   sá   ø€ ô 	‰ÑÔØ ‘_ˆŒ
Ø,ˆÔØ"ˆŒØ"ˆŒä,FØØØØØ*?À5Ò*H™hÐNcô-
ˆÔ)ô +EØ�{ J°	ÐMbô+
ˆÔ'ô -GØ�{ J°	ÐMbô-
ˆÔ)ô !#§	¡	¨)°YÀXÔ NˆÔÜ Ÿi™i¨	°9À8ÔLˆÔÜ "§	¡	¨)°YÀXÔ NˆÔä—z‘zÐ"5Ó6ˆ�r%   c                 ó´   — |j                   \  }}}| j                  | j                  z  }|j                  ||| j                  |«      j	                  dddd«      S )Nr   rh   r   r   )r/   rO   r‹   ri   rj   )r>   rY   rk   rn   Ú_Úhead_dims         r&   Ú"rearrange_for_multi_head_attentionz3CvtSelfAttention.rearrange_for_multi_head_attentionç   sV   € Ø%1×%7Ñ%7Ñ"ˆ
�K Ø—>‘> T§^¡^Ñ3ˆà× Ñ  ¨[¸$¿.¹.È(ÓS×[Ñ[Ð\]Ð_`ÐbcÐefÓgÐgr%   c                 ó`  — | j                   rt        j                  |d||z  gd«      \  }}|j                  \  }}}|j	                  ddd«      j                  ||||«      }| j                  |«      }| j                  |«      }	| j                  |«      }
| j                   rKt        j                  |	fd¬«      }	t        j                  ||fd¬«      }t        j                  ||
fd¬«      }
| j                  | j                  z  }| j                  | j                  |	«      «      }	| j                  | j                  |«      «      }| j                  | j                  |
«      «      }
t        j                   d|	|g«      | j"                  z  }t        j$                  j&                  j)                  |d¬«      }| j+                  |«      }t        j                   d||
g«      }|j                  \  }}}}|j	                  dddd«      j-                  «       j                  ||| j                  |z  «      }|S )	Nr   r   rh   ©Údimzbhlk,bhtk->bhltéÿÿÿÿzbhlt,bhtv->bhlvr   )rŠ   r!   Úsplitr/   rj   ri   r�   rŒ   rŽ   ÚcatrO   r‹   rž   r�   r‘   r’   Úeinsumr‰   r   Ú
functionalÚsoftmaxrU   Ú
contiguous)r>   rY   rl   rm   Ú	cls_tokenrk   rn   rN   ÚkeyÚqueryÚvaluer�   Úattention_scoreÚattention_probsÚcontextrœ   s                   r&   rA   zCvtSelfAttention.forwardí   sñ  € Ø×ÒÜ&+§k¡k°,ÀÀFÈUÁNÐ@SÐUVÓ&WÑ#ˆI�|Ø0<×0BÑ0BÑ-ˆ
�K à#×+Ñ+¨A¨q°!Ó4×9Ñ9¸*ÀlÐTZÐ\aÓbˆà×-Ñ-¨lÓ;ˆØ×1Ñ1°,Ó?ˆØ×1Ñ1°,Ó?ˆà×ÒÜ—I‘I˜y¨%Ð0°aÔ8ˆEÜ—)‘)˜Y¨Ð,°!Ô4ˆCÜ—I‘I˜y¨%Ð0°aÔ8ˆEà—>‘> T§^¡^Ñ3ˆà×7Ñ7¸×8MÑ8MÈeÓ8TÓUˆØ×5Ñ5°d×6IÑ6IÈ#Ó6NÓOˆØ×7Ñ7¸×8MÑ8MÈeÓ8TÓUˆäŸ,™,Ð'8¸5À#¸,ÓGÈ$Ï*É*ÑTˆÜŸ(™(×-Ñ-×5Ñ5°oÈ2Ð5ÓNˆØŸ,™, Ó7ˆä—,‘,Ð0°?ÀEÐ2JÓKˆà&Ÿ}™}Ñˆˆ1ˆk˜1Ø—/‘/ ! Q¨¨1Ó-×8Ñ8Ó:×?Ñ?À
ÈKÐY]×YgÑYgÐjrÑYrÓsˆØˆr%   ©T)r   r   r   r=   rž   rA   rH   rI   s   @r&   r…   r…   ½   s   ø„ ð õ'7òRhör%   r…   c                   ó(   ‡ — e Zd ZdZˆ fd„Zd„ Zˆ xZS )ÚCvtSelfOutputz 
    The residual connection is defined in CvtLayer instead of here (as is the case with other models), due to the
    layernorm applied before each block.
    c                 óŒ   •— t         ‰| �  «        t        j                  ||«      | _        t        j
                  |«      | _        y r;   )r<   r=   r   r�   ÚdenserT   rU   )r>   rO   Ú	drop_rater?   s      €r&   r=   zCvtSelfOutput.__init__  s0   ø€ Ü‰ÑÔÜ—Y‘Y˜y¨)Ó4ˆŒ
Ü—z‘z )Ó,ˆ�r%   c                 óJ   — | j                  |«      }| j                  |«      }|S r;   ©r´   rU   ©r>   rY   Úinput_tensors      r&   rA   zCvtSelfOutput.forward  s$   € Ø—z‘z ,Ó/ˆØ—|‘| LÓ1ˆØÐr%   rZ   rI   s   @r&   r²   r²     s   ø„ ñô
-ö
r%   r²   c                   ó.   ‡ — e Zd Z	 dˆ fd„	Zd„ Zd„ Zˆ xZS )ÚCvtAttentionc                 ó–   •— t         ‰| �  «        t        |||||||||	|
|«      | _        t	        ||«      | _        t        «       | _        y r;   )r<   r=   r…   Ú	attentionr²   r6   ÚsetÚpruned_heads)r>   r‹   rO   r]   r“   r”   r•   r–   r—   r˜   r™   rµ   rŠ   r?   s                €r&   r=   zCvtAttention.__init__   sW   ø€ ô 	‰ÑÔÜ)ØØØØØØØØ!ØØØó
ˆŒô $ I¨yÓ9ˆŒÜ›EˆÕr%   c                 ó>  — t        |«      dk(  ry t        || j                  j                  | j                  j                  | j
                  «      \  }}t        | j                  j                  |«      | j                  _        t        | j                  j                  |«      | j                  _        t        | j                  j                  |«      | j                  _	        t        | j                  j                  |d¬«      | j                  _        | j                  j                  t        |«      z
  | j                  _        | j                  j                  | j                  j                  z  | j                  _        | j
                  j                  |«      | _        y )Nr   r   r    )Úlenr   r½   Únum_attention_headsÚattention_head_sizer¿   r   r«   rª   r¬   r6   r´   Úall_head_sizeÚunion)r>   ÚheadsÚindexs      r&   Úprune_headszCvtAttention.prune_heads@  s  € Üˆu‹:˜Š?ØÜ7Ø�4—>‘>×5Ñ5°t·~±~×7YÑ7YÐ[_×[lÑ[ló
‰ˆˆuô
  2°$·.±.×2FÑ2FÈÓNˆ�‰ÔÜ/°·±×0BÑ0BÀEÓJˆ�‰ÔÜ1°$·.±.×2FÑ2FÈÓNˆ�‰ÔÜ.¨t¯{©{×/@Ñ/@À%ÈQÔOˆ�‰Ôð .2¯^©^×-OÑ-OÔRUÐV[ÓR\Ñ-\ˆ�‰Ô*Ø'+§~¡~×'IÑ'IÈDÏNÉN×LnÑLnÑ'nˆ�‰Ô$Ø ×-Ñ-×3Ñ3°EÓ:ˆÕr%   c                 óP   — | j                  |||«      }| j                  ||«      }|S r;   )r½   r6   )r>   rY   rl   rm   Úself_outputÚattention_outputs         r&   rA   zCvtAttention.forwardR  s+   € Ø—n‘n \°6¸5ÓAˆØŸ;™; {°LÓAÐØÐr%   r°   )r   r   r   r=   rÈ   rA   rH   rI   s   @r&   r»   r»     s   ø„ ð õ"ò@;ö$ r%   r»   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚCvtIntermediatec                 ó¢   •— t         ‰| �  «        t        j                  |t	        ||z  «      «      | _        t        j                  «       | _        y r;   )r<   r=   r   r�   Úintr´   ÚGELUÚ
activation)r>   rO   Ú	mlp_ratior?   s      €r&   r=   zCvtIntermediate.__init__Y  s7   ø€ Ü‰ÑÔÜ—Y‘Y˜y¬#¨i¸)Ñ.CÓ*DÓEˆŒ
ÜŸ'™'›)ˆ�r%   c                 óJ   — | j                  |«      }| j                  |«      }|S r;   )r´   rÑ   rw   s     r&   rA   zCvtIntermediate.forward^  s$   € Ø—z‘z ,Ó/ˆØ—‘ |Ó4ˆØÐr%   rx   rI   s   @r&   rÍ   rÍ   X  s   ø„ ô$ö
r%   rÍ   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )Ú	CvtOutputc                 ó¤   •— t         ‰| �  «        t        j                  t	        ||z  «      |«      | _        t        j                  |«      | _        y r;   )r<   r=   r   r�   rÏ   r´   rT   rU   )r>   rO   rÒ   rµ   r?   s       €r&   r=   zCvtOutput.__init__e  s:   ø€ Ü‰ÑÔÜ—Y‘Yœs 9¨yÑ#8Ó9¸9ÓEˆŒ
Ü—z‘z )Ó,ˆ�r%   c                 óT   — | j                  |«      }| j                  |«      }||z   }|S r;   r·   r¸   s      r&   rA   zCvtOutput.forwardj  s.   € Ø—z‘z ,Ó/ˆØ—|‘| LÓ1ˆØ# lÑ2ˆØÐr%   rx   rI   s   @r&   rÕ   rÕ   d  s   ø„ ô-ö
r%   rÕ   c                   ó,   ‡ — e Zd ZdZ	 dˆ fd„	Zd„ Zˆ xZS )ÚCvtLayerzb
    CvtLayer composed by attention layers, normalization and multi-layer perceptrons (mlps).
    c                 óZ  •— t         ‰| �  «        t        |||||||||	|
||«      | _        t	        ||«      | _        t        |||«      | _        |dkD  rt        |¬«      nt        j                  «       | _        t        j                  |«      | _        t        j                  |«      | _        y )Nr,   )r(   )r<   r=   r»   r½   rÍ   ÚintermediaterÕ   r6   r9   r   ÚIdentityr7   rd   Úlayernorm_beforeÚlayernorm_after)r>   r‹   rO   r]   r“   r”   r•   r–   r—   r˜   r™   rµ   rÒ   Údrop_path_raterŠ   r?   s                  €r&   r=   zCvtLayer.__init__v  s£   ø€ ô" 	‰ÑÔÜ%ØØØØØØØØ!ØØØØó
ˆŒô ,¨I°yÓAˆÔÜ 	¨9°iÓ@ˆŒØBPÐSVÒBVœ¨~Õ>Ô\^×\gÑ\gÓ\iˆŒÜ "§¡¨YÓ 7ˆÔÜ!Ÿ|™|¨IÓ6ˆÕr%   c                 ó  — | j                  | j                  |«      ||«      }|}| j                  |«      }||z   }| j                  |«      }| j	                  |«      }| j                  ||«      }| j                  |«      }|S r;   )r½   rÝ   r7   rÞ   rÛ   r6   )r>   rY   rl   rm   Úself_attention_outputrË   Úlayer_outputs          r&   rA   zCvtLayer.forward�  s‘   € Ø $§¡Ø×!Ñ! ,Ó/ØØó!
Ðð
 1ÐØŸ>™>Ð*:Ó;Ðð (¨,Ñ6ˆð ×+Ñ+¨LÓ9ˆØ×(Ñ(¨Ó6ˆð —{‘{ <°Ó>ˆØ—~‘~ lÓ3ˆØÐr%   r°   rZ   rI   s   @r&   rÙ   rÙ   q  s   ø„ ñð& õ%7öNr%   rÙ   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚCvtStagec                 óx  •— t         ‰| �  «        || _        || _        | j                  j                  | j                     rFt        j                  t        j                  dd| j                  j                  d   «      «      | _        t        |j                  | j                     |j                  | j                     | j                  dk(  r|j                  n|j                  | j                  dz
     |j                  | j                     |j                  | j                     |j                  | j                     ¬«      | _        t        j"                  d|j$                  | j                     |j&                  |   «      D �cg c]  }|j)                  «       ‘Œ }}t        j*                  t-        |j&                  | j                     «      D �cg c�]T  }t/        |j0                  | j                     |j                  | j                     |j2                  | j                     |j4                  | j                     |j6                  | j                     |j8                  | j                     |j:                  | j                     |j<                  | j                     |j>                  | j                     |j@                  | j                     |j                  | j                     || j                     |jB                  | j                     |j                  | j                     ¬«      ‘�ŒW c}Ž | _"        y c c}w c c}w )Nr   r¢   r   )rM   rP   rN   rO   rQ   rV   )r‹   rO   r]   r“   r”   r–   r•   r—   r˜   r™   rµ   rß   rÒ   rŠ   )#r<   r=   ÚconfigÚstager©   r   Ú	Parameterr!   ÚrandnrO   rK   Úpatch_sizesÚpatch_striderN   Úpatch_paddingrµ   Ú	embeddingÚlinspacerß   ÚdepthÚitemÚ
SequentialÚrangerÙ   r‹   Ú
kernel_qkvr“   r”   r–   r•   r—   r˜   r™   rÒ   Úlayers)r>   ræ   rç   ÚxÚdrop_path_ratesrœ   r?   s         €r&   r=   zCvtStage.__init__´  sw  ø€ Ü‰ÑÔØˆŒØˆŒ
Ø�;‰;× Ñ  §¡Ò,ÜŸ\™\¬%¯+©+°a¸¸D¿K¹K×<QÑ<QÐRTÑ<UÓ*VÓWˆDŒNä&Ø×)Ñ)¨$¯*©*Ñ5Ø×&Ñ& t§z¡zÑ2Ø04·
±
¸a²˜×,Ò,ÀV×EUÑEUÐVZ×V`ÑV`ÐcdÑVdÑEeØ×&Ñ& t§z¡zÑ2Ø×(Ñ(¨¯©Ñ4Ø×)Ñ)¨$¯*©*Ñ5ô
ˆŒô .3¯^©^¸A¸v×?TÑ?TÐUY×U_ÑU_Ñ?`Ðbh×bnÑbnÐotÑbuÓ-vÖw¨˜1Ÿ6™6�8ÐwˆÐwä—m‘mô$ ˜vŸ|™|¨D¯J©JÑ7Ó8÷#ð" ô! Ø$×.Ñ.¨t¯z©zÑ:Ø$×.Ñ.¨t¯z©zÑ:Ø &× 1Ñ 1°$·*±*Ñ =Ø$×.Ñ.¨t¯z©zÑ:Ø%×0Ñ0°·±Ñ<Ø$×.Ñ.¨t¯z©zÑ:Ø#Ÿ_™_¨T¯Z©ZÑ8Ø*0×*FÑ*FÀtÇzÁzÑ*RØ#Ÿ_™_¨T¯Z©ZÑ8Ø(.×(BÑ(BÀ4Ç:Á:Ñ(NØ$×.Ñ.¨t¯z©zÑ:Ø#2°4·:±:Ñ#>Ø$×.Ñ.¨t¯z©zÑ:Ø#)×#3Ñ#3°D·J±JÑ#?÷òð
ˆ�ùò xùòs   ÆL2ÇEL7c                 óZ  — d }| j                  |«      }|j                  \  }}}}|j                  ||||z  «      j                  ddd«      }| j                  j
                  | j                     r6| j
                  j                  |dd«      }t        j                  ||fd¬«      }| j                  D ]  } ||||«      }|}Œ | j                  j
                  | j                     rt        j                  |d||z  gd«      \  }}|j                  ddd«      j                  ||||«      }||fS )Nr   rh   r   r¢   r    )rí   r/   ri   rj   ræ   r©   rç   Úexpandr!   r¤   rô   r£   )	r>   rY   r©   rk   rN   rl   rm   ÚlayerÚlayer_outputss	            r&   rA   zCvtStage.forwardÜ  s'  € Øˆ	Ø—~‘~ lÓ3ˆØ2>×2DÑ2DÑ/ˆ
�L &¨%à#×(Ñ(¨°\À6ÈEÁ>ÓR×ZÑZÐ[\Ð^_ÐabÓcˆØ�;‰;× Ñ  §¡Ò,ØŸ™×-Ñ-¨j¸"¸bÓAˆIÜ Ÿ9™9 i°Ð%>ÀAÔFˆLà—[‘[ò 	)ˆEÙ! ,°¸Ó>ˆMØ(‰Lð	)ð �;‰;× Ñ  §¡Ò,Ü&+§k¡k°,ÀÀFÈUÁNÐ@SÐUVÓ&WÑ#ˆI�|Ø#×+Ñ+¨A¨q°!Ó4×9Ñ9¸*ÀlÐTZÐ\aÓbˆØ˜YÐ&Ð&r%   rx   rI   s   @r&   rä   rä   ³  s   ø„ ô&
öP'r%   rä   c                   ó&   ‡ — e Zd Zˆ fd„Zdd„Zˆ xZS )Ú
CvtEncoderc                 óô   •— t         ‰| �  «        || _        t        j                  g «      | _        t        t        |j                  «      «      D ]'  }| j
                  j                  t        ||«      «       Œ) y r;   )r<   r=   ræ   r   Ú
ModuleListÚstagesrò   rÁ   rï   Úappendrä   )r>   ræ   Ú	stage_idxr?   s      €r&   r=   zCvtEncoder.__init__ñ  s[   ø€ Ü‰ÑÔØˆŒÜ—m‘m BÓ'ˆŒÜœs 6§<¡<Ó0Ó1ò 	<ˆIØ�K‰K×Ñœx¨°	Ó:Õ;ñ	<r%   c                 óÂ   — |rdnd }|}d }t        | j                  «      D ]  \  }} ||«      \  }}|sŒ||fz   }Œ |st        d„ |||fD «       «      S t        |||¬«      S )Nr$   c              3   ó&   K  — | ]	  }|€Œ|–— Œ y ­wr;   r$   )Ú.0Úvs     r&   ú	<genexpr>z%CvtEncoder.forward.<locals>.<genexpr>  s   è ø€ Òb˜qÐTUÑTaœÑbùs   ‚Š©r   r   r   )Ú	enumeraterÿ   Útupler   )	r>   rX   Úoutput_hidden_statesÚreturn_dictÚall_hidden_statesrY   r©   rœ   Ústage_modules	            r&   rA   zCvtEncoder.forwardø  sˆ   € Ù"6™B¸DÐØ#ˆàˆ	Ü!*¨4¯;©;Ó!7ò 	HÑˆA�Ù&2°<Ó&@Ñ#ˆL˜)Ú#Ø$5¸¸Ñ$GÑ!ð	Hñ
 ÜÑb \°9Ð>OÐ$PÔbÓbÐbä*Ø*Ø%Ø+ô
ð 	
r%   )FTrx   rI   s   @r&   rü   rü   ð  s   ø„ ô<÷
r%   rü   c                   ó(   — e Zd ZdZeZdZdZdgZd„ Z	y)ÚCvtPreTrainedModelz†
    An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained
    models.
    ÚcvtrX   rÙ   c                 óR  — t        |t        j                  t        j                  f«      r‹t        j                  j                  |j                  j                  d| j                  j                  ¬«      |j                  _        |j                  �%|j                  j                  j                  «        yyt        |t        j                  «      rJ|j                  j                  j                  «        |j                  j                  j                  d«       yt        |t        «      r~| j                  j                  |j                      rZt        j                  j                  |j                  j                  d| j                  j                  ¬«      |j                  _        yyy)zInitialize the weightsr,   )ÚmeanÚstdNg      ð?)r^   r   r�   rb   ÚinitÚtrunc_normal_ÚweightÚdataræ   Úinitializer_rangerr   Úzero_rd   Úfill_rä   r©   rç   )r>   Úmodules     r&   Ú_init_weightsz CvtPreTrainedModel._init_weights  s  € ä�fœrŸy™y¬"¯)©)Ð4Ô5Ü!#§¡×!6Ñ!6°v·}±}×7IÑ7IÐPSÐY]×YdÑYd×YvÑYvÐ!6Ó!wˆF�M‰MÔØ�{‰{Ð&Ø—‘× Ñ ×&Ñ&Õ(ð 'ä˜¤§¡Ô-Ø�K‰K×Ñ×"Ñ"Ô$Ø�M‰M×Ñ×$Ñ$ SÕ)Ü˜¤Ô)Ø�{‰{×$Ñ$ V§\¡\Ò2Ü(*¯©×(=Ñ(=Ø×$Ñ$×)Ñ)°¸¿¹×9VÑ9Vð )>ó )�× Ñ Õ%ð 3ð *r%   N)
r   r   r   r    r   Úconfig_classÚbase_model_prefixÚmain_input_nameÚ_no_split_modulesr  r$   r%   r&   r  r    s&   „ ñð
 €LØÐØ$€OØ#˜Ðór%   r  aE  
    This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. Use it
    as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and
    behavior.

    Parameters:
        config ([`CvtConfig`]): Model configuration class with all the parameters of the model.
            Initializing with a config file does not load the weights associated with the model, only the
            configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
aE  
    Args:
        pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`):
            Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See [`CvtImageProcessor.__call__`]
            for details.
        output_hidden_states (`bool`, *optional*):
            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
            more detail.
        return_dict (`bool`, *optional*):
            Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple.
z]The bare Cvt Model transformer outputting raw hidden-states without any specific head on top.c                   ó¬   ‡ — e Zd Zd
ˆ fd„	Zd„ Z ee«       eee	e
de¬«      	 	 	 ddeej                     dee   dee   deee	f   fd	„«       «       Zˆ xZS )ÚCvtModelc                 ór   •— t         ‰| �  |«       || _        t        |«      | _        | j                  «        y r;   )r<   r=   ræ   rü   ÚencoderÚ	post_init)r>   ræ   Úadd_pooling_layerr?   s      €r&   r=   zCvtModel.__init__D  s-   ø€ Ü‰Ñ˜Ô ØˆŒÜ! &Ó)ˆŒØ�‰Õr%   c                 ó˜   — |j                  «       D ]7  \  }}| j                  j                  |   j                  j	                  |«       Œ9 y)z�
        Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base
        class PreTrainedModel
        N)Úitemsr$  rù   r½   rÈ   )r>   Úheads_to_prunerù   rÆ   s       r&   Ú_prune_headszCvtModel._prune_headsJ  sE   € ð
 +×0Ñ0Ó2ò 	C‰LˆE�5Ø�L‰L×Ñ˜uÑ%×/Ñ/×;Ñ;¸EÕBñ	Cr%   Úvision)Ú
checkpointÚoutput_typer  ÚmodalityÚexpected_outputrX   r
  r  r*   c                 ó  — |�|n| j                   j                  }|�|n| j                   j                  }|€t        d«      ‚| j	                  |||¬«      }|d   }|s	|f|dd  z   S t        ||j                  |j                  ¬«      S )Nz You have to specify pixel_values©r
  r  r   r   r  )ræ   r
  Úuse_return_dictÚ
ValueErrorr$  r   r   r   )r>   rX   r
  r  Úencoder_outputsÚsequence_outputs         r&   rA   zCvtModel.forwardR  sª   € ð %9Ð$DÑ È$Ï+É+×JjÑJjð 	ð &1Ð%<‘kÀ$Ç+Á+×B]ÑB]ˆàÐÜÐ?Ó@Ð@àŸ,™,ØØ!5Ø#ð 'ó 
ˆð
 *¨!Ñ,ˆáØ#Ð%¨¸¸Ð(;Ñ;Ð;ä*Ø-Ø+×;Ñ;Ø)×7Ñ7ô
ð 	
r%   r°   )NNN)r   r   r   r=   r*  r   ÚCVT_INPUTS_DOCSTRINGr   Ú_CHECKPOINT_FOR_DOCr   Ú_CONFIG_FOR_DOCÚ_EXPECTED_OUTPUT_SHAPEr   r!   rF   Úboolr   r   rA   rH   rI   s   @r&   r"  r"  ?  s’   ø„ õ
òCñ +Ð+?Ó@ÙØ&Ø/Ø$ØØ.ôð 04Ø/3Ø&*ñ	
à˜uŸ|™|Ñ,ð
ð ' t™nð
ð ˜d‘^ð	
ð
 
ˆuÐ1Ð1Ñ	2ò
óó Aô
r%   r"  z¤
    Cvt Model transformer with an image classification head on top (a linear layer on top of the final hidden state of
    the [CLS] token) e.g. for ImageNet.
    c                   óÂ   ‡ — e Zd Zˆ fd„Z ee«       eeee	e
¬«      	 	 	 	 d	deej                     deej                     dee   dee   deeef   f
d„«       «       Zˆ xZS )
ÚCvtForImageClassificationc                 ó‚  •— t         ‰| �  |«       |j                  | _        t        |d¬«      | _        t        j                  |j                  d   «      | _        |j                  dkD  r-t        j                  |j                  d   |j                  «      nt        j                  «       | _        | j                  «        y )NF)r&  r¢   r   )r<   r=   Ú
num_labelsr"  r  r   rd   rO   Ú	layernormr�   rÜ   Ú
classifierr%  )r>   ræ   r?   s     €r&   r=   z"CvtForImageClassification.__init__�  s–   ø€ Ü‰Ñ˜Ô à ×+Ñ+ˆŒÜ˜F°eÔ<ˆŒÜŸ™ f×&6Ñ&6°rÑ&:Ó;ˆŒð CI×BSÑBSÐVWÒBWŒB�I‰I�f×&Ñ& rÑ*¨F×,=Ñ,=Ô>Ô]_×]hÑ]hÓ]jð 	Œð
 	�‰Õr%   )r,  r-  r  r/  rX   Úlabelsr
  r  r*   c                 ób  — |�|n| j                   j                  }| j                  |||¬«      }|d   }|d   }| j                   j                  d   r| j	                  |«      }nI|j
                  \  }}	}
}|j                  ||	|
|z  «      j                  ddd«      }| j	                  |«      }|j                  d¬«      }| j                  |«      }d}|��¯| j                   j                  €¡| j                   j                  dk(  rd| j                   _
        nv| j                   j                  dkD  rL|j                  t        j                  k(  s|j                  t        j                  k(  rd	| j                   _
        nd
| j                   _
        | j                   j                  dk(  rSt!        «       }| j                   j                  dk(  r& ||j#                  «       |j#                  «       «      }n– |||«      }nŒ| j                   j                  d	k(  rGt%        «       } ||j                  d| j                   j                  «      |j                  d«      «      }n,| j                   j                  d
k(  rt'        «       } |||«      }|s|f|dd z   }|�|f|z   S |S t)        |||j*                  ¬«      S )aŠ  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the image classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        Nr1  r   r   r¢   rh   r    Ú
regressionÚsingle_label_classificationÚmulti_label_classification)ÚlossÚlogitsr   )ræ   r2  r  r©   r?  r/   ri   rj   r  r@  Úproblem_typer>  r-   r!   ÚlongrÏ   r
   Úsqueezer	   r   r   r   )r>   rX   rA  r
  r  Úoutputsr5  r©   rk   rN   rl   rm   Úsequence_output_meanrG  rF  Úloss_fctr6   s                    r&   rA   z!CvtForImageClassification.forward�  s`  € ð( &1Ð%<‘kÀ$Ç+Á+×B]ÑB]ˆØ—(‘(ØØ!5Ø#ð ó 
ˆð " !™*ˆØ˜A‘Jˆ	Ø�;‰;× Ñ  Ò$Ø"Ÿn™n¨YÓ7‰Oà6E×6KÑ6KÑ3ˆJ˜ f¨eà-×2Ñ2°:¸|ÈVÐV[É^Ó\×dÑdÐefÐhiÐklÓmˆOØ"Ÿn™n¨_Ó=ˆOà.×3Ñ3¸Ð3Ó:ÐØ—‘Ð!5Ó6ˆàˆØÑØ�{‰{×'Ñ'Ð/Ø—;‘;×)Ñ)¨QÒ.Ø/;�D—K‘KÕ,Ø—[‘[×+Ñ+¨aÒ/°V·\±\ÄUÇZÁZÒ5OÐSY×S_ÑS_Ôch×clÑclÒSlØ/L�D—K‘KÕ,à/K�D—K‘KÔ,à�{‰{×'Ñ'¨<Ò7Ü"›9�Ø—;‘;×)Ñ)¨QÒ.Ù# F§N¡NÓ$4°f·n±nÓ6FÓG‘Dá# F¨FÓ3‘DØ—‘×)Ñ)Ð-JÒJÜ+Ó-�Ù §¡¨B°·±×0FÑ0FÓ GÈÏÉÐUWËÓY‘Ø—‘×)Ñ)Ð-IÒIÜ,Ó.�Ù ¨Ó/�áØ�Y ¨¨ Ñ,ˆFØ)-Ð)9�T�G˜fÑ$ÐE¸vÐEä3¸ÀfÐ\c×\qÑ\qÔrÐrr%   )NNNN)r   r   r   r=   r   r6  r   Ú_IMAGE_CLASS_CHECKPOINTr   r8  Ú_IMAGE_CLASS_EXPECTED_OUTPUTr   r!   rF   r:  r   r   rA   rH   rI   s   @r&   r<  r<  y  s¦   ø„ ôñ +Ð+?Ó@ÙØ*Ø8Ø$Ø4ô	ð 04Ø)-Ø/3Ø&*ñ<sà˜uŸ|™|Ñ,ð<sð ˜Ÿ™Ñ&ð<sð ' t™nð	<sð
 ˜d‘^ð<sð 
ˆuÐ:Ð:Ñ	;ò<sóó Aô<sr%   r<  )r<  r"  r  )r,   F)Ar    Úcollections.abcr_   Údataclassesr   Útypingr   r   r   r!   Útorch.utils.checkpointr   Útorch.nnr   r	   r
   Ú
file_utilsr   r   r   Úmodeling_outputsr   r   Úmodeling_utilsr   r   r   Úutilsr   Úconfiguration_cvtr   Ú
get_loggerr   Úloggerr8  r7  r9  rN  rO  r   rF   rE   r:  r7   ÚModuler9   rK   rR   rp   rz   r}   r…   r²   r»   rÍ   rÕ   rÙ   rä   rü   r  ÚCVT_START_DOCSTRINGr6  r"  r<  Ú__all__r$   r%   r&   ú<module>r_     s  ðñ ã Ý !ß )Ñ )ã Û Ý ß AÑ Aç qÑ qß Qß cÑ cÝ Ý (ð 
ˆ×	Ñ	˜HÓ	%€ð €ð )Ð Ú)Ð ð -Ð Ø1Ð ð ôB +ó Bó ðBñ*�U—\‘\ð ¨eð ÀTð ÐV[×VbÑVbó ô*-�"—)‘)ô -ô�B—I‘Iô ô$˜Ÿ	™	ô ô2 R§Y¡Yô ô( r§y¡yô ô
 §¡ô 
ôN�r—y‘yô Nôb�B—I‘Iô ô"6 �2—9‘9ô 6 ôr	�b—i‘iô 	ô
�—	‘	ô 
ô?ˆr�y‰yô ?ôD:'ˆr�y‰yô :'ôz
�—‘ô 
ô8˜ô ð6	Ð ð
Ð ñ ØcØóô3
Ð!ó 3
ó	ð3
ñl ðð óôRsÐ 2ó RsóðRsòj J�r%   