Ë
    T^(hèW  ã                   óÄ  — d Z ddlZddlmZ ddlmZmZmZ ddlZddl	Zddlm
Z
 ddlmZ ddlmZmZ dd	lmZmZ d
dlmZ  ej*                  e«      Ze G d„ de«      «       Z G d„ de
j2                  «      Z G d„ de
j2                  «      Z G d„ de
j2                  «      Z G d„ de
j2                  «      Z G d„ de
j2                  «      Z G d„ de
j2                  «      Zy)zTPyTorch IdeficsVision model: a copy of CLIPVisionModel using a simpler config objecté    N)Ú	dataclass)ÚOptionalÚTupleÚUnion)Únné   )ÚACT2FN)ÚBaseModelOutputÚBaseModelOutputWithPooling)ÚModelOutputÚloggingé   )ÚIdeficsVisionConfigc                   óÆ   — e Zd ZU dZdZeej                     ed<   dZ	eej                     ed<   dZ
eeej                  df      ed<   dZeeej                  df      ed<   y)ÚIdeficsVisionModelOutputaÝ  
    Base class for vision model's outputs that also contains image embeddings of the pooling of the last hidden states.

    Args:
        image_embeds (`torch.FloatTensor` of shape `(batch_size, output_dim)` *optional* returned when model is initialized with `with_projection=True`):
            The image embeddings obtained by applying the projection layer to the pooler_output.
        last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
            Sequence of hidden-states at the output of the last layer of the model.
        hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
            Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, +
            one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`.

            Hidden-states of the model at the output of each layer plus the optional initial embedding outputs.
        attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
            Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
            sequence_length)`.

            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
            heads.
    NÚimage_embedsÚlast_hidden_state.Úhidden_statesÚ
attentions)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r   ÚtorchÚFloatTensorÚ__annotations__r   r   r   r   © ó    ú`/var/www/skyplay_api_hub/venv/lib/python3.12/site-packages/transformers/models/idefics/vision.pyr   r   "   sr   … ñð* 15€L�(˜5×,Ñ,Ñ-Ó4Ø59Ð�x × 1Ñ 1Ñ2Ó9Ø=A€M�8˜E %×"3Ñ"3°SÐ"8Ñ9Ñ:ÓAØ:>€J�˜˜u×0Ñ0°#Ð5Ñ6Ñ7Ô>r   r   c                   ó¢   ‡ — e Zd Zdefˆ fd„Zdej                  dededej                  fd„Zddej                  d	e
dej                  fd
„Zˆ xZS )ÚIdeficsVisionEmbeddingsÚconfigc                 óÚ  •— t         ‰| �  «        || _        |j                  | _        |j
                  | _        |j                  | _        t        j                  t        j                  | j                  «      «      | _        t        j                  |j                  | j                  | j                  | j                  d¬«      | _        | j
                  | j                  z  dz  | _        | j                  dz   | _        t        j"                  | j                   | j                  «      | _        | j'                  dt        j(                  | j                   «      j+                  d«      d¬«       y )NF)Úin_channelsÚout_channelsÚkernel_sizeÚstrideÚbiasé   r   Úposition_ids)r   éÿÿÿÿ)Ú
persistent)ÚsuperÚ__init__r"   Úhidden_sizeÚ	embed_dimÚ
image_sizeÚ
patch_sizer   Ú	Parameterr   ÚrandnÚclass_embeddingÚConv2dÚnum_channelsÚpatch_embeddingÚnum_patchesÚnum_positionsÚ	EmbeddingÚposition_embeddingÚregister_bufferÚarangeÚexpand©Úselfr"   Ú	__class__s     €r   r.   z IdeficsVisionEmbeddings.__init__A   s	  ø€ Ü‰ÑÔØˆŒØ×+Ñ+ˆŒØ ×+Ñ+ˆŒØ ×+Ñ+ˆŒä!Ÿ|™|¬E¯K©K¸¿¹Ó,GÓHˆÔä!Ÿy™yØ×+Ñ+ØŸ™ØŸ™Ø—?‘?Øô 
ˆÔð !ŸO™O¨t¯©Ñ>À1ÑDˆÔØ!×-Ñ-°Ñ1ˆÔÜ"$§,¡,¨t×/AÑ/AÀ4Ç>Á>Ó"RˆÔØ×Ñ˜^¬U¯\©\¸$×:LÑ:LÓ-M×-TÑ-TÐU\Ó-]ÐjoÐÕpr   Ú
embeddingsÚheightÚwidthÚreturnc                 ó¼  — |j                   d   dz
  }| j                  | j                  «      }|j                   d   dz
  }||k(  r||k(  r|S |dd…df   }|dd…dd…f   }|j                   d   }	|| j                  j                  z  }
|| j                  j                  z  }|
dz   |dz   }}
t        j                  |«      }|j                  dt        |«      t        |«      |	«      }|j                  dddd«      }|j                  t        j                  k(  }|r4t        j                  d«       |j                  t        j                   «      }t"        j$                  j'                  ||
|z  ||z  fd	d
¬«      }|r|j                  t        j                  «      }t        |
«      |j                   d   k7  st        |«      |j                   d   k7  rBt)        dt        |
«      t        |«      f› d|j                   d   |j                   d   f› d�«      ‚|j                  dddd«      j+                  dd|	«      }t        j,                  |j/                  d«      |fd¬«      S )a#  
        This method allows to interpolate the pre-trained position encodings, to be able to use the model on higher
        resolution images.

        Source:
        https://github.com/facebookresearch/dino/blob/de9ee3df6cf39fac952ab558447af1fa1365362a/vision_transformer.py#L174
        r   Nr   r+   gš™™™™™¹?r   r)   zËUpcasting patch_pos_embed to fp32 for interpolation since `upsample_bicubic2d_out_frame` in nn.functional.interpolate is not implemented for 'torch.bfloat16' dtype. This will result in a slight overhead.ÚbicubicF)Úscale_factorÚmodeÚalign_cornerséþÿÿÿzNumber of patches for images (z/) don't match the shape of position embedding (ú)©Údim)Úshaper<   r*   r"   r2   ÚmathÚsqrtÚreshapeÚintÚpermuteÚdtyper   Úbfloat16ÚloggerÚwarning_onceÚtoÚfloatr   Ú
functionalÚinterpolateÚ
ValueErrorÚviewÚcatÚ	unsqueeze)rA   rC   rD   rE   r9   Ú	pos_embedr:   Úclass_pos_embedÚpatch_pos_embedr0   Únum_h_patchesÚnum_w_patchesÚsqrt_num_positionsÚfp32_upcastings                 r   Úinterpolate_pos_encodingz0IdeficsVisionEmbeddings.interpolate_pos_encodingX   sf  € ð !×&Ñ& qÑ)¨AÑ-ˆØ×+Ñ+¨D×,=Ñ,=Ó>ˆ	Ø!Ÿ™¨Ñ*¨QÑ.ˆØ˜-Ò'¨F°eªOØÐØ#¢A q D™/ˆØ#¢A q¡r EÑ*ˆà×$Ñ$ RÑ(ˆ	Ø $§+¡+×"8Ñ"8Ñ8ˆØ §¡×!7Ñ!7Ñ7ˆð (5°sÑ':¸MÈCÑ<O�}ˆÜ!ŸY™Y }Ó5ÐØ)×1Ñ1°!´SÐ9KÓ5LÌcÐRdÓNeÐgpÓqˆØ)×1Ñ1°!°Q¸¸1Ó=ˆØ(×.Ñ.´%·.±.Ñ@ˆÙÜ×Ñðhôð .×0Ñ0´·±Ó=ˆOÜŸ-™-×3Ñ3ØØ'Ð*<Ñ<¸mÐN`Ñ>`ÐaØØð	 4ó 
ˆñ Ø-×0Ñ0´·±Ó@ˆOÜˆ}Ó ×!6Ñ!6°rÑ!:Ò:¼cÀ-Ó>PÐTc×TiÑTiÐjlÑTmÒ>mÜØ0´°]Ó1CÄSÈÓEWÐ1WÐ0Xð Y0Ø0?×0EÑ0EÀbÑ0IÈ?×K`ÑK`ÐacÑKdÐ0dÐ/eÐefðhóð ð *×1Ñ1°!°Q¸¸1Ó=×BÑBÀ1ÀbÈ)ÓTˆÜ�y‰y˜/×3Ñ3°AÓ6¸ÐHÈaÔPÐPr   Úpixel_valuesri   c                 ó`  — |j                   \  }}}}|sJ|| j                  k7  s|| j                  k7  r,t        d|› d|› d| j                  › d| j                  › d�	«      ‚| j                  j                  j
                  }| j                  |j                  |¬«      «      }|j                  d«      j                  dd«      }| j                  j                  |dd«      }	t        j                  |	|gd¬	«      }
|r|
| j                  |
||«      z   }
|
S |
| j                  | j                  «      z   }
|
S )
NzInput image size (Ú*z) doesn't match model (z8). You should try to set `interpolate_pos_encoding=True`)rV   r)   r   r+   rN   )rP   r1   r^   r8   ÚweightrV   rZ   ÚflattenÚ	transposer5   r?   r   r`   ri   r<   r*   )rA   rj   ri   Ú
batch_sizer7   rD   rE   Útarget_dtypeÚpatch_embedsÚclass_embedsrC   s              r   ÚforwardzIdeficsVisionEmbeddings.forward‰   s8  € Ø2>×2DÑ2DÑ/ˆ
�L &¨%Ù'Ø˜Ÿ™Ò(¨E°T·_±_Ò,DÜ Ø(¨¨°°%°ð 9ØŸ™Ð)¨¨4¯?©?Ð*;Ð;sðuóð ð
 ×+Ñ+×2Ñ2×8Ñ8ˆØ×+Ñ+¨L¯O©OÀ,¨OÓ,OÓPˆà#×+Ñ+¨AÓ.×8Ñ8¸¸AÓ>ˆà×+Ñ+×2Ñ2°:¸qÀ"ÓEˆÜ—Y‘Y ¨lÐ;ÀÔCˆ
ñ $Ø# d×&CÑ&CÀJÐPVÐX]Ó&^Ñ^ˆJð Ðð $ d×&=Ñ&=¸d×>OÑ>OÓ&PÑPˆJàÐr   ©F)r   r   r   r   r.   r   ÚTensorrT   ri   r   Úboolrt   Ú__classcell__©rB   s   @r   r!   r!   @   sm   ø„ ðqÐ2õ qð./Q°5·<±<ð /QÈð /QÐUXð /QÐ]b×]iÑ]ió /Qñb E×$5Ñ$5ð ÐQUð Ðbg×bnÑbn÷ r   r!   c                   óô   ‡ — e Zd ZdZˆ fd„Zdej                  dedefd„Z	 	 	 ddej                  de	ej                     d	e	ej                     d
e	e
   deej                  e	ej                     f   f
d„Zˆ xZS )ÚIdeficsVisionAttentionz=Multi-headed attention from 'Attention Is All You Need' paperc                 ó
  •— t         ‰| �  «        || _        |j                  | _        |j
                  | _        | j                  | j                  z  | _        | j                  | j                  z  | j                  k7  r&t        d| j                  › d| j                  › d�«      ‚| j                  dz  | _	        |j                  | _        t        j                  | j                  | j                  «      | _        t        j                  | j                  | j                  «      | _        t        j                  | j                  | j                  «      | _        t        j                  | j                  | j                  «      | _        y )Nz;embed_dim must be divisible by num_heads (got `embed_dim`: z and `num_heads`: z).g      à¿)r-   r.   r"   r/   r0   Únum_attention_headsÚ	num_headsÚhead_dimr^   ÚscaleÚattention_dropoutÚdropoutr   ÚLinearÚk_projÚv_projÚq_projÚout_projr@   s     €r   r.   zIdeficsVisionAttention.__init__§   s  ø€ Ü‰ÑÔØˆŒØ×+Ñ+ˆŒØ×3Ñ3ˆŒØŸ™¨$¯.©.Ñ8ˆŒØ�=‰=˜4Ÿ>™>Ñ)¨T¯^©^Ò;ÜØMÈdÏnÉnÐM]ð ^Ø—N‘NÐ# 2ð'óð ð —]‘] DÑ(ˆŒ
Ø×/Ñ/ˆŒä—i‘i §¡°·±Ó?ˆŒÜ—i‘i §¡°·±Ó?ˆŒÜ—i‘i §¡°·±Ó?ˆŒÜŸ	™	 $§.¡.°$·.±.ÓAˆ�r   ÚtensorÚseq_lenÚbszc                 óŽ   — |j                  ||| j                  | j                  «      j                  dd«      j	                  «       S )Nr   r)   )r_   r~   r   ro   Ú
contiguous)rA   rˆ   r‰   rŠ   s       r   Ú_shapezIdeficsVisionAttention._shapeº   s7   € Ø�{‰{˜3 ¨¯©¸¿¹ÓG×QÑQÐRSÐUVÓW×bÑbÓdÐdr   r   Úattention_maskÚcausal_attention_maskÚoutput_attentionsrF   c                 ó”  — |j                  «       \  }}}| j                  |«      | j                  z  }| j                  | j	                  |«      d|«      }	| j                  | j                  |«      d|«      }
|| j                  z  d| j                  f} | j                  |||«      j                  |Ž } |	j                  |Ž }	 |
j                  |Ž }
|	j                  d«      }t        j                  ||	j                  dd«      «      }|j                  «       || j                  z  ||fk7  r/t        d|| j                  z  ||f› d|j                  «       › �«      ‚|�{|j                  «       |d||fk7  r#t        d|d||f› d|j                  «       › �«      ‚|j                  || j                  ||«      |z   }|j                  || j                  z  ||«      }|�{|j                  «       |d||fk7  r#t        d|d||f› d|j                  «       › �«      ‚|j                  || j                  ||«      |z   }|j                  || j                  z  ||«      }t        j                  j                  |d¬«      }|r?|j                  || j                  ||«      }|j                  || j                  z  ||«      }nd}t        j                  j!                  || j                   | j"                  ¬	«      }t        j                  ||
«      }|j                  «       || j                  z  || j                  fk7  r7t        d
|| j                  || j                  f› d|j                  «       › �«      ‚|j                  || j                  || j                  «      }|j                  dd«      }|j%                  |||«      }| j'                  |«      }||fS )z#Input shape: Batch x Time x Channelr+   r   r)   z$Attention weights should be of size z	, but is Nz!Attention mask should be of size rN   )ÚpÚtrainingz `attn_output` should be of size )Úsizer†   r€   r�   r„   r…   r~   r   r_   r   Úbmmro   r^   r   r\   Úsoftmaxr‚   r“   rS   r‡   )rA   r   rŽ   r�   r�   rŠ   Útgt_lenr0   Úquery_statesÚ
key_statesÚvalue_statesÚ
proj_shapeÚsrc_lenÚattn_weightsÚattn_weights_reshapedÚ
attn_probsÚattn_outputs                    r   rt   zIdeficsVisionAttention.forward½   sÕ  € ð #0×"4Ñ"4Ó"6ÑˆˆW�ið —{‘{ =Ó1°D·J±JÑ>ˆØ—[‘[ §¡¨]Ó!;¸RÀÓEˆ
Ø—{‘{ 4§;¡;¨}Ó#=¸rÀ3ÓGˆà˜DŸN™NÑ*¨B°·±Ð>ˆ
ØC�t—{‘{ <°¸#Ó>×CÑCÀZÐPˆØ$�Z—_‘_ jÐ1ˆ
Ø(�|×(Ñ(¨*Ð5ˆà—/‘/ !Ó$ˆÜ—y‘y ¨z×/CÑ/CÀAÀqÓ/IÓJˆà×ÑÓ 3¨¯©Ñ#7¸À'Ð"JÒJÜØ6¸¸d¿n¹nÑ8LÈgÐW^Ð7_Ð6`ð aØ ×%Ñ%Ó'Ð(ð*óð ð !Ð,Ø$×)Ñ)Ó+°°Q¸ÀÐ/IÒIÜ Ø7¸¸aÀÈ'Ð8RÐ7Sð TØ-×2Ñ2Ó4Ð5ð7óð ð (×,Ñ,¨S°$·.±.À'È7ÓSÐVkÑkˆLØ'×,Ñ,¨S°4·>±>Ñ-AÀ7ÈGÓTˆLàÐ%Ø×"Ñ"Ó$¨¨a°¸'Ð(BÒBÜ Ø7¸¸aÀÈ'Ð8RÐ7SÐS\Ð]k×]pÑ]pÓ]rÐ\sÐtóð ð (×,Ñ,¨S°$·.±.À'È7ÓSÐVdÑdˆLØ'×,Ñ,¨S°4·>±>Ñ-AÀ7ÈGÓTˆLä—}‘}×,Ñ,¨\¸rÐ,ÓBˆáð
 %1×$5Ñ$5°c¸4¿>¹>È7ÐT[Ó$\Ð!Ø0×5Ñ5°c¸D¿N¹NÑ6JÈGÐU\Ó]‰Là$(Ð!ä—]‘]×*Ñ*¨<¸4¿<¹<ÐRV×R_ÑR_Ð*Ó`ˆ
ä—i‘i 
¨LÓ9ˆà×ÑÓ #¨¯©Ñ"6¸ÀÇÁÐ!OÒOÜØ2°C¸¿¹ÈÐRV×R_ÑR_Ð3`Ð2að bØ×$Ñ$Ó&Ð'ð)óð ð
 "×&Ñ& s¨D¯N©N¸GÀTÇ]Á]ÓSˆØ!×+Ñ+¨A¨qÓ1ˆØ!×)Ñ)¨#¨w¸	ÓBˆà—m‘m KÓ0ˆàÐ1Ð1Ð1r   )NNF)r   r   r   r   r.   r   rv   rT   r�   r   rw   r   rt   rx   ry   s   @r   r{   r{   ¤   s¥   ø„ ÙGôBð&e˜UŸ\™\ð e°Cð e¸có eð 26Ø8<Ø,1ñL2à—|‘|ðL2ð ! §¡Ñ.ðL2ð  (¨¯©Ñ5ð	L2ð
 $ D™>ðL2ð 
ˆu�|‰|˜X e§l¡lÑ3Ð3Ñ	4÷L2r   r{   c                   óV   ‡ — e Zd Zˆ fd„Zdej
                  dej
                  fd„Zˆ xZS )ÚIdeficsVisionMLPc                 ó  •— t         ‰| �  «        || _        t        |j                     | _        t        j                  |j                  |j                  «      | _
        t        j                  |j                  |j                  «      | _        y ©N)r-   r.   r"   r	   Ú
hidden_actÚactivation_fnr   rƒ   r/   Úintermediate_sizeÚfc1Úfc2r@   s     €r   r.   zIdeficsVisionMLP.__init__  sd   ø€ Ü‰ÑÔØˆŒÜ# F×$5Ñ$5Ñ6ˆÔÜ—9‘9˜V×/Ñ/°×1IÑ1IÓJˆŒÜ—9‘9˜V×5Ñ5°v×7IÑ7IÓJˆ�r   r   rF   c                 ól   — | j                  |«      }| j                  |«      }| j                  |«      }|S r¤   )r¨   r¦   r©   )rA   r   s     r   rt   zIdeficsVisionMLP.forward  s4   € ØŸ™ Ó/ˆØ×*Ñ*¨=Ó9ˆØŸ™ Ó/ˆØÐr   )r   r   r   r.   r   rv   rt   rx   ry   s   @r   r¢   r¢     s$   ø„ ôKð U§\¡\ð °e·l±l÷ r   r¢   c                   ó    ‡ — e Zd Zdefˆ fd„Z	 d	dej                  dej                  dej                  dee   de	ej                     f
d„Zˆ xZS )
ÚIdeficsVisionEncoderLayerr"   c                 óD  •— t         ‰| �  «        |j                  | _        t	        |«      | _        t        j                  | j                  |j                  ¬«      | _	        t        |«      | _        t        j                  | j                  |j                  ¬«      | _        y ©N)Úeps)r-   r.   r/   r0   r{   Ú	self_attnr   Ú	LayerNormÚlayer_norm_epsÚlayer_norm1r¢   ÚmlpÚlayer_norm2r@   s     €r   r.   z"IdeficsVisionEncoderLayer.__init__  sm   ø€ Ü‰ÑÔØ×+Ñ+ˆŒÜ/°Ó7ˆŒÜŸ<™<¨¯©¸F×<QÑ<QÔRˆÔÜ# FÓ+ˆŒÜŸ<™<¨¯©¸F×<QÑ<QÔRˆÕr   r   rŽ   r�   r�   rF   c                 óÎ   — |}| j                  |«      }| j                  ||||¬«      \  }}||z   }|}| j                  |«      }| j                  |«      }||z   }|f}|r||fz  }|S )aI  
        Args:
            hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)`
            attention_mask (`torch.FloatTensor`): attention mask of size
                `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values.
                `(config.encoder_attention_heads,)`.
            output_attentions (`bool`, *optional*):
                Whether or not to return the attentions tensors of all attention layers. See `attentions` under
                returned tensors for more detail.
        )r   rŽ   r�   r�   )r³   r°   rµ   r´   )rA   r   rŽ   r�   r�   Úresidualr�   Úoutputss           r   rt   z!IdeficsVisionEncoderLayer.forward&  s’   € ð" !ˆà×(Ñ(¨Ó7ˆØ&*§n¡nØ'Ø)Ø"7Ø/ð	 '5ó '
Ñ#ˆ�|ð ! =Ñ0ˆà ˆØ×(Ñ(¨Ó7ˆØŸ™ Ó/ˆØ  =Ñ0ˆà Ð"ˆáØ˜�Ñ&ˆGàˆr   ru   )r   r   r   r   r.   r   rv   r   rw   r   r   rt   rx   ry   s   @r   r¬   r¬     sg   ø„ ðSÐ2õ Sð -2ñ&à—|‘|ð&ð Ÿ™ð&ð  %Ÿ|™|ð	&ð
 $ D™>ð&ð 
ˆu× Ñ Ñ	!÷&r   r¬   c                   ó¤   ‡ — e Zd ZdZdefˆ fd„Z	 	 	 	 	 ddeej                     deej                     dee	   dee	   dee	   d	e
eef   fd
„Zˆ xZS )ÚIdeficsVisionEncoderz¿
    Transformer encoder consisting of `config.num_hidden_layers` self attention layers. Each layer is a
    [`IdeficsVisionEncoderLayer`].

    Args:
        config: IdeficsVisionConfig
    r"   c                 óÐ   •— t         ‰| �  «        || _        t        j                  t        |j                  «      D �cg c]  }t        |«      ‘Œ c}«      | _        d| _	        y c c}w )NF)
r-   r.   r"   r   Ú
ModuleListÚrangeÚnum_hidden_layersr¬   ÚlayersÚgradient_checkpointing)rA   r"   Ú_rB   s      €r   r.   zIdeficsVisionEncoder.__init__Y  sQ   ø€ Ü‰ÑÔØˆŒÜ—m‘mÔPUÐV\×VnÑVnÓPoÖ$pÈ1Ô%>¸vÕ%FÒ$pÓqˆŒØ&+ˆÕ#ùò %qs   ½A#rŽ   r�   r�   Úoutput_hidden_statesÚreturn_dictrF   c                 ó  — |�|n| j                   j                  }|�|n| j                   j                  }|�|n| j                   j                  }|rdnd}|rdnd}|}	t	        | j
                  «      D ]b  \  }
}|r||	fz   }| j                  r,| j                  r | j                  |j                  |	|||«      }n ||	|||¬«      }|d   }	|sŒZ||d   fz   }Œd |r||	fz   }|st        d„ |	||fD «       «      S t        |	||¬«      S )aÕ  
        Args:
            inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
                Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
                This is useful if you want more control over how to convert `input_ids` indices into associated vectors
                than the model's internal embedding lookup matrix.
            attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
                Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:

                - 1 for tokens that are **not masked**,
                - 0 for tokens that are **masked**.

                [What are attention masks?](../glossary#attention-mask)
            causal_attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
                Causal mask for the text model. Mask values selected in `[0, 1]`:

                - 1 for tokens that are **not masked**,
                - 0 for tokens that are **masked**.

                [What are attention masks?](../glossary#attention-mask)
            output_attentions (`bool`, *optional*):
                Whether or not to return the attentions tensors of all attention layers. See `attentions` under
                returned tensors for more detail.
            output_hidden_states (`bool`, *optional*):
                Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors
                for more detail.
            return_dict (`bool`, *optional*):
                Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
        Nr   )r�   r   r   c              3   ó&   K  — | ]	  }|€Œ|–— Œ y ­wr¤   r   )Ú.0Úvs     r   ú	<genexpr>z/IdeficsVisionEncoder.forward.<locals>.<genexpr>«  s   è ø€ Òe˜qÐWXÑWdœÑeùs   ‚Š)r   r   r   )r"   r�   rÂ   Úuse_return_dictÚ	enumerater¿   rÀ   r“   Ú_gradient_checkpointing_funcÚ__call__Útupler
   )rA   Úinputs_embedsrŽ   r�   r�   rÂ   rÃ   Úencoder_statesÚall_attentionsr   ÚidxÚencoder_layerÚlayer_outputss                r   rt   zIdeficsVisionEncoder.forward_  sH  € ðL 2CÐ1NÑ-ÐTX×T_ÑT_×TqÑTqÐà$8Ð$DÑ È$Ï+É+×JjÑJjð 	ð &1Ð%<‘kÀ$Ç+Á+×B]ÑB]ˆá3™¸ˆÙ0™°dˆà%ˆÜ"+¨D¯K©KÓ"8ò 	FÑˆC�Ù#Ø!/°=Ð2BÑ!B�Ø×*Ò*¨t¯}ª}Ø $× AÑ AØ!×*Ñ*Ø!Ø"Ø)Ø%ó!‘ñ !.Ø!Ø"Ø)Ø&7ô	!�ð *¨!Ñ,ˆMâ Ø!/°=ÀÑ3CÐ2EÑ!E‘ð-	Fñ0  Ø+¨}Ð.>Ñ>ˆNáÜÑe ]°NÀNÐ$SÔeÓeÐeÜØ+¸>ÐVdô
ð 	
r   )NNNNN)r   r   r   r   r   r.   r   r   rv   rw   r   r   r
   rt   rx   ry   s   @r   rº   rº   P  s–   ø„ ñð,Ð2õ ,ð 26Ø8<Ø,0Ø/3Ø&*ñO
ð ! §¡Ñ.ðO
ð  (¨¯©Ñ5ð	O
ð
 $ D™>ðO
ð ' t™nðO
ð ˜d‘^ðO
ð 
ˆu�oÐ%Ñ	&÷O
r   rº   c                   óŒ   ‡ — e Zd Zdefˆ fd„Z	 	 	 	 	 d
deej                     dee   dee   dee   dee   de	e
ef   fd	„Zˆ xZS )ÚIdeficsVisionTransformerr"   c                 ó   •— t         ‰| �  «        || _        |j                  }t	        |«      | _        t        j                  ||j                  ¬«      | _	        t        |«      | _        t        j                  ||j                  ¬«      | _        y r®   )r-   r.   r"   r/   r!   rC   r   r±   r²   Úpre_layrnormrº   ÚencoderÚpost_layernorm)rA   r"   r0   rB   s      €r   r.   z!IdeficsVisionTransformer.__init__³  sj   ø€ Ü‰ÑÔØˆŒØ×&Ñ&ˆ	ä1°&Ó9ˆŒÜŸL™L¨¸×8MÑ8MÔNˆÔÜ+¨FÓ3ˆŒÜ Ÿl™l¨9¸&×:OÑ:OÔPˆÕr   rj   r�   rÂ   ri   rÃ   rF   c                 óÌ  — |�|n| j                   j                  }|�|n| j                   j                  }|�|n| j                   j                  }|€t	        d«      ‚| j                  ||¬«      }| j                  |«      }| j                  ||||¬«      }|d   }|dd…ddd…f   }	| j                  |	«      }	|s
||	f|dd z   S t        ||	|j                  |j                  ¬«      S )z
        Returns:

        Nz You have to specify pixel_values)ri   )rÎ   r�   rÂ   rÃ   r   r   )r   Úpooler_outputr   r   )r"   r�   rÂ   rÉ   r^   rC   r×   rØ   rÙ   r   r   r   )
rA   rj   r�   rÂ   ri   rÃ   r   Úencoder_outputsr   Úpooled_outputs
             r   rt   z IdeficsVisionTransformer.forward¾  s  € ð 2CÐ1NÑ-ÐTX×T_ÑT_×TqÑTqÐà$8Ð$DÑ È$Ï+É+×JjÑJjð 	ð &1Ð%<‘kÀ$Ç+Á+×B]ÑB]ˆàÐÜÐ?Ó@Ð@àŸ™¨ÐOg˜ÓhˆØ×)Ñ)¨-Ó8ˆàŸ,™,Ø'Ø/Ø!5Ø#ð	 'ó 
ˆð ,¨AÑ.ÐØ)ª!¨Q²¨'Ñ2ˆØ×+Ñ+¨MÓ:ˆáØ% }Ð5¸ÈÈÐ8KÑKÐKä)Ø/Ø'Ø)×7Ñ7Ø&×1Ñ1ô	
ð 	
r   )NNNFN)r   r   r   r   r.   r   r   r   rw   r   r   r   rt   rx   ry   s   @r   rÕ   rÕ   ²  sˆ   ø„ ðQÐ2õ Qð 59Ø,0Ø/3Ø38Ø&*ñ+
à˜u×0Ñ0Ñ1ð+
ð $ D™>ð+
ð ' t™nð	+
ð
 #+¨4¡.ð+
ð ˜d‘^ð+
ð 
ˆuÐ0Ð0Ñ	1÷+
r   rÕ   ) r   rQ   Údataclassesr   Útypingr   r   r   r   Útorch.utils.checkpointr   Úactivationsr	   Úmodeling_outputsr
   r   Úutilsr   r   Úconfiguration_ideficsr   Ú
get_loggerr   rX   r   ÚModuler!   r{   r¢   r¬   rº   rÕ   r   r   r   ú<module>rç      sÅ   ðñ [ã Ý !ß )Ñ )ã Û Ý å !ß Kß )Ý 6ð 
ˆ×	Ñ	˜HÓ	%€ð ô?˜{ó ?ó ð?ô:`˜bŸi™iô `ôHe2˜RŸY™Yô e2ôR�r—y‘yô ô / §	¡	ô /ôf^
˜2Ÿ9™9ô ^
ôD7
˜rŸy™yõ 7
r   