Ë
    T^(h€ ã                   óø  — d Z ddlmZ ddlmZmZmZmZmZ ddl	Z	ddl
Z	ddl	mZ ddlmZ ddlmZ dd	lmZmZ dd
lmZ ddlmZ ddlmZmZ ddlmZmZ ddlmZmZm Z m!Z! ddl"m#Z# ddl$m%Z% ddl&m'Z'm(Z(m)Z)  e jT                  e+«      Z,dZ-e G d„ de«      «       Z.e G d„ de«      «       Z/ G d„ dej`                  «      Z1	 dQdej`                  de	jd                  de	jd                  de	jd                  dee	jd                     d e3d!e3fd"„Z4 G d#„ d$ej`                  «      Z5 G d%„ d&ej`                  «      Z6 G d'„ d(ej`                  «      Z7 G d)„ d*ej`                  «      Z8 G d+„ d,ej`                  «      Z9 G d-„ d.ej`                  «      Z:d/Z; ed0e;«       G d1„ d2e«      «       Z<d3Z= ed4e;«       G d5„ d6e<«      «       Z>d7e	jd                  d8e?d9e	jd                  fd:„Z@ G d;„ d<ej`                  «      ZA G d=„ d>ej`                  «      ZB G d?„ d@ej`                  «      ZCdAZ= edBdCdDe;«       G dE„ dFe<«      «       ZD G dG„ dHej`                  «      ZEdIZ= edJe;«       G dK„ dLe<«      «       ZF edMe;«       G dN„ dOe<e«      «       ZGg dP¢ZHy)RzPyTorch Idefics2 model.é    )Ú	dataclass)ÚCallableÚListÚOptionalÚTupleÚUnionN)Únn)ÚCrossEntropyLossé   )ÚACT2FN)ÚCacheÚDynamicCache)ÚGenerationMixin)Ú_prepare_4d_attention_mask)ÚBaseModelOutputÚModelOutput)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)Úadd_start_docstringsÚ%add_start_docstrings_to_model_forwardÚloggingÚreplace_return_docstrings)Údeprecate_kwargé   )Ú	AutoModelé   )ÚIdefics2ConfigÚIdefics2PerceiverConfigÚIdefics2VisionConfigr   c                   óø   — e Zd ZU dZdZeej                     ed<   dZ	ee
e
ej                           ed<   dZee
ej                        ed<   dZee
ej                        ed<   dZee
ej                        ed<   y)ÚIdefics2BaseModelOutputWithPastaá	  
    Base class for Idefics2 model's outputs that may also contain a past key/values (to speed up sequential decoding).
    Args:
        last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
            Sequence of hidden-states at the output of the last layer of the model.
            If `past_key_values` is used only the last hidden-state of the sequences of shape `(batch_size, 1,
            hidden_size)` is output.
        past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
            Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape
            `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and optionally if
            `config.is_encoder_decoder=True` 2 additional tensors of shape `(batch_size, num_heads,
            encoder_sequence_length, embed_size_per_head)`.
            Contains pre-computed hidden-states (key and values in the self-attention blocks and optionally if
            `config.is_encoder_decoder=True` in the cross-attention blocks) that can be used (see `past_key_values`
            input) to speed up sequential decoding.
        hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
            Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, +
            one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`.
            Hidden-states of the model at the output of each layer plus the optional initial embedding outputs.
        attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
            Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
            sequence_length)`.
            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
            heads.
        image_hidden_states (`tuple(torch.FloatTensor)`, *optional*):
            Tuple of `torch.FloatTensor` (one for the output of the image embeddings, `(batch_size, num_images,
            sequence_length, hidden_size)`.
            image_hidden_states of the model produced by the vision encoder, and optionally by the perceiver
    NÚlast_hidden_stateÚpast_key_valuesÚhidden_statesÚ
attentionsÚimage_hidden_states)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r"   r   ÚtorchÚFloatTensorÚ__annotations__r#   r   r$   r%   r&   © ó    úl/var/www/skyplay_api_hub/venv/lib/python3.12/site-packages/transformers/models/idefics2/modeling_idefics2.pyr!   r!   /   s�   … ñð< 6:Ð�x × 1Ñ 1Ñ2Ó9ØAE€O�X˜e E¨%×*;Ñ*;Ñ$<Ñ=Ñ>ÓEØ8<€M�8˜E %×"3Ñ"3Ñ4Ñ5Ó<Ø59€J�˜˜u×0Ñ0Ñ1Ñ2Ó9Ø>BÐ˜ %¨×(9Ñ(9Ñ":Ñ;ÔBr/   r!   c                   ó  — e Zd ZU dZdZeej                     ed<   dZ	eej                     ed<   dZ
eeej                        ed<   dZeeej                        ed<   dZeeej                        ed<   dZeeej                        ed<   y)	ÚIdefics2CausalLMOutputWithPastaÐ  
    Base class for Idefics2 causal language model (or autoregressive) outputs.
    Args:
        loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
            Language modeling loss (for next-token prediction).
        logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`):
            Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
        past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
            Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape
            `(batch_size, num_heads, sequence_length, embed_size_per_head)`)
            Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see
            `past_key_values` input) to speed up sequential decoding.
        hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
            Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, +
            one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`.
            Hidden-states of the model at the output of each layer plus the optional initial embedding outputs.
        attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
            Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
            sequence_length)`.
            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
            heads.
        image_hidden_states (`tuple(torch.FloatTensor)`, *optional*):
            Tuple of `torch.FloatTensor` (one for the output of the image embeddings, `(batch_size, num_images,
            sequence_length, hidden_size)`.
            image_hidden_states of the model produced by the vision encoder, and optionally by the perceiver
    NÚlossÚlogitsr#   r$   r%   r&   )r'   r(   r)   r*   r3   r   r+   r,   r-   r4   r#   r   r$   r   r%   r&   r.   r/   r0   r2   r2   V   sœ   … ñð6 )-€Dˆ(�5×$Ñ$Ñ
%Ó,Ø*.€FˆH�U×&Ñ&Ñ'Ó.Ø9=€O�X˜d 5×#4Ñ#4Ñ5Ñ6Ó=Ø8<€M�8˜E %×"3Ñ"3Ñ4Ñ5Ó<Ø59€J�˜˜u×0Ñ0Ñ1Ñ2Ó9Ø>BÐ˜ %¨×(9Ñ(9Ñ":Ñ;ÔBr/   r2   c                   óx   ‡ — e Zd ZdZdefˆ fd„Zdej                  dej                  dej                  fd„Z
ˆ xZS )ÚIdefics2VisionEmbeddingsaP  
    This is a modified version of `siglip.modelign_siglip.SiglipVisionEmbeddings` to enable images of variable
    resolution.

    The modifications are adapted from [Patch n' Pack: NaViT, a Vision Transformer for any Aspect Ratio and Resolution](https://arxiv.org/abs/2307.06304)
    which allows treating images in their native aspect ratio and without the need to resize them to the same
    fixed size. In particular, we start from the original pre-trained SigLIP model
    (which uses images of fixed-size square images) and adapt it by training on images of variable resolutions.
    Úconfigc                 óú  •— t         ‰| �  «        |j                  | _        |j                  | _        |j
                  | _        t        j                  |j                  | j                  | j
                  | j
                  d¬«      | _	        | j                  | j
                  z  | _
        | j                  dz  | _        | j                  | _        t        j                  | j                  | j                  «      | _        y )NÚvalid)Úin_channelsÚout_channelsÚkernel_sizeÚstrideÚpaddingr   )ÚsuperÚ__init__Úhidden_sizeÚ	embed_dimÚ
image_sizeÚ
patch_sizer	   ÚConv2dÚnum_channelsÚpatch_embeddingÚnum_patches_per_sideÚnum_patchesÚnum_positionsÚ	EmbeddingÚposition_embedding©Úselfr7   Ú	__class__s     €r0   r@   z!Idefics2VisionEmbeddings.__init__‡   s¼   ø€ Ü‰ÑÔØ×+Ñ+ˆŒØ ×+Ñ+ˆŒØ ×+Ñ+ˆŒä!Ÿy™yØ×+Ñ+ØŸ™ØŸ™Ø—?‘?Øô 
ˆÔð %)§O¡O°t·±Ñ$FˆÔ!Ø×4Ñ4°aÑ7ˆÔØ!×-Ñ-ˆÔÜ"$§,¡,¨t×/AÑ/AÀ4Ç>Á>Ó"RˆÕr/   Úpixel_valuesÚpatch_attention_maskÚreturnc                 óÂ  — |j                   \  }}}}| j                  |«      }|j                  d«      j                  dd«      }|| j                  z  || j                  z  }
}	t        j                  d| j                  z  dd| j                  z  «      }t        j                  ||	|
z  fd¬«      }t        |«      D ]ß  \  }}|d d …df   j                  «       }|d   j                  «       }t        j                  ddd|z  «      }t        j                  ddd|z  «      }t        j                  ||d¬«      }t        j                  ||d¬«      }|d d …d f   | j                  z  |z   j                  «       }|||   |j                  d	«      j                  «       <   Œá |j                  | j                  j                   j"                  «      }|| j                  |«      z   }|S )
Nr   r   g      ð?r   )ÚsizeÚ
fill_valuegé!çýÿï?T)Úrightéÿÿÿÿ)ÚshaperG   ÚflattenÚ	transposerD   r+   ÚarangerH   ÚfullÚ	enumerateÚsumÚ	bucketizeÚviewÚcpuÚtorL   ÚweightÚdevice)rN   rP   rQ   Ú
batch_sizeÚ_Úmax_im_hÚmax_im_wÚpatch_embedsÚ
embeddingsÚmax_nb_patches_hÚmax_nb_patches_wÚ
boundariesÚposition_idsÚ	batch_idxÚp_attn_maskÚnb_patches_hÚnb_patches_wÚfractional_coords_hÚfractional_coords_wÚbucket_coords_hÚbucket_coords_wÚpos_idss                         r0   Úforwardz Idefics2VisionEmbeddings.forwardš   sÌ  € Ø,8×,>Ñ,>Ñ)ˆ
�A�x à×+Ñ+¨LÓ9ˆØ!×)Ñ)¨!Ó,×6Ñ6°q¸!Ó<ˆ
à-5¸¿¹Ñ-HÈ(ÐVZ×VeÑVeÑJeÐ*ÐÜ—\‘\ ! d×&?Ñ&?Ñ"?ÀÀaÈ$×JcÑJcÑFcÓdˆ
Ü—z‘z¨
Ð4DÐGWÑ4WÐ'XÐefÔgˆä&/Ð0DÓ&Eò 	JÑ"ˆI�{Ø&¢q¨! tÑ,×0Ñ0Ó2ˆLØ& q™>×-Ñ-Ó/ˆLä"'§,¡,¨q°(¸AÀÑ<LÓ"MÐÜ"'§,¡,¨q°(¸AÀÑ<LÓ"MÐä#Ÿo™oÐ.AÀ:ÐUYÔZˆOÜ#Ÿo™oÐ.AÀ:ÐUYÔZˆOà&¢q¨$ wÑ/°$×2KÑ2KÑKÈoÑ]×fÑfÓhˆGØBIˆL˜Ñ# K×$4Ñ$4°RÓ$8×$<Ñ$<Ó$>Ò?ð	Jð $—‘ t×'>Ñ'>×'EÑ'E×'LÑ'LÓMˆØ $×"9Ñ"9¸,Ó"GÑGˆ
ØÐr/   )r'   r(   r)   r*   r   r@   r+   r,   Ú
BoolTensorÚTensorrx   Ú__classcell__©rO   s   @r0   r6   r6   |   sD   ø„ ñðSÐ3õ Sð& E×$5Ñ$5ð ÈU×M]ÑM]ð Ðbg×bnÑbn÷ r/   r6   ÚmoduleÚqueryÚkeyÚvalueÚattention_maskÚscalingÚdropoutc                 ól  — t        | d«      r,t        || j                  «      }t        || j                  «      }t        j                  ||j                  dd«      «      |z  }|�#|d d …d d …d d …d |j                  d   …f   }	||	z   }t        j                  j                  |dt        j                  ¬«      j                  |j                  «      }t        j                  j                  ||| j                  ¬«      }t        j                  ||«      }
|
j                  dd«      j                  «       }
|
|fS )	NÚnum_key_value_groupsr   r   éþÿÿÿrW   )ÚdimÚdtype)ÚpÚtrainingr   )ÚhasattrÚ	repeat_kvr…   r+   ÚmatmulrZ   rX   r	   Ú
functionalÚsoftmaxÚfloat32rb   rˆ   rƒ   rŠ   Ú
contiguous)r}   r~   r   r€   r�   r‚   rƒ   ÚkwargsÚattn_weightsÚcausal_maskÚattn_outputs              r0   Úeager_attention_forwardr–   ¶   s  € ô ˆvÐ-Ô.Ü˜˜V×8Ñ8Ó9ˆÜ˜% ×!<Ñ!<Ó=ˆä—<‘<  s§}¡}°Q¸Ó':Ó;¸gÑE€LØÐ!Ø$¢Qªª1¨o°·	±	¸"±¨oÐ%=Ñ>ˆØ# kÑ1ˆä—=‘=×(Ñ(¨¸2ÄUÇ]Á]Ð(ÓS×VÑVÐW\×WbÑWbÓc€LÜ—=‘=×(Ñ(¨¸È6Ï?É?Ð(Ó[€LÜ—,‘,˜|¨UÓ3€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€Kà˜Ð$Ð$r/   c                   ó¬   ‡ — e Zd ZdZˆ fd„Z	 	 ddej                  deej                     dee   de	ej                  eej                     f   fd„Z
ˆ xZS )	ÚIdefics2VisionAttentionz=Multi-headed attention from 'Attention Is All You Need' paperc                 ó  •— t         ‰| �  «        || _        |j                  | _        |j
                  | _        | j                  | j                  z  | _        | j                  | j                  z  | j                  k7  r&t        d| j                  › d| j                  › d�«      ‚| j                  dz  | _	        |j                  | _        t        j                  | j                  | j                  «      | _        t        j                  | j                  | j                  «      | _        t        j                  | j                  | j                  «      | _        t        j                  | j                  | j                  «      | _        d| _        y )Nz;embed_dim must be divisible by num_heads (got `embed_dim`: z and `num_heads`: z).ç      à¿F)r?   r@   r7   rA   rB   Únum_attention_headsÚ	num_headsÚhead_dimÚ
ValueErrorÚscaleÚattention_dropoutrƒ   r	   ÚLinearÚk_projÚv_projÚq_projÚout_projÚ	is_causalrM   s     €r0   r@   z Idefics2VisionAttention.__init__Ö   s  ø€ Ü‰ÑÔØˆŒØ×+Ñ+ˆŒØ×3Ñ3ˆŒØŸ™¨$¯.©.Ñ8ˆŒØ�=‰=˜4Ÿ>™>Ñ)¨T¯^©^Ò;ÜØMÈdÏnÉnÐM]ð ^Ø—N‘NÐ# 2ð'óð ð —]‘] DÑ(ˆŒ
Ø×/Ñ/ˆŒä—i‘i §¡°·±Ó?ˆŒÜ—i‘i §¡°·±Ó?ˆŒÜ—i‘i §¡°·±Ó?ˆŒÜŸ	™	 $§.¡.°$·.±.ÓAˆŒð ˆ�r/   r$   r�   Úoutput_attentionsrR   c           
      ó¤  — |j                   \  }}}| j                  |«      }| j                  |«      }| j                  |«      }	|j	                  ||| j
                  | j                  «      j                  dd«      }|j	                  ||| j
                  | j                  «      j                  dd«      }|	j	                  ||| j
                  | j                  «      j                  dd«      }	t        }
| j                  j                  dk7  rN| j                  j                  dk(  r|rt        j                  d«       nt        | j                  j                     }
 |
| |||	|| j                  | j                  | j                   sdn| j"                  ¬«      \  }}|j%                  |||«      j'                  «       }| j)                  |«      }|sd}||fS )	z#Input shape: Batch x Time x Channelr   r   ÚeagerÚsdpaúã`torch.nn.functional.scaled_dot_product_attention` does not support `output_attentions=True`. Falling back to eager attention. This warning can be removed using the argument `attn_implementation="eager"` when loading the model.ç        ©r¦   r‚   rƒ   N)rX   r¤   r¢   r£   r`   rœ   r�   rZ   r–   r7   Ú_attn_implementationÚloggerÚwarning_oncer   r¦   rŸ   rŠ   rƒ   Úreshaper‘   r¥   )rN   r$   r�   r§   re   Ú
seq_lengthrB   ÚqueriesÚkeysÚvaluesÚattention_interfacer•   r“   s                r0   rx   zIdefics2VisionAttention.forwardì   s”  € ð -:×,?Ñ,?Ñ)ˆ
�J 	à—+‘+˜mÓ,ˆØ�{‰{˜=Ó)ˆØ—‘˜]Ó+ˆà—,‘,˜z¨:°t·~±~ÀtÇ}Á}ÓU×_Ñ_Ð`aÐcdÓeˆØ�y‰y˜ Z°·±ÀÇÁÓO×YÑYÐZ[Ð]^Ó_ˆØ—‘˜Z¨°T·^±^ÀTÇ]Á]ÓS×]Ñ]Ð^_ÐabÓcˆä(?ÐØ�;‰;×+Ñ+¨wÒ6Ø�{‰{×/Ñ/°6Ò9Ñ>OÜ×#Ñ#ðLõô
 '>¸d¿k¹k×>^Ñ>^Ñ&_Ð#á$7ØØØØØØ—n‘nØ—J‘JØ#Ÿ}š}‘C°$·,±,ô	%
Ñ!ˆ�\ð "×)Ñ)¨*°jÀ)ÓL×WÑWÓYˆØ—m‘m KÓ0ˆá ØˆLà˜LÐ(Ð(r/   ©NF)r'   r(   r)   r*   r@   r+   rz   r   Úboolr   rx   r{   r|   s   @r0   r˜   r˜   Ò   sf   ø„ ÙGôð2 26Ø,1ñ	-)à—|‘|ð-)ð ! §¡Ñ.ð-)ð $ D™>ð	-)ð
 
ˆu�|‰|˜X e§l¡lÑ3Ð3Ñ	4÷-)r/   r˜   c                   óV   ‡ — e Zd Zˆ fd„Zdej
                  dej
                  fd„Zˆ xZS )ÚIdefics2VisionMLPc                 ó  •— t         ‰| �  «        || _        t        |j                     | _        t        j                  |j                  |j                  «      | _
        t        j                  |j                  |j                  «      | _        y ©N)r?   r@   r7   r   Ú
hidden_actÚactivation_fnr	   r¡   rA   Úintermediate_sizeÚfc1Úfc2rM   s     €r0   r@   zIdefics2VisionMLP.__init__  sd   ø€ Ü‰ÑÔØˆŒÜ# F×$5Ñ$5Ñ6ˆÔÜ—9‘9˜V×/Ñ/°×1IÑ1IÓJˆŒÜ—9‘9˜V×5Ñ5°v×7IÑ7IÓJˆ�r/   r$   rR   c                 ól   — | j                  |«      }| j                  |«      }| j                  |«      }|S r¼   )rÀ   r¾   rÁ   )rN   r$   s     r0   rx   zIdefics2VisionMLP.forward%  s4   € ØŸ™ Ó/ˆØ×*Ñ*¨=Ó9ˆØŸ™ Ó/ˆØÐr/   )r'   r(   r)   r@   r+   rz   rx   r{   r|   s   @r0   rº   rº     s$   ø„ ôKð U§\¡\ð °e·l±l÷ r/   rº   c                   ó6   ‡ — e Zd Zdedededefˆ fd„Zd„ Zˆ xZS )ÚIdefics2MLPrA   r¿   Úoutput_sizer½   c                 óì   •— t         ‰| �  «        t        j                  ||d¬«      | _        t        j                  ||d¬«      | _        t        j                  ||d¬«      | _        t        |   | _        y ©NF©Úbias)	r?   r@   r	   r¡   Ú	gate_projÚup_projÚ	down_projr   Úact_fn)rN   rA   r¿   rÅ   r½   rO   s        €r0   r@   zIdefics2MLP.__init__-  s[   ø€ ô 	‰ÑÔÜŸ™ ;Ð0AÈÔNˆŒÜ—y‘y Ð.?ÀeÔLˆŒÜŸ™Ð#4°kÈÔNˆŒÜ˜ZÑ(ˆ�r/   c                 ó„   — | j                  | j                  | j                  |«      «      | j                  |«      z  «      S r¼   )rÌ   rÍ   rÊ   rË   )rN   Úxs     r0   rx   zIdefics2MLP.forward:  s0   € Ø�~‰~˜dŸk™k¨$¯.©.¸Ó*;Ó<¸t¿|¹|ÈA»ÑNÓOÐOr/   )r'   r(   r)   ÚintÚstrr@   rx   r{   r|   s   @r0   rÄ   rÄ   ,  s4   ø„ ð)àð)ð ð)ð ð	)ð
 õ)öPr/   rÄ   c                   ó.   ‡ — e Zd ZdZdefˆ fd„Zd„ Zˆ xZS )Ú%Idefics2MultiheadAttentionPoolingHeadzMultihead Attention Pooling.r7   c                 óâ  •— t         ‰| �  «        t        j                  t	        j
                  dd|j                  «      «      | _        t        j                  j                  |j                  |j                  d¬«      | _
        t        j                  |j                  |j                  ¬«      | _        t        |j                  |j                  |j                   |j                  ¬«      | _        y )Nr   T)Úbatch_first©Úeps)rA   r¿   r½   rÅ   )r?   r@   r	   Ú	Parameterr+   ÚrandnrA   ÚprobeÚMultiheadAttentionr›   Ú	attentionÚ	LayerNormÚlayer_norm_epsÚ	layernormrÄ   r¿   r½   ÚmlprM   s     €r0   r@   z.Idefics2MultiheadAttentionPoolingHead.__init__B  s©   ø€ Ü‰ÑÔä—\‘\¤%§+¡+¨a°°F×4FÑ4FÓ"GÓHˆŒ
ÜŸ™×4Ñ4°V×5GÑ5GÈ×IcÑIcÐquÐ4ÓvˆŒÜŸ™ f×&8Ñ&8¸f×>SÑ>SÔTˆŒäØ×*Ñ*Ø$×6Ñ6Ø×(Ñ(Ø×*Ñ*ô	
ˆ�r/   c                 óæ   — |j                   d   }| j                  j                  |dd«      }| j                  |||«      d   }|}| j	                  |«      }|| j                  |«      z   }|d d …df   S )Nr   r   )rX   rÚ   ÚrepeatrÜ   rß   rà   )rN   Úhidden_statere   rÚ   Úresiduals        r0   rx   z-Idefics2MultiheadAttentionPoolingHead.forwardP  sv   € Ø!×'Ñ'¨Ñ*ˆ
Ø—
‘
×!Ñ! *¨a°Ó3ˆà—~‘~ e¨\¸<ÓHÈÑKˆàˆØ—~‘~ lÓ3ˆØ $§(¡(¨<Ó"8Ñ8ˆàšA˜q˜DÑ!Ð!r/   )r'   r(   r)   r*   r   r@   rx   r{   r|   s   @r0   rÓ   rÓ   ?  s   ø„ Ù&ð
Ð3õ 
ö
"r/   rÓ   c            
       óˆ   ‡ — e Zd Zdefˆ fd„Z	 ddej                  dej                  dee   de	ej                     fd„Zˆ xZS )	ÚIdefics2EncoderLayerr7   c                 óD  •— t         ‰| �  «        |j                  | _        t	        |«      | _        t        j                  | j                  |j                  ¬«      | _	        t        |«      | _        t        j                  | j                  |j                  ¬«      | _        y )NrÖ   )r?   r@   rA   rB   r˜   Ú	self_attnr	   rÝ   rÞ   Úlayer_norm1rº   rà   Úlayer_norm2rM   s     €r0   r@   zIdefics2EncoderLayer.__init__^  sm   ø€ Ü‰ÑÔØ×+Ñ+ˆŒÜ0°Ó8ˆŒÜŸ<™<¨¯©¸F×<QÑ<QÔRˆÔÜ$ VÓ,ˆŒÜŸ<™<¨¯©¸F×<QÑ<QÔRˆÕr/   r$   r�   r§   rR   c                 óÌ   — |}| j                  |«      }| j                  |||¬«      \  }}||z   }|}| j                  |«      }| j                  |«      }||z   }|f}|r||fz  }|S )a=  
        Args:
            hidden_states (`torch.FloatTensor`):
                Input to the layer of shape `(batch, seq_len, embed_dim)`.
            attention_mask (`torch.FloatTensor`):
                Attention mask of shape `(batch, 1, q_len, k_v_seq_len)` where padding elements are indicated by very large negative values.
            output_attentions (`bool`, *optional*, defaults to `False`):
                Whether or not to return the attentions tensors of all attention layers. See `attentions` under
                returned tensors for more detail.
        )r$   r�   r§   )ré   rè   rê   rà   )rN   r$   r�   r§   rä   r“   Úoutputss          r0   rx   zIdefics2EncoderLayer.forwardg  s�   € ð  !ˆà×(Ñ(¨Ó7ˆØ&*§n¡nØ'Ø)Ø/ð '5ó '
Ñ#ˆ�|ð
 ! =Ñ0ˆà ˆØ×(Ñ(¨Ó7ˆØŸ™ Ó/ˆØ  =Ñ0ˆà Ð"ˆáØ˜�Ñ&ˆGàˆr/   )F)r'   r(   r)   r   r@   r+   rz   r   r¸   r   r,   rx   r{   r|   s   @r0   ræ   ræ   ]  sY   ø„ ðSÐ3õ Sð -2ñ	$à—|‘|ð$ð Ÿ™ð$ð $ D™>ð	$ð
 
ˆu× Ñ Ñ	!÷$r/   ræ   c                   ó„   ‡ — e Zd ZdZdefˆ fd„Z	 	 	 	 d
deej                     dee	   dee	   dee	   de
eef   f
d	„Zˆ xZS )ÚIdefics2Encoderzµ
    Transformer encoder consisting of `config.num_hidden_layers` self attention layers. Each layer is a
    [`Idefics2EncoderLayer`].

    Args:
        config: Idefics2Config
    r7   c                 óÐ   •— t         ‰| �  «        || _        t        j                  t        |j                  «      D �cg c]  }t        |«      ‘Œ c}«      | _        d| _	        y c c}w r·   )
r?   r@   r7   r	   Ú
ModuleListÚrangeÚnum_hidden_layersræ   ÚlayersÚgradient_checkpointing)rN   r7   rf   rO   s      €r0   r@   zIdefics2Encoder.__init__˜  sP   ø€ Ü‰ÑÔØˆŒÜ—m‘mÌ5ÐQW×QiÑQiÓKjÖ$kÀaÔ%9¸&Õ%AÒ$kÓlˆŒØ&+ˆÕ#ùò %ls   ½A#r�   r§   Úoutput_hidden_statesÚreturn_dictrR   c                 óì  — |�|n| j                   j                  }|�|n| j                   j                  }|�|n| j                   j                  }|rdnd}|rdnd}|}| j                  D ]]  }	|r||fz   }| j
                  r+| j                  r| j                  |	j                  |||«      }
n |	|||¬«      }
|
d   }|sŒU||
d   fz   }Œ_ |r||fz   }|st        d„ |||fD «       «      S t        |||¬«      S )ad  
        Args:
            inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
                Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
                This is useful if you want more control over how to convert `input_ids` indices into associated vectors
                than the model's internal embedding lookup matrix.
            attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
                Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:

                - 1 for tokens that are **not masked**,
                - 0 for tokens that are **masked**.

                [What are attention masks?](../glossary#attention-mask)
            output_attentions (`bool`, *optional*):
                Whether or not to return the attentions tensors of all attention layers. See `attentions` under
                returned tensors for more detail.
            output_hidden_states (`bool`, *optional*):
                Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors
                for more detail.
            return_dict (`bool`, *optional*):
                Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
        Nr.   )r§   r   r   c              3   ó&   K  — | ]	  }|€Œ|–— Œ y ­wr¼   r.   ©Ú.0Úvs     r0   ú	<genexpr>z*Idefics2Encoder.forward.<locals>.<genexpr>á  s   è ø€ Òe˜qÐWXÑWdœÑeùó   ‚Š©r"   r$   r%   )r7   r§   rõ   Úuse_return_dictró   rô   rŠ   Ú_gradient_checkpointing_funcÚ__call__Útupler   )rN   Úinputs_embedsr�   r§   rõ   rö   Úencoder_statesÚall_attentionsr$   Úencoder_layerÚlayer_outputss              r0   rx   zIdefics2Encoder.forwardŸ  s7  € ð< 2CÐ1NÑ-ÐTX×T_ÑT_×TqÑTqÐà$8Ð$DÑ È$Ï+É+×JjÑJjð 	ð &1Ð%<‘kÀ$Ç+Á+×B]ÑB]ˆá3™¸ˆÙ0™°dˆà%ˆØ!Ÿ[™[ò 	FˆMÙ#Ø!/°=Ð2BÑ!B�Ø×*Ò*¨t¯}ª}Ø $× AÑ AØ!×*Ñ*Ø!Ø"Ø%ó	!‘ñ !.Ø!Ø"Ø&7ô!�ð *¨!Ñ,ˆMâ Ø!/°=ÀÑ3CÐ2EÑ!E‘ð)	Fñ,  Ø+¨}Ð.>Ñ>ˆNáÜÑe ]°NÀNÐ$SÔeÓeÐeÜØ+¸>ÐVdô
ð 	
r/   ©NNNN)r'   r(   r)   r*   r   r@   r   r+   rz   r¸   r   r   r   rx   r{   r|   s   @r0   rî   rî   �  s~   ø„ ñð,˜~õ ,ð 26Ø,0Ø/3Ø&*ñE
ð ! §¡Ñ.ðE
ð $ D™>ð	E
ð
 ' t™nðE
ð ˜d‘^ðE
ð 
ˆu�oÐ%Ñ	&÷E
r/   rî   ai  
    This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
    library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
    etc.)

    This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
    Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
    and behavior.

    Parameters:
        config ([`Idefics2Config`] or [`Idefics2VisionConfig`]):
            Model configuration class with all the parameters of the model. Initializing with a config file does not
            load the weights associated with the model, only the configuration. Check out the
            [`~PreTrainedModel.from_pretrained`] method to load the model weights.
zVThe bare Idefics2 Model outputting raw hidden-states without any specific head on top.c                   ó:   — e Zd ZeZdZdZg d¢ZdZdZ	dZ
dZdZd„ Zy)ÚIdefics2PreTrainedModelÚmodelT)r˜   rÄ   ÚIdefics2PerceiverLayerÚIdefics2DecoderLayerr#   c                 ó  — t        | j                  d«      r| j                  j                  n#| j                  j                  «       j                  }t        |d«      r'|j                  j
                  j                  d|¬«       t        |t        j                  t        j                  f«      rY|j                  j
                  j                  d|¬«       |j                  �%|j                  j
                  j                  «        y y t        |t        j                  «      rf|j                  j
                  j                  d|¬«       |j                  �2|j                  j
                  |j                     j                  «        y y y )NÚinitializer_rangeÚclass_embeddingr¬   )ÚmeanÚstd)r‹   r7   r  Úget_text_configr  ÚdataÚnormal_Ú
isinstancer	   r¡   rE   rc   rÉ   Úzero_rK   Úpadding_idx)rN   r}   r  s      r0   Ú_init_weightsz%Idefics2PreTrainedModel._init_weights  s$  € ô �t—{‘{Ð$7Ô8ð �K‰K×)Ò)à—‘×,Ñ,Ó.×@Ñ@ð 	ô �6Ð,Ô-Ø×"Ñ"×'Ñ'×/Ñ/°S¸cÐ/ÔBä�fœrŸy™y¬"¯)©)Ð4Ô5Ø�M‰M×Ñ×&Ñ&¨C°SÐ&Ô9Ø�{‰{Ð&Ø—‘× Ñ ×&Ñ&Õ(ð 'ä˜¤§¡Ô-Ø�M‰M×Ñ×&Ñ&¨C°SÐ&Ô9Ø×!Ñ!Ð-Ø—‘×"Ñ" 6×#5Ñ#5Ñ6×<Ñ<Õ>ð .ð .r/   N)r'   r(   r)   r   Úconfig_classÚbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_skip_keys_device_placementÚ_supports_flash_attn_2Ú_supports_sdpaÚ_supports_flex_attnÚ_supports_cache_classr  r.   r/   r0   r
  r
  ø  s=   „ ð
 "€LØÐØ&*Ð#ÚtÐØ"3ÐØ!ÐØ€NØÐØ Ðó?r/   r
  a<  
    Args:
        pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)):
            The tensors corresponding to the input images. Pixel values can be obtained using
            [`AutoImageProcessor`]. See [`CLIPImageProcessor.__call__`] for details ([]`LlavaProcessor`] uses
            [`CLIPImageProcessor`] for processing images).
        pixel_attention_mask (`torch.Tensor` of shape `(batch_size, image_size, image_size)`, *optional*):
            Mask to avoid performing attention on padding pixel indices.
        output_attentions (`bool`, *optional*):
            Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
            tensors for more detail.
        output_hidden_states (`bool`, *optional*):
            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
            more detail.
        return_dict (`bool`, *optional*):
            Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
zAIdefics2 vision encoder model that returnss raw image embeddings.c                   óœ   ‡ — e Zd ZeZdZdZdZdefˆ fd„Zd„ Z	d„ Z
	 	 	 	 ddeej                     dee   dee   d	ee   d
eeef   f
d„Zˆ xZS )ÚIdefics2VisionTransformerTr7   c                 óþ   •— t         ‰| �  |«       |j                  }|| _        t	        |«      | _        t        |«      | _        t        j                  ||j                  ¬«      | _        |j                  dk(  | _        y ©NrÖ   Úflash_attention_2)r?   r@   rA   r7   r6   rj   rî   Úencoderr	   rÝ   rÞ   Úpost_layernormr®   Ú_use_flash_attention_2)rN   r7   rB   rO   s      €r0   r@   z"Idefics2VisionTransformer.__init__8  sg   ø€ Ü‰Ñ˜Ô Ø×&Ñ&ˆ	àˆŒÜ2°6Ó:ˆŒÜ& vÓ.ˆŒÜ Ÿl™l¨9¸&×:OÑ:OÔPˆÔØ&,×&AÑ&AÐEXÑ&XˆÕ#r/   c                 ó   — | j                   S r¼   ©rj   ©rN   s    r0   Úget_input_embeddingsz.Idefics2VisionTransformer.get_input_embeddingsB  s   € Ø�‰Ðr/   c                 ó   — || _         y r¼   r,  ©rN   r€   s     r0   Úset_input_embeddingsz.Idefics2VisionTransformer.set_input_embeddingsE  s	   € Øˆ�r/   rQ   r§   rõ   rö   rR   c                 ó0  — |�|n| j                   j                  }|�|n| j                   j                  }|�|n| j                   j                  }|j	                  d«      }|€}| j                   j
                  }t        j                  ||j	                  d«      |z  |j	                  d«      |z  f«      }|j                  t        j                  |j                  ¬«      }| j                  ||¬«      }|j                  |d«      }t        j                  | «      sd }n"| j                  st        ||j                   «      }| j#                  |||||¬«      }	|	d   }
| j%                  |
«      }
|s	|
f|	dd  z   S t'        |
|	j(                  |	j*                  ¬	«      S )
Nr   r   r   ©rˆ   rd   ©rP   rQ   rW   )r  r�   r§   rõ   rö   r   rþ   )r7   r§   rõ   rÿ   rT   rD   r+   Úonesrb   r¸   rd   rj   r`   Úanyr*  r   rˆ   r(  r)  r   r$   r%   )rN   rP   rQ   r§   rõ   rö   re   rD   r$   Úencoder_outputsr"   s              r0   rx   z!Idefics2VisionTransformer.forwardH  s®  € ð 2CÐ1NÑ-ÐTX×T_ÑT_×TqÑTqÐà$8Ð$DÑ È$Ï+É+×JjÑJjð 	ð &1Ð%<‘kÀ$Ç+Á+×B]ÑB]ˆà!×&Ñ& qÓ)ˆ
ØÐ'ØŸ™×/Ñ/ˆJÜ#(§:¡:àØ ×%Ñ% aÓ(¨JÑ6Ø ×%Ñ% aÓ(¨JÑ6ðó$Ð ð $8×#:Ñ#:ÄÇÁÐT`×TgÑTgÐ#:Ó#hÐ àŸ™°\ÐXl˜Ómˆà3×8Ñ8¸ÀRÓHÐô �y‰yÐ.Ð.Ô/Ø#'Ñ Ø×,Ò,Ü#=Ð>RÐTa×TgÑTgÓ#hÐ àŸ,™,Ø'Ø/Ø/Ø!5Ø#ð 'ó 
ˆð ,¨AÑ.ÐØ ×/Ñ/Ð0AÓBÐáØ%Ð'¨/¸!¸"Ð*=Ñ=Ð=äØ/Ø)×7Ñ7Ø&×1Ñ1ô
ð 	
r/   r  )r'   r(   r)   r   r  r   Ú_supports_flash_attention_2r!  r@   r.  r1  r   r+   ry   r¸   r   r   r   rx   r{   r|   s   @r0   r$  r$  .  s›   ø„ ð
 (€LØ€NØ"&ÐØÐðYÐ3õ Yòò ð <@Ø,0Ø/3Ø&*ñ7
ð ' u×'7Ñ'7Ñ8ð7
ð $ D™>ð	7
ð
 ' t™nð7
ð ˜d‘^ð7
ð 
ˆu�oÐ%Ñ	&÷7
r/   r$  r$   Ún_reprR   c                 óª   — | j                   \  }}}}|dk(  r| S | dd…dd…ddd…dd…f   j                  |||||«      } | j                  |||z  ||«      S )zÔ
    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
    r   N)rX   Úexpandr±   )r$   r9  ÚbatchÚnum_key_value_headsÚslenr�   s         r0   rŒ   rŒ   ƒ  so   € ð
 2?×1DÑ1DÑ.€EÐ  hØ�‚zØÐØ!¢!¢Q¨ªa²Ð"2Ñ3×:Ñ:¸5ÐBUÐW\Ð^bÐdlÓm€MØ× Ñ  Ð(;¸eÑ(CÀTÈ8ÓTÐTr/   c                   ó,   ‡ — e Zd Zdˆ fd„	Zd„ Zd„ Zˆ xZS )ÚIdefics2RMSNormc                 óŠ   •— t         ‰| �  «        t        j                  t	        j
                  |«      «      | _        || _        y)z>
        Idefics2RMSNorm is equivalent to T5LayerNorm
        N)r?   r@   r	   rØ   r+   r5  rc   Úvariance_epsilon)rN   rA   r×   rO   s      €r0   r@   zIdefics2RMSNorm.__init__‘  s1   ø€ ô 	‰ÑÔÜ—l‘l¤5§:¡:¨kÓ#:Ó;ˆŒØ #ˆÕr/   c                 ó"  — |j                   }|j                  t        j                  «      }|j	                  d«      j                  dd¬«      }|t        j                  || j                  z   «      z  }| j                  |j                  |«      z  S )Nr   rW   T)Úkeepdim)	rˆ   rb   r+   r�   Úpowr  ÚrsqrtrB  rc   )rN   r$   Úinput_dtypeÚvariances       r0   rx   zIdefics2RMSNorm.forward™  sy   € Ø#×)Ñ)ˆØ%×(Ñ(¬¯©Ó7ˆØ ×$Ñ$ QÓ'×,Ñ,¨R¸Ð,Ó>ˆØ%¬¯©°H¸t×?TÑ?TÑ4TÓ(UÑUˆØ�{‰{˜]×-Ñ-¨kÓ:Ñ:Ð:r/   c                 ó^   — t        | j                  j                  «      › d| j                  › �S )Nz, eps=)r  rc   rX   rB  r-  s    r0   Ú
extra_reprzIdefics2RMSNorm.extra_repr   s*   € Ü˜Ÿ™×)Ñ)Ó*Ð+¨6°$×2GÑ2GÐ1HÐIÐIr/   )g�íµ ÷Æ°>)r'   r(   r)   r@   rx   rJ  r{   r|   s   @r0   r@  r@  �  s   ø„ õ$ò;öJr/   r@  c                   ó:  ‡ — e Zd Zddee   ddfˆ fd„Z	 	 	 	 	 ddej                  dej                  deej                     deej                     d	ee	ej                        d
e
de
de	ej                  eej                     ee	ej                        f   fd„Zˆ xZS )ÚIdefics2PerceiverAttentionNÚ	layer_idxrR   c                 óJ  •— t         ‰| �  «        || _        d| _        |j                  | _        |j
                  | _        |j                  | _        |j                  | _	        | j                  | j                  z  | _
        |j                  | _        | j                  dz  | _        t        j                  | j                  | j                  | j                  z  d¬«      | _        t        j                  | j                  | j                  | j                  z  d¬«      | _        t        j                  | j                  | j                  | j                  z  d¬«      | _        t        j                  | j                  | j                  z  | j                  d¬«      | _        d| _        y)ziPerceiver Cross-Attention Module --> let long-form inputs be `context`, resampled embeddings be `latents`Nrš   FrÈ   )r?   r@   r7   rM  rA   Úresampler_n_headsrœ   Úresampler_head_dimr�   r=  r…   r    r‚   r	   r¡   r¤   r¢   r£   Úo_projr¦   ©rN   r7   rM  rO   s      €r0   r@   z#Idefics2PerceiverAttention.__init__¥  s2  ø€ ä‰ÑÔØˆŒØˆŒØ!×-Ñ-ˆÔØ×1Ñ1ˆŒØ×1Ñ1ˆŒØ#)×#=Ñ#=ˆÔ Ø$(§N¡N°d×6NÑ6NÑ$NˆÔ!Ø!'×!9Ñ!9ˆÔØ—}‘} dÑ*ˆŒä—i‘i × 0Ñ 0°$·.±.À4Ç=Á=Ñ2PÐW\Ô]ˆŒÜ—i‘i × 0Ñ 0°$×2JÑ2JÈTÏ]É]Ñ2ZÐafÔgˆŒÜ—i‘i × 0Ñ 0°$×2JÑ2JÈTÏ]É]Ñ2ZÐafÔgˆŒÜ—i‘i §¡°·±Ñ >À×@PÑ@PÐW\Ô]ˆŒàˆ�r/   ÚlatentsÚcontextr�   rn   Úpast_key_valuer§   Ú	use_cachec           
      ó|  — |j                  «       \  }}	}
|	|j                  «       d   z   }t        j                  ||gd¬«      }| j                  |«      }| j	                  |«      }| j                  |«      }|j                  ||	| j                  | j                  «      j                  dd«      }|j                  ||| j                  | j                  «      j                  dd«      }|j                  ||| j                  | j                  «      j                  dd«      }t        | d|«      }|� |j                  ||| j                  «      \  }}t        }| j                  j                   dk7  rN| j                  j                   dk(  r|rt"        j%                  d	«       nt&        | j                  j                      } || ||||| j(                  | j*                  | j,                  sd
n| j.                  ¬«      \  }}|j1                  ||	| j                  | j                  z  «      }| j3                  |«      }|sd}|||fS )aÚ  
        Runs Perceiver Self-Attention, with special (context, latents) appended along the `seq` dimension!

        Args:
            latents (`torch.Tensor`): Tensor of shape [bsz, n_latents, embed_dim] representing fixed length latents to compress to.
            context (`torch.Tensor`): Tensor of shape [bsz, seq, embed_dim] representing long-form context to resample.
            attention_mask (`torch.Tensor`, *optional*): Tensor of shape [bsz, 1, seq, n_latents] representing attention mask.
            position_ids (`torch.LongTensor`, *optional*): Tensor of shape [bsz, seq] representing position indices of each input token.
            past_key_value (`Tuple[torch.Tensor]`, *optional*): Tuple of tensors containing cached key and value states.
            output_attentions (`bool`, *optional*, defaults to `False`): Whether to return attention weights.
            use_cache (`bool`, *optional*, defaults to `False`): Whether to use past_key_value for caching.
        r   r†   ©r‡   r   rU  Nr©   rª   r«   r¬   r­   )rT   r+   Úconcatr¤   r¢   r£   r`   rœ   r�   rZ   r=  ÚgetattrÚupdaterM  r–   r7   r®   r¯   r°   r   r¦   r‚   rŠ   r    r±   rQ  )rN   rS  rT  r�   rn   rU  r§   rV  ÚbszÚq_lenrf   Ú
kv_seq_lenr$   r³   r´   rµ   r¶   r•   r“   s                      r0   rx   z"Idefics2PerceiverAttention.forward¹  sú  € ð,  Ÿ™›‰ˆˆU�AØ˜WŸ\™\›^¨AÑ.Ñ.ˆ
äŸ™ g¨wÐ%7¸RÔ@ˆà—+‘+˜gÓ&ˆØ�{‰{˜=Ó)ˆØ—‘˜]Ó+ˆà—,‘,˜s E¨4¯>©>¸4¿=¹=ÓI×SÑSÐTUÐWXÓYˆØ�y‰y˜˜j¨$×*BÑ*BÀDÇMÁMÓR×\Ñ\Ð]^Ð`aÓbˆØ—‘˜S *¨d×.FÑ.FÈÏÉÓV×`Ñ`ÐabÐdeÓfˆä  Ð'7¸ÓHˆàÐ%Ø)×0Ñ0°°v¸t¿~¹~ÓN‰LˆD�&ä(?ÐØ�;‰;×+Ñ+¨wÒ6Ø�{‰{×/Ñ/°6Ò9Ñ>OÜ×#Ñ#ðLõô
 '>¸d¿k¹k×>^Ñ>^Ñ&_Ð#á$7ØØØØØØ—n‘nØ—L‘LØ#Ÿ}š}‘C°$×2HÑ2Hô	%
Ñ!ˆ�\ð "×)Ñ)¨#¨u°d·n±nÀtÇ}Á}Ñ6TÓUˆØ—k‘k +Ó.ˆá ØˆLà˜L¨.Ð8Ð8r/   r¼   ©NNNFF)r'   r(   r)   r   rÐ   r@   r+   rz   Ú
LongTensorr   r¸   rx   r{   r|   s   @r0   rL  rL  ¤  sØ   ø„ ñ¨(°3©-ð À4õ ð0 26Ø37Ø8<Ø"'ØñC9à—‘ðC9ð —‘ðC9ð ! §¡Ñ.ð	C9ð
 ˜u×/Ñ/Ñ0ðC9ð !  u§|¡|Ñ!4Ñ5ðC9ð  ðC9ð ðC9ð 
ˆu�|‰|˜X e§l¡lÑ3°X¸eÀEÇLÁLÑ>QÑ5RÐRÑ	S÷C9r/   rL  c                   ó6  ‡ — e Zd Zdefˆ fd„Z	 	 	 	 	 ddej                  dej                  deej                     deej                     dee	ej                        dee
   d	ee
   d
e	ej                  ee	ej                  ej                  f      f   fd„Zˆ xZS )r  rM  c                 ó0  •— t         ‰| �  «        |j                  | _        |j                  | _        |j
                  | _        |j                  | _        t        | j                  | j                  ¬«      | _	        t        | j                  | j                  ¬«      | _
        t        ||¬«      | _        t        | j                  | j                  ¬«      | _        t        |j                  |j                  dz  |j                  |j                  ¬«      | _        y )NrÖ   )rM  é   ©rA   r¿   rÅ   r½   )r?   r@   rA   Úresampler_n_latentsÚ	n_latentsÚresampler_depthÚdepthÚrms_norm_epsr@  Úinput_latents_normÚinput_context_normrL  rè   Úpost_attention_layernormrÄ   r½   rà   rR  s      €r0   r@   zIdefics2PerceiverLayer.__init__   sØ   ø€ Ü‰ÑÔØ!×-Ñ-ˆÔØ×3Ñ3ˆŒØ×+Ñ+ˆŒ
Ø"×/Ñ/ˆÔä"1°$×2BÑ2BÈ×HYÑHYÔ"ZˆÔÜ"1°$×2BÑ2BÈ×HYÑHYÔ"ZˆÔÜ3°FÀiÔPˆŒÜ(7¸×8HÑ8HÈd×N_ÑN_Ô(`ˆÔ%ÜØ×*Ñ*Ø$×0Ñ0°1Ñ4Ø×*Ñ*Ø×(Ñ(ô	
ˆ�r/   rS  rT  r�   rn   rU  r§   rV  rR   c                 ó   — |}	| j                  |«      }| j                  |«      }| j                  |||¬«      \  }}
}|	|z   }|}	| j                  |«      }| j	                  |«      }|	|z   }|f}|r||
fz  }|r||fz  }|S )aŸ  
        Args:
            latents (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)`
            context (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)`
            attention_mask (`torch.FloatTensor`, *optional*): attention mask of size
                `(batch, sequence_length)` where padding elements are indicated by 0.
            output_attentions (`bool`, *optional*):
                Whether or not to return the attentions tensors of all attention layers. See `attentions` under
                returned tensors for more detail.
            use_cache (`bool`, *optional*):
                If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding
                (see `past_key_values`).
            past_key_value (`Tuple(torch.FloatTensor)`, *optional*): cached past key and value projection states
        )rS  rT  r�   )rj  rk  rè   rl  rà   )rN   rS  rT  r�   rn   rU  r§   rV  r’   rä   Úself_attn_weightsÚpresent_key_valuerì   s                r0   rx   zIdefics2PerceiverLayer.forward  s¶   € ð2 ˆà×)Ñ)¨'Ó2ˆØ×)Ñ)¨'Ó2ˆà8<¿¹ØØØ)ð 9Gó 9
Ñ5ˆÐ"Ð$5ð
 ˜WÑ$ˆØˆà×/Ñ/°Ó8ˆØ—(‘(˜7Ó#ˆØ˜WÑ$ˆà�*ˆáØÐ)Ð+Ñ+ˆGáØÐ)Ð+Ñ+ˆGàˆr/   r_  )r'   r(   r)   rÐ   r@   r+   rz   r   r`  r   r¸   r,   rx   r{   r|   s   @r0   r  r  ÿ  sÐ   ø„ ð
¨#õ 
ð, 26Ø37Ø8<Ø,1Ø$)ñ2à—‘ð2ð —‘ð2ð ! §¡Ñ.ð	2ð
 ˜u×/Ñ/Ñ0ð2ð !  u§|¡|Ñ!4Ñ5ð2ð $ D™>ð2ð ˜D‘>ð2ð 
ˆu× Ñ  (¨5°×1BÑ1BÀE×DUÑDUÐ1UÑ+VÑ"WÐWÑ	X÷2r/   r  a3  
    Args:
        context (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_dim)`):
            The hidden states of the image after vision encoder and modality projection.
        attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
            Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:

            - 1 for tokens that are **not masked**,
            - 0 for tokens that are **masked**.

            [What are attention masks?](../glossary#attention-mask)
z`Idefics2 perceiver resampler model that performs `depth` blocks of cross-attention with a fixed zn`n_latents` inputs to decrease embedding sequence length. The Resampler acts as a form of learned pooling and zjis derived from [Perceiver: General Perception with Iterative Attention](https://arxiv.org/abs/2103.03206)c                   ó€   ‡ — e Zd ZeZdZdZdZdˆ fd„Zde	j                  de	j                  de	j                  fd„Zˆ xZS )ÚIdefics2PerceiverResamplerTrR   c           	      óZ  •— t         ‰| �  |«       |j                  | _        |j                  | _        |j                  | _        |j                  | _        |j                  | _        t        j                  t        j                  | j
                  | j                  «      «      | _        t        j                  t        | j                  «      D �cg c]  }t!        ||«      ‘Œ c}«      | _        t%        | j                  | j                  ¬«      | _        |j(                  dk(  | _        y c c}w r&  )r?   r@   rA   r½   re  rf  rg  rh  ri  r	   rØ   r+   r5  rS  rð   rñ   r  ró   r@  Únormr®   r*  )rN   r7   ÚidxrO   s      €r0   r@   z#Idefics2PerceiverResampler.__init__a  sà   ø€ Ü‰Ñ˜Ô Ø!×-Ñ-ˆÔØ ×+Ñ+ˆŒØ×3Ñ3ˆŒØ×+Ñ+ˆŒ
Ø"×/Ñ/ˆÔô —|‘|¤E§J¡J¨t¯~©~¸t×?OÑ?OÓ$PÓQˆŒô —m‘mÔTYÐZ^×ZdÑZdÓTeÖ$fÈSÔ%;¸FÀCÕ%HÒ$fÓgˆŒÜ# D×$4Ñ$4¸$×:KÑ:KÔLˆŒ	à&,×&AÑ&AÐEXÑ&XˆÕ#ùò %gs   ÃD(rT  r�   c           
      óF  — | j                   j                  d«      j                  |j                  d   g| j                   j	                  «       ¢­«      }t        j                  |j	                  d«      |j	                  d«      f|j                  |j                  ¬«      }t        j                  ||gd¬«      }| j                  s"t        ||j                  | j                  ¬«      n|}|}| j                  D ]  } ||||d d dd¬«      }|d   }Œ | j                  |«      }|S )	Nr   r   r3  rW   rX  )Útgt_lenF)r�   rn   rU  r§   rV  )rS  Ú	unsqueezer;  rX   rT   r+   r5  rˆ   rd   Úcatr*  r   rf  ró   rs  )rN   rT  r�   rS  Úlatent_attention_maskÚcompressed_contextÚperceiver_layerr  s           r0   rx   z"Idefics2PerceiverResampler.forwardr  s  € ð —,‘,×(Ñ(¨Ó+×2Ñ2°G·M±MÀ!Ñ4DÐ3[ÀtÇ|Á|×GXÑGXÓGZÑ3[Ó\ˆä %§
¡
Ø× Ñ  Ó# W§\¡\°!£_Ð5¸^×=QÑ=QÐZh×ZoÑZoô!
Ðô Ÿ™ NÐ4IÐ#JÐPRÔSˆð ×.Ò.ô ' ~°w·}±}ÈdÏnÉnÕ]àð 	ð %ÐØ#Ÿ{™{ò 	2ˆOÙ+Ø"ØØ-Ø!Ø#Ø"'ØôˆMð "/¨qÑ!1Ñð	2ð "ŸY™YÐ'9Ó:Ðà!Ð!r/   )rR   N)r'   r(   r)   r   r  r   r8  r!  r@   r+   rz   rx   r{   r|   s   @r0   rq  rq  U  sN   ø„ ð +€LØ€NØ"&ÐØÐõYð"""à—‘ð""ð Ÿ™ð""ð 
�‰÷	""r/   rq  c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚIdefics2Connectorc                 ó2  •— t         ‰| �  «        t        |j                  j                  |j
                  j                  |j
                  j                  |j
                  j                  ¬«      | _        t        j                  |j                  «      | _        y )Nrd  )r?   r@   rÄ   Úvision_configrA   Útext_configr¿   r½   Úmodality_projectionrq  Ú_from_configÚperceiver_configÚperceiver_resamplerrM   s     €r0   r@   zIdefics2Connector.__init__˜  sr   ø€ Ü‰ÑÔÜ#.Ø×,Ñ,×8Ñ8Ø$×0Ñ0×BÑBØ×*Ñ*×6Ñ6Ø×)Ñ)×4Ñ4ô	$
ˆÔ ô $>×#JÑ#JÈ6×KbÑKbÓ#cˆÕ r/   c                 óN   — | j                  |«      }| j                  ||¬«      }|S )N)rT  r�   )r�  r„  )rN   r&   r�   s      r0   rx   zIdefics2Connector.forward¢  s2   € Ø"×6Ñ6Ð7JÓKÐØ"×6Ñ6Ð?RÐcqÐ6ÓrÐØ"Ð"r/   )r'   r(   r)   r@   rx   r{   r|   s   @r0   r}  r}  —  s   ø„ ôdö#r/   r}  a´  
    Args:
        input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
            Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide
            it.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are input IDs?](../glossary#input-ids)
        attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
            Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:

            - 1 for tokens that are **not masked**,
            - 0 for tokens that are **masked**.

            [What are attention masks?](../glossary#attention-mask)

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            If `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see
            `past_key_values`).

            If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`]
            and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more
            information on the default strategy.

            - 1 indicates the head is **not masked**,
            - 0 indicates the head is **masked**.
        position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
            config.n_positions - 1]`. [What are position IDs?](../glossary#position-ids)
        past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
            Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape
            `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape
            `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`.

            Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention
            blocks) that can be used (see `past_key_values` input) to speed up sequential decoding.

            If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that
            don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all
            `decoder_input_ids` of shape `(batch_size, sequence_length)`.
        inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
            Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
            is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
            model's internal embedding lookup matrix.
        pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)):
            The tensors corresponding to the input images. Pixel values can be obtained using
            [`AutoImageProcessor`]. See [`CLIPImageProcessor.__call__`] for details ([]`LlavaProcessor`] uses
            [`CLIPImageProcessor`] for processing images).
        pixel_attention_mask (`torch.Tensor` of shape `(batch_size, image_size, image_size)`, *optional*):
            Mask to avoid performing attention on padding pixel indices.
        image_hidden_states (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`):
            The hidden states of the image encoder after modality projection and perceiver resampling.
        use_cache (`bool`, *optional*):
            If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see
            `past_key_values`).
        output_attentions (`bool`, *optional*):
            Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
            tensors for more detail.
        output_hidden_states (`bool`, *optional*):
            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
            more detail.
        return_dict (`bool`, *optional*):
            Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
        cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*):
            Indices depicting the position of the input sequence tokens in the sequence. Contrarily to `position_ids`,
            this tensor is not affected by padding. It is used to update the cache in the correct position and to infer
            the complete sequence length.
zQIdefics2 model consisting of a SIGLIP vision encoder and Mistral language decoderc                    ó  ‡ — e Zd Zdefˆ fd„Zd„ Zd„ Zd„ Zd„ Zde	j                  dee	j                     d	ee	j                     fd
„Z ede«      	 	 	 	 	 	 	 	 	 	 	 	 	 ddee	j                     dee	j                     dee	j                     deee	j"                        dee	j"                     dee	j"                     dee	j$                     d	ee	j"                     dee   dee   dee   dee	j                     dee   deeef   fd„«       Zˆ xZS )ÚIdefics2Modelr7   c                 ó0  •— t         ‰| �  |«       | j                  j                  j                  | _        | j                  j                  j                  | _        t        j                  |j                  «      | _
        t        |«      | _        t        j                  |j                  «      | _        |j                   j"                  | _        | j                  j&                  | _        |j                  j(                  dk(  | _        | j-                  «        y )Nr'  )r?   r@   r7   r€  Úpad_token_idr  Ú
vocab_sizer$  r‚  r  Úvision_modelr}  Ú	connectorr   Úfrom_configÚ
text_modelrƒ  re  Úimage_seq_lenÚimage_token_idr®   r*  Ú	post_initrM   s     €r0   r@   zIdefics2Model.__init__÷  sÆ   ø€ Ü‰Ñ˜Ô ØŸ;™;×2Ñ2×?Ñ?ˆÔØŸ+™+×1Ñ1×<Ñ<ˆŒä5×BÑBÀ6×CWÑCWÓXˆÔÜ*¨6Ó2ˆŒÜ#×/Ñ/°×0BÑ0BÓCˆŒà#×4Ñ4×HÑHˆÔØ"Ÿk™k×8Ñ8ˆÔà&,×&8Ñ&8×&MÑ&MÐQdÑ&dˆÔ#à�‰Õr/   c                 óª   ‡— ˆfd„Šd„ }| j                  «       j                  |«      | _         ‰| j                  «      j                  |«      | _        y)aE  
        Enables the gradients for the input embeddings.

        This is useful for lora when using gradient checkpointing.
        c.f. https://github.com/huggingface/peft/issues/1402#issuecomment-1913675032

        Override to set output.requires_grad = True for both the decoder's and vision model's embeddings.
        c                 ó–   •— t        t        | j                  «       «      «      dk(  r| S  ‰t        | j                  «       «      d   «      S )Nr   )ÚlenÚlistÚchildren)r}   Úget_lowest_modules    €r0   r—  zCIdefics2Model.enable_input_require_grads.<locals>.get_lowest_module  s?   ø€ Ü”4˜Ÿ™Ó)Ó*Ó+¨qÒ0à�ñ )¬¨f¯o©oÓ.?Ó)@ÀÑ)CÓDÐDr/   c                 ó&   — |j                  d«       y ©NT©Úrequires_grad_©r}   ÚinputÚoutputs      r0   Úmake_inputs_require_gradszKIdefics2Model.enable_input_require_grads.<locals>.make_inputs_require_grads  ó   € Ø×!Ñ! $Õ'r/   N)r.  Úregister_forward_hookÚ_text_require_grads_hookr‹  Ú_vision_require_grads_hook)rN   rŸ  r—  s     @r0   Úenable_input_require_gradsz(Idefics2Model.enable_input_require_grads  sN   ø€ ô	Eò	(ð )-×(AÑ(AÓ(C×(YÑ(YÐZsÓ(tˆÔ%Ù*;¸D×<MÑ<MÓ*N×*dÑ*dØ%ó+
ˆÕ'r/   c                 ól   — | j                   j                  «        | j                  j                  «        y r¼   ©r¢  Úremover£  r-  s    r0   Údisable_input_require_gradsz)Idefics2Model.disable_input_require_grads!  ó&   € Ø×%Ñ%×,Ñ,Ô.Ø×'Ñ'×.Ñ.Õ0r/   c                 ó6   — | j                   j                  «       S r¼   )rŽ  r.  r-  s    r0   r.  z"Idefics2Model.get_input_embeddings%  s   € Ø�‰×3Ñ3Ó5Ð5r/   c                 ó:   — | j                   j                  |«       y r¼   )rŽ  r1  r0  s     r0   r1  z"Idefics2Model.set_input_embeddings(  s   € Ø�‰×,Ñ,¨UÕ3r/   Ú	input_idsr  r&   c                 óÄ   — |j                   \  }}}|| j                  k(  }|j                  «       }|j                  d|«      }	|	j	                  |j
                  «      ||<   |S )aq  
        This method aims at merging the token embeddings with the image hidden states into one single sequence of vectors that are fed to the transformer LM.
        The merging happens as follows:
        - The text token sequence is: `tok_1 tok_2 tok_3 <fake_token_around_image> <image> <image> ... <image> <fake_token_around_image> tok_4`.
        - We get the image hidden states for the image through the vision encoder (and potentially the perceiver), and that hidden state is then projected into the text embedding space.
        We thus have a sequence of image hidden states of size (1, image_seq_len, hidden_dim), where 1 is for batch_size of 1 image and hidden_dim is the hidden_dim of the LM transformer.
        - The merging happens so that we obtain the following sequence: `vector_tok_1 vector_tok_2 vector_tok_3 vector_fake_tok_around_image {sequence of image_seq_len image hidden states} vector_fake_toke_around_image vector_tok_4`. That sequence is fed to the LM.
        - To fit the format of that sequence, `input_ids`, `input_embeds`, `attention_mask` are all 3 adapted to insert the image hidden states.
        rW   )rX   r�  Úcloner`   rb   rd   )
rN   r¬  r  r&   Ú
num_imagesrf   Úvision_hidden_sizeÚspecial_image_token_maskÚnew_inputs_embedsÚreshaped_image_hidden_statess
             r0   Úinputs_mergerzIdefics2Model.inputs_merger+  sp   € ð -@×,EÑ,EÑ)ˆ
�AÐ)Ø#,°×0CÑ0CÑ#CÐ Ø)×/Ñ/Ó1ÐØ':×'?Ñ'?ÀÐDVÓ'WÐ$Ø6R×6UÑ6UÐVg×VnÑVnÓ6oÐÐ2Ñ3Ø Ð r/   aÙ  
        Inputs fed to the model can have an arbitrary number of images. To account for this, pixel_values fed to
        the model have image padding -> (batch_size, max_num_images, 3, max_heights, max_widths) where
        max_num_images is the maximum number of images among the batch_size samples in the batch.

        Padding images are not needed beyond padding the pixel_values at the entrance of the model.
        For efficiency, we only pass through the vision_model's forward the real images by
        discarding the padding images i.e. pixel_values of size (image_batch_size, 3, height, width) where
        image_batch_size would be 7 when num_images_per_sample=[1, 3, 1, 2] and max_num_images would be 3.
        r�   rn   r#   rP   Úpixel_attention_maskrV  r§   rõ   Úcache_positionrö   rR   c                 ó|  — |
�|
n| j                   j                  }
|�|n| j                   j                  }|	�|	n| j                   j                  }	|�|n| j                   j                  }| j
                  r/| j                  j                  r|	rt        j                  d«       d}	|�|j                  \  }}n|�|j                  \  }}}nt        d«      ‚d}d}|	rYt        |t        «      s9d}|€t        «       }n*t        j                  |«      }t        j                  d«       |j!                  «       }|�|€|dk(  rt        d«      ‚|€  | j                  j#                  «       |«      }|�|�t        d«      ‚|��ð|j                  \  }}}}}|j%                  | j&                  ¬	«      } |j(                  ||z  g|j                  d
d  ¢­Ž }|j                  dd  j+                  «       }|dk(  j-                  d¬«      |k7  }||   j/                  «       }|€at1        j2                  |j5                  d«      |j5                  d
«      |j5                  d«      ft0        j6                  |j8                  ¬«      }n6 |j(                  ||z  g|j                  d
d  ¢­Ž }||   j/                  «       }| j                   j:                  j<                  }|j?                  d||¬«      }|j?                  d
||¬«      }|j-                  d¬«      ||z  k(  j7                  «       }| jA                  ||¬«      jB                  }| jE                  ||j)                  |j5                  d«      d«      ¬«      }n)|�'|j%                  | j&                  |j8                  ¬«      }|dk(  r|�|�| jG                  |||¬«      }| j                  |||||	|
|||¬«	      }|r!|	r|jH                  jK                  «       |_$        |stM        d„ g |¢|‘D «       «      S tO        |jB                  |jH                  |jP                  |jR                  |¬«      S )NzZ`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...Fz5You have to specify either input_ids or inputs_embedsr   TzÿWe detected that you are passing `past_key_values` as a tuple of tuples. This is deprecated and will be removed in v4.47. Please convert your cache or use an appropriate `Cache` class (https://huggingface.co/docs/transformers/kv_cache#legacy-cache-format)zWWhen first calling the model, if input_embeds are passed, input_ids should not be None.zMYou cannot specify both pixel_values and image_hidden_states at the same time)rˆ   r   r   r¬   )rW   r†   éýÿÿÿrX  r   )rT   rˆ   rd   )Ú	dimensionrT   Ústep)rW   r†   r4  rW   )r�   r3  )r¬  r  r&   )	r  r�   rn   r#   rV  r§   rõ   r¶  rö   c              3   ó&   K  — | ]	  }|€Œ|–— Œ y ­wr¼   r.   rù   s     r0   rü   z(Idefics2Model.forward.<locals>.<genexpr>Ñ  s   è ø€ ÒU˜qÀqÁ}œÑUùrý   )r"   r#   r$   r%   r&   )*r7   r§   rõ   rV  rÿ   rŠ   rŽ  rô   r¯   r°   rX   rž   r  r   r   Úfrom_legacy_cacheÚget_seq_lengthr.  rb   rˆ   r`   Únumelr^   r‘   r+   r5  rT   r¸   rd   r  rD   Úunfoldr‹  r"   rŒ  r´  r#   Úto_legacy_cacher  r!   r$   r%   )rN   r¬  r�   rn   r#   r  rP   rµ  r&   rV  r§   rõ   r¶  rö   re   r²   rf   Úpast_seen_tokensÚreturn_legacy_cacher¯  rF   ÚheightÚwidthÚnb_values_per_imageÚreal_images_indsrD   Úpatches_subgridrQ   rì   s                                r0   rx   zIdefics2Model.forwardA  sœ  € ð: 2CÐ1NÑ-ÐTX×T_ÑT_×TqÑTqÐà$8Ð$DÑ È$Ï+É+×JjÑJjð 	ð "+Ð!6‘I¸D¿K¹K×<QÑ<Qˆ	Ø%0Ð%<‘kÀ$Ç+Á+×B]ÑB]ˆà�=Š=˜TŸ_™_×CÒCÉ	Ü×ÑØlôð ˆIð Ð Ø%.§_¡_Ñ"ˆJ™
ØÐ&Ø(5×(;Ñ(;Ñ%ˆJ˜
¡AäÐTÓUÐUàÐà#ÐÙÜ˜o¬uÔ5Ø&*Ð#Ø"Ð*Ü&2£n‘Oä&2×&DÑ&DÀ_Ó&U�OÜ×'Ñ'ðbôð
  /×=Ñ=Ó?ÐàÐ$¨Ð):Ð?OÐSTÒ?TÜÐvÓwÐwàÐ ØB˜DŸO™O×@Ñ@ÓBÀ9ÓMˆMð Ð#Ð(;Ð(GÜÐlÓmÐmØÑ%ØBN×BTÑBTÑ?ˆJ˜
 L°&¸%Ø'Ÿ?™?°·±˜?Ó<ˆLØ,˜<×,Ñ,¨Z¸*Ñ-DÐ^À|×GYÑGYÐZ[ÐZ\ÐG]Ò^ˆLð #/×"4Ñ"4°Q°RÐ"8×">Ñ">Ó"@ÐØ ,°Ñ 3×8Ñ8¸\Ð8ÓJÐNaÑaÐØ'Ð(8Ñ9×DÑDÓFˆLð $Ð+Ü',§z¡zØ&×+Ñ+¨AÓ.°×0AÑ0AÀ!Ó0DÀl×FWÑFWÐXYÓFZÐ[ÜŸ*™*Ø'×.Ñ.ô(Ñ$ð (AÐ';×'@Ñ'@Ø Ñ+ð(Ø.B×.HÑ.HÈÈÐ.Lò(Ð$ð (<Ð<LÑ'M×'XÑ'XÓ'ZÐ$àŸ™×2Ñ2×=Ñ=ˆJØ2×9Ñ9ÀAÈJÐ]gÐ9ÓhˆOØ-×4Ñ4¸qÀzÐXbÐ4ÓcˆOØ$3×$7Ñ$7¸HÐ$7Ó$EÈÐV`ÑI`Ñ$`×#fÑ#fÓ#hÐ ð #'×"3Ñ"3Ø)Ø%9ð #4ó #÷  Ñð  ð #'§.¡.Ø#Ð4H×4MÑ4MÈl×N_ÑN_Ð`aÓNbÐdfÓ4gð #1ó #Ñð !Ð,Ø"5×"8Ñ"8¸t¿z¹zÐR[×RbÑRbÐ"8Ó"cÐà˜qÒ  ]Ð%>ÐCVÐCbð !×.Ñ.Ø#Ø+Ø$7ð /ó ˆMð —/‘/Ø'Ø)Ø%Ø+ØØ/Ø!5Ø)Ø#ð "ó 

ˆñ ¡9Ø&-×&=Ñ&=×&MÑ&MÓ&OˆGÔ#áÜÑUÐ$C gÐ$CÐ/BÐ$CÔUÓUÐUä.Ø%×7Ñ7Ø#×3Ñ3Ø!×/Ñ/Ø×)Ñ)Ø 3ô
ð 	
r/   )NNNNNNNNNNNNN)r'   r(   r)   r   r@   r¤  r¨  r.  r1  r+   r`  r   rz   r´  r   ÚIDEFICS2_INPUTS_DOCSTRINGr   r,   ry   r¸   r   r   r!   rx   r{   r|   s   @r0   r‡  r‡  ò  sÃ  ø„ ð
˜~õ ò 
ò41ò6ò4ð!à×#Ñ#ð!ð   §¡Ñ-ð!ð & e§l¡lÑ3ó	!ñ, +ð		ð 	"óð 15Ø15Ø37Ø=AØ59Ø48Ø;?Ø;?Ø$(Ø,0Ø/3Ø59Ø&*ñK
à˜E×,Ñ,Ñ-ðK
ð ! §¡Ñ.ðK
ð ˜u×/Ñ/Ñ0ð	K
ð
 " $ u×'8Ñ'8Ñ"9Ñ:ðK
ð   × 1Ñ 1Ñ2ðK
ð ˜u×0Ñ0Ñ1ðK
ð ' u×'7Ñ'7Ñ8ðK
ð & e×&7Ñ&7Ñ8ðK
ð ˜D‘>ðK
ð $ D™>ðK
ð ' t™nðK
ð ! ×!1Ñ!1Ñ2ðK
ð ˜d‘^ðK
ð 
ˆuÐ5Ð5Ñ	6òK
óôK
r/   r‡  zThe Idefics2 Model with a language modeling head. It is made up a SigLIP vision encoder, with a language modeling head on top. c            &       ó|  ‡ — e Zd ZdgZˆ fd„Zd„ Zd„ Zd„ Zd„ Zd„ Z	d„ Z
 ed	d
d¬«       ee«       eee¬«      	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d!deej&                     deej(                     deej&                     deeej,                        deej,                     deej,                     deej.                     deej,                     deej&                     dee   dee   dee   dee   deej&                     deeej(                  f   deeef   f d„«       «       «       Z	 	 	 	 	 	 	 	 d"ˆ fd„	Zˆ fd„Zed „ «       Z ˆ xZ!S )#Ú Idefics2ForConditionalGenerationzlm_head.weightc                 óZ  •— t         ‰| �  |«       t        |«      | _        | j                  j
                  | _        t        j                  |j                  j                  |j                  j                  d¬«      | _        |j                  j                  | _
        | j                  «        y rÇ   )r?   r@   r‡  r  r7   r�  r	   r¡   r€  rA   rŠ  Úlm_headr‘  rM   s     €r0   r@   z)Idefics2ForConditionalGeneration.__init__ã  sz   ø€ Ü‰Ñ˜Ô Ü" 6Ó*ˆŒ
Ø"Ÿk™k×8Ñ8ˆÔä—y‘y ×!3Ñ!3×!?Ñ!?À×ASÑAS×A^ÑA^ÐejÔkˆŒØ ×,Ñ,×7Ñ7ˆŒð 	�‰Õr/   c                 óÂ   — d„ }| j                  «       j                  |«      | _        | j                  j                  j                  «       j                  |«      | _        y)zŸ
        Enables the gradients for the input embeddings. This is useful for fine-tuning adapter weights while keeping
        the model weights fixed.
        c                 ó&   — |j                  d«       y r™  rš  rœ  s      r0   rŸ  z^Idefics2ForConditionalGeneration.enable_input_require_grads.<locals>.make_inputs_require_gradsô  r   r/   N)r.  r¡  r¢  r  r‹  r£  )rN   rŸ  s     r0   r¤  z;Idefics2ForConditionalGeneration.enable_input_require_gradsî  sO   € ò	(ð )-×(AÑ(AÓ(C×(YÑ(YÐZsÓ(tˆÔ%Ø*.¯*©*×*AÑ*A×*VÑ*VÓ*X×*nÑ*nØ%ó+
ˆÕ'r/   c                 ól   — | j                   j                  «        | j                  j                  «        y r¼   r¦  r-  s    r0   r¨  z<Idefics2ForConditionalGeneration.disable_input_require_gradsü  r©  r/   c                 óJ   — | j                   j                  j                  «       S r¼   )r  rŽ  r.  r-  s    r0   r.  z5Idefics2ForConditionalGeneration.get_input_embeddings   s   € Ø�z‰z×$Ñ$×9Ñ9Ó;Ð;r/   c                 óN   — | j                   j                  j                  |«       y r¼   )r  rŽ  r1  r0  s     r0   r1  z5Idefics2ForConditionalGeneration.set_input_embeddings  s   € Ø�
‰
×Ñ×2Ñ2°5Õ9r/   c                 ó   — | j                   S r¼   ©rÌ  r-  s    r0   Úget_output_embeddingsz6Idefics2ForConditionalGeneration.get_output_embeddings  s   € Ø�|‰|Ðr/   c                 ó   — || _         y r¼   rÓ  )rN   Únew_embeddingss     r0   Úset_output_embeddingsz6Idefics2ForConditionalGeneration.set_output_embeddings	  s	   € Ø%ˆ�r/   Únum_logits_to_keepz4.50Úlogits_to_keep)ÚversionÚnew_name)Úoutput_typer  r¬  r�   rn   r#   r  rP   rµ  r&   ÚlabelsrV  r§   rõ   rö   r¶  rR   c                 ó"  — |�|n| j                   j                  }|�|n| j                   j                  }|�|n| j                   j                  }| j	                  |||||||||
||||¬«      }|d   }t        |t        «      rt        | d«      n|}| j                  |dd…|dd…f   «      }d}|	��|j                  «       }|	j                  |j                  «      }	|�s|dd…|j                  d   dz
   d…f   j                  |j                  «      }|ddd…dd…f   |dk7     j                  «       }|	ddd…f   |dk7     j                  «       }n1|ddd…dd…f   j                  «       }|	ddd…f   j                  «       }t        «       } ||j                  d|j!                  d«      «      |j                  d«      «      }|s|f|dd z   }|�|f|z   S |S t#        |||j$                  |j&                  |j(                  |j*                  ¬«      S )a4  
            labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
                Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
                config.vocab_size]` or `model.image_token_id` (where `model` is your instance of `Idefics2ForConditionalGeneration`).
                Tokens with indices set to `model.image_token_id` are ignored (masked), the loss is only
                computed for the tokens with labels in `[0, ..., config.vocab_size]`.

            logits_to_keep (`int` or `torch.Tensor`, *optional*):
                If an `int`, compute logits for the last `logits_to_keep` tokens. If `0`, calculate logits for all
                `input_ids` (special case). Only last token logits are needed for generation, and calculating them only for that
                token can save memory, which becomes pretty significant for long sequences or large vocabulary size.
                If a `torch.Tensor`, must be 1D corresponding to the indices to keep in the sequence length dimension.
                This is useful when using packed tensor format (single dimension for batch and sequence length).

        Returns:

        Example:

        ```python
        >>> import requests
        >>> import torch
        >>> from PIL import Image
        >>> from io import BytesIO

        >>> from transformers import AutoProcessor, AutoModelForVision2Seq
        >>> from transformers.image_utils import load_image

        >>> # Note that passing the image urls (instead of the actual pil images) to the processor is also possible
        >>> image1 = load_image("https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg")
        >>> image2 = load_image("https://cdn.britannica.com/59/94459-050-DBA42467/Skyline-Chicago.jpg")
        >>> image3 = load_image("https://cdn.britannica.com/68/170868-050-8DDE8263/Golden-Gate-Bridge-San-Francisco.jpg")

        >>> processor = AutoProcessor.from_pretrained("HuggingFaceM4/idefics2-8b-base")
        >>> model = AutoModelForVision2Seq.from_pretrained("HuggingFaceM4/idefics2-8b-base", device_map="auto")

        >>> BAD_WORDS_IDS = processor.tokenizer(["<image>", "<fake_token_around_image>"], add_special_tokens=False).input_ids
        >>> EOS_WORDS_IDS = [processor.tokenizer.eos_token_id]

        >>> # Create inputs
        >>> prompts = [
        ...   "<image>In this image, we can see the city of New York, and more specifically the Statue of Liberty.<image>In this image,",
        ...   "In which city is that bridge located?<image>",
        ... ]
        >>> images = [[image1, image2], [image3]]
        >>> inputs = processor(images=images, text=prompts, padding=True, return_tensors="pt").to("cuda")

        >>> # Generate
        >>> generated_ids = model.generate(**inputs, bad_words_ids=BAD_WORDS_IDS, max_new_tokens=20)
        >>> generated_texts = processor.batch_decode(generated_ids, skip_special_tokens=True)

        >>> print(generated_texts)
        ['In this image, we can see the city of New York, and more specifically the Statue of Liberty. In this image, we can see the city of New York, and more specifically the Statue of Liberty.\n\n', 'In which city is that bridge located?\n\nThe bridge is located in the city of Pittsburgh, Pennsylvania.\n\n\nThe bridge is']
        ```N)r¬  r�   rn   r#   r  rP   rµ  r&   rV  r§   rõ   r¶  rö   r   r   .rW   )r3   r4   r#   r$   r%   r&   )r7   r§   rõ   rÿ   r  r  rÐ   ÚslicerÌ  Úfloatrb   rd   rX   r‘   r
   r`   rT   r2   r#   r$   r%   r&   )rN   r¬  r�   rn   r#   r  rP   rµ  r&   rÝ  rV  r§   rõ   rö   r¶  rÙ  rì   r$   Úslice_indicesr4   r3   Úshift_attention_maskÚshift_logitsÚshift_labelsÚloss_fctrž  s                             r0   rx   z(Idefics2ForConditionalGeneration.forward  sO  € ðV 2CÐ1NÑ-ÐTX×T_ÑT_×TqÑTqÐà$8Ð$DÑ È$Ï+É+×JjÑJjð 	ð &1Ð%<‘kÀ$Ç+Á+×B]ÑB]ˆð —*‘*ØØ)Ø%Ø+Ø'Ø%Ø!5Ø 3ØØ/Ø!5Ø)Ø#ð ó 
ˆð    ™
ˆä8BÀ>ÔSVÔ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜mªA¨}ºaÐ,?Ñ@ÓAˆàˆØÑà—\‘\“^ˆFØ—Y‘Y˜vŸ}™}Ó-ˆFàÐ)ð (6²a¸6¿<¹<È¹?ÈQÑ;NÐ9OÑ9QÐ6QÑ'R×'UÑ'UÐV\×VcÑVcÓ'dÐ$Ø% c¨3¨B¨3² kÑ2Ð3GÈ1Ñ3LÑM×XÑXÓZ�Ø% c¨1©2 g™Ð/CÀqÑ/HÑI×TÑTÓV‘à% c¨3¨B¨3² kÑ2×=Ñ=Ó?�Ø% c¨1©2 g™×9Ñ9Ó;�ä'Ó)ˆHÙ˜L×-Ñ-¨b°,×2CÑ2CÀBÓ2GÓHÈ,×J[ÑJ[Ð\^ÓJ_Ó`ˆDáØ�Y ¨¨ Ñ,ˆFØ'+Ð'7�D�7˜VÑ#ÐC¸VÐCä-ØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)Ø '× ;Ñ ;ô
ð 	
r/   c
                 óp   •— t        ‰| �  |f||||||||	dœ|
¤Ž}|�|d   dk(  r||d<   |�
d |d<   d |d<   |S )N)r#   r�   r  r¶  rP   rµ  r&   rÙ  r   r¬  rP   rµ  )r?   Úprepare_inputs_for_generation)rN   r¬  r#   r�   r  r¶  rP   rµ  r&   rÙ  r’   Úmodel_inputsrO   s               €r0   rç  z>Idefics2ForConditionalGeneration.prepare_inputs_for_generation“  s}   ø€ ô  ‘wÑ<Øð
à+Ø)Ø'Ø)Ø%Ø!5Ø 3Ø)ñ
ð ñ
ˆð Ð$¨¸Ñ):¸aÒ)?Ø(1ˆL˜Ñ%àÐ*Ø+/ˆL˜Ñ(Ø37ˆLÐ/Ñ0àÐr/   c                 óL   •— t        ‰| �  d|||dœ|¤Ž}|j                  |d<   |S )N)rì   Úmodel_kwargsÚis_encoder_decoderr&   r.   )r?   Ú#_update_model_kwargs_for_generationr&   )rN   rì   rê  rë  r’   rO   s        €r0   rì  zDIdefics2ForConditionalGeneration._update_model_kwargs_for_generation»  sC   ø€ Ü‘wÑBð 
ØØ%Ø1ñ
ð ñ	
ˆð /6×.IÑ.IˆÐ*Ñ+ØÐr/   c                 óJ   ‡— d}| D ]  }|t        ˆfd„|D «       «      fz  }Œ |S )Nr.   c              3   ót   •K  — | ]/  }|j                  d ‰j                  |j                  «      «      –— Œ1 y­w)r   N)Úindex_selectrb   rd   )rú   Ú
past_stateÚbeam_idxs     €r0   rü   zBIdefics2ForConditionalGeneration._reorder_cache.<locals>.<genexpr>Ì  s.   øè ø€ ÒnÐU_�j×-Ñ-¨a°·±¸Z×=NÑ=NÓ1O×PÑnùs   ƒ58)r  )r#   rñ  Úreordered_pastÚ
layer_pasts    `  r0   Ú_reorder_cachez/Idefics2ForConditionalGeneration._reorder_cacheÆ  s?   ø€ ð ˆØ)ò 	ˆJØÜÓnÐcmÔnÓnðñ ‰Nð	ð Ðr/   )NNNNNNNNNNNNNNr   )NNNNNNNN)"r'   r(   r)   Ú_tied_weights_keysr@   r¤  r¨  r.  r1  rÔ  r×  r   r   rÈ  r   r2   Ú_CONFIG_FOR_DOCr   r+   r`  rz   r   r,   ry   r¸   r   rÐ   r   rx   rç  rì  Ústaticmethodrô  r{   r|   s   @r0   rÊ  rÊ  Ü  s%  ø„ ð
 +Ð+Ðô	ò
ò1ò<ò:òò&ñ Ð)°6ÐDTÔUÙ*Ð+DÓEÙÐ+IÐXgÔhð 15Ø15Ø37Ø=AØ59Ø48Ø;?Ø;?Ø-1Ø$(Ø,0Ø/3Ø&*Ø59Ø34ñ!B
à˜E×,Ñ,Ñ-ðB
ð ! §¡Ñ.ðB
ð ˜u×/Ñ/Ñ0ð	B
ð
 " $ u×'8Ñ'8Ñ"9Ñ:ðB
ð   × 1Ñ 1Ñ2ðB
ð ˜u×0Ñ0Ñ1ðB
ð ' u×'7Ñ'7Ñ8ðB
ð & e×&7Ñ&7Ñ8ðB
ð ˜×)Ñ)Ñ*ðB
ð ˜D‘>ðB
ð $ D™>ðB
ð ' t™nðB
ð ˜d‘^ðB
ð ! ×!1Ñ!1Ñ2ðB
ð  ˜c 5§<¡<Ð/Ñ0ð!B
ð" 
ˆuÐ4Ð4Ñ	5ò#B
ó ió Fó VðB
ðN ØØØØØ!Ø Øõ&ôP	ð ñó ôr/   rÊ  )rÊ  r
  r‡  )r¬   )Ir*   Údataclassesr   Útypingr   r   r   r   r   r+   Útorch.utils.checkpointr	   Útorch.nnr
   Úactivationsr   Úcache_utilsr   r   Ú
generationr   Úmodeling_attn_mask_utilsr   Úmodeling_outputsr   r   Úmodeling_utilsr   r   Úutilsr   r   r   r   Úutils.deprecationr   Úautor   Úconfiguration_idefics2r   r   r   Ú
get_loggerr'   r¯   rö  r!   r2   ÚModuler6   rz   rà  r–   r˜   rº   rÄ   rÓ   ræ   rî   ÚIDEFICS2_START_DOCSTRINGr
  rÈ  r$  rÐ   rŒ   r@  rL  r  rq  r}  r‡  rÊ  Ú__all__r.   r/   r0   ú<module>r
     sñ  ðñ å !ß 9Õ 9ã Û Ý Ý %å !ß .Ý )Ý Bß <ß F÷ó õ 1Ý ß aÑ að 
ˆ×	Ñ	˜HÓ	%€à"€ð ô#C kó #Có ð#CðL ô!C [ó !Có ð!CôH7˜rŸy™yô 7ðB ñ%Ø�I‰Ið%à�<‰<ð%ð 
�‰ð%ð �<‰<ð	%ð
 ˜UŸ\™\Ñ*ð%ð ð%ð ó%ô8G)˜bŸi™iô G)ôV˜Ÿ	™	ô ôP�"—)‘)ô Pô&"¨B¯I©Iô "ô<.˜2Ÿ9™9ô .ôdU
�b—i‘iô U
ðpÐ ñ" Ø\Øóô?˜oó ?ó	ð?ð>Ð ñ& ØKØóôM
Ð 7ó M
ó	ðM
ðb	U˜UŸ\™\ð 	U°#ð 	U¸%¿,¹,ó 	UôJ�b—i‘iô Jô(X9 §¡ô X9ôvE˜RŸY™Yô EðPÐ ñ ØfØtØpØó	ô9"Ð!8ó 9"óð9"ôx#˜Ÿ	™	ô #ð"GÐ ñT Ø[Øóôc
Ð+ó c
ó	ðc
ñL ð JØóônÐ'>Àó nó	ðnòb [�r/   