Ë
    T^(hïS  ã                   ó  — d Z ddlmZmZmZ ddlZddlZddlmZ ddlm	Z	 ddl
mZ ddlmZ dd	lmZ dd
lmZmZmZ ddlmZ  ej,                  e«      Zd„ Z G d„ dej4                  «      Zd„ Zd$d„Z G d„ dej4                  «      Z G d„ dej4                  «      Z G d„ dej4                  «      Z  G d„ dej4                  «      Z! G d„ dej4                  «      Z"dZ# G d„ de	«      Z$dZ%d „ Z& ed!e#«       G d"„ d#e$«      «       Z'd#dgZ(y)%zPyTorch Pixtral model.é    )ÚOptionalÚTupleÚUnionN)Únné   )ÚPreTrainedModel)ÚACT2FN)ÚBaseModelOutput)Údynamic_rope_update)Úadd_start_docstringsÚ%add_start_docstrings_to_model_forwardÚloggingé   )ÚPixtralVisionConfigc                 ó’  — g }| D ]¬  }|j                   dd  \  }}t        j                  t        j                  |«      t        j                  |«      d¬«      }t        j                  |d¬«      j                  dd«      j                  dd«      \  }}||z  |z   }	|j                  |	d d …df   «       Œ® t        j                  |«      S )NéþÿÿÿÚij)Úindexingéÿÿÿÿ©Údimé   r   )	ÚshapeÚtorchÚmeshgridÚarangeÚstackÚreshapeÚchunkÚappendÚcat)
Úpatch_embeds_listÚ	max_widthÚ	positionsÚpatchÚheightÚwidthÚmeshÚh_gridÚv_gridÚidss
             új/var/www/skyplay_api_hub/venv/lib/python3.12/site-packages/transformers/models/pixtral/modeling_pixtral.pyÚposition_ids_in_meshgridr-   &   s³   € Ø€IØ"ò $ˆØŸ™ B CÐ(‰ˆ�Ü�~‰~œeŸl™l¨6Ó2´E·L±LÀÓ4GÐRVÔWˆÜŸ™ T¨rÔ2×:Ñ:¸2¸qÓA×GÑGÈÈ2ÓN‰ˆ�Ø�yÑ  6Ñ)ˆØ×Ñ˜šQ ˜T™Õ#ð$ô �9‰9�YÓÐó    c                   ó\   ‡ — e Zd ZdZdˆ fd„	Z ej                  «       ed„ «       «       Zˆ xZ	S )ÚPixtralRotaryEmbeddinga  
    The key with pixtral embedding is just that you have a frequency for each pixel positions.
    If you have height x width pixels (or embedding pixels), then the frequency used for ROPE
    is given by indexing the pre_computed frequency on the width and height.

    What you output is of dimension (batch, height * width, dim) with dim the embed dim.

    This simply means that for each image hidden state, you are going to add
    a corresponding positional embedding, based on its index in the grid.
    c                 ó˜  •— t         ‰
| �  «        d| _        |j                  | _        |j
                  | _        |j                  |j                  z  }d| j                  t        j                  d| j                  d«      j                  «       | j                  z  z  z  }t        j                  ||j                  ¬«      }t        j                  ||j                  ¬«      }t        j                  ||d d d…   «      j                  «       }t        j                  ||dd d…   «      j                  «       }t        j                  |d d …d d d …f   j                  d|d«      |d d d …d d …f   j                  |dd«      gd¬«      j!                  d| j                  dz  «      }	| j#                  d	t        j                  |	|	fd¬«      d
¬«       y )NÚdefaultg      ð?r   r   )Údevicer   r   r   Úinv_freqF)Ú
persistent)ÚsuperÚ__init__Ú	rope_typeÚhead_dimr   Ú
rope_thetaÚbaseÚ
image_sizeÚ
patch_sizer   r   Úfloatr3   Úouterr!   Úrepeatr   Úregister_buffer)ÚselfÚconfigr3   Úmax_patches_per_sideÚfreqsÚhÚwÚfreqs_hÚfreqs_wr4   Ú	__class__s             €r,   r7   zPixtralRotaryEmbedding.__init__=   s}  ø€ Ü‰ÑÔØ"ˆŒØ—?‘?ˆŒØ×%Ñ%ˆŒ	Ø%×0Ñ0°F×4EÑ4EÑEÐØ�t—y‘y¤U§\¡\°!°T·X±X¸qÓ%A×%GÑ%GÓ%IÈDÏHÉHÑ%TÑUÑVˆä�L‰LÐ-°e·l±lÔCˆÜ�L‰LÐ-°e·l±lÔCˆä—+‘+˜a ¡s¨ s¡Ó,×2Ñ2Ó4ˆÜ—+‘+˜a  q t¨! t¡Ó-×3Ñ3Ó5ˆÜ—9‘9àš˜4¢˜
Ñ#×*Ñ*¨1Ð.BÀAÓFØ˜ša¢˜
Ñ#×*Ñ*Ð+?ÀÀAÓFðð ô
÷ ‰'�"�d—h‘h !‘mÓ
$ð 	ð 	×Ñ˜Z¬¯©°H¸hÐ3GÈRÔ)PÐ]bÐÕcr.   c                 óÖ  — | j                   |   }t        |j                  j                  t        «      r/|j                  j                  dk7  r|j                  j                  nd}t        j                  |d¬«      5  |}|j                  «       }|j                  «       }d d d «       j                  |j                  ¬«      j                  |j                  ¬«      fS # 1 sw Y   ŒAxY w)NÚmpsÚcpuF)Údevice_typeÚenabled)Údtype)r4   Ú
isinstancer3   ÚtypeÚstrr   ÚautocastÚcosÚsinÚtorP   )rB   ÚxÚposition_idsrE   rN   ÚembrU   rV   s           r,   ÚforwardzPixtralRotaryEmbedding.forwardV   s¬   € ð —‘˜lÑ+ˆä'1°!·(±(·-±-ÄÔ'EÈ!Ï(É(Ï-É-Ð[`ÒJ`�a—h‘h—m’mÐfkˆÜ�^‰^¨¸UÔCñ 	ØˆCØ—'‘'“)ˆCØ—'‘'“)ˆC÷	ð
 �v‰v˜AŸG™GˆvÓ$ c§f¡f°1·7±7 fÓ&;Ð;Ð;÷	ð 	ús   Á<#CÃC(©N)
Ú__name__Ú
__module__Ú__qualname__Ú__doc__r7   r   Úno_gradr   r[   Ú__classcell__©rJ   s   @r,   r0   r0   1   s2   ø„ ñ	õdð2 €U‡]�]ƒ_Øñ	<ó ó ô	<r.   r0   c                 óš   — | dd| j                   d   dz  …f   }| d| j                   d   dz  d…f   }t        j                  | |fd¬«      S )z*Rotates half the hidden dims of the input..Nr   r   r   )r   r   r!   )rX   Úx1Úx2s      r,   Úrotate_halfrg   e   sZ   € à	
ˆ3Ð"�!—'‘'˜"‘+ Ñ"Ð"Ð"Ñ	#€BØ	
ˆ3�—‘˜‘˜qÑ Ñ"Ð"Ñ	#€BÜ�9‰9�r�c˜2�Y BÔ'Ð'r.   c                 óž   — |j                  |«      }|j                  |«      }| |z  t        | «      |z  z   }||z  t        |«      |z  z   }||fS )aÛ  Applies Rotary Position Embedding to the query and key tensors.

    Args:
        q (`torch.Tensor`): The query tensor.
        k (`torch.Tensor`): The key tensor.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        position_ids (`torch.Tensor`, *optional*):
            Deprecated and unused.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    )Ú	unsqueezerg   )ÚqÚkrU   rV   rY   Úunsqueeze_dimÚq_embedÚk_embeds           r,   Úapply_rotary_pos_embro   l   sY   € ð( �-‰-˜Ó
&€CØ
�-‰-˜Ó
&€CØ�3‰wœ; q›>¨CÑ/Ñ0€GØ�3‰wœ; q›>¨CÑ/Ñ0€GØ�GÐÐr.   c                   óê   ‡ — e Zd ZdZˆ fd„Z	 	 	 d	dej                  deej                     deeej                  ej                  f      dee	   deej                  eej                     f   f
d„Z
ˆ xZS )
ÚPixtralAttentionz=Multi-headed attention from 'Attention Is All You Need' paperc                 ó‚  •— t         ‰| �  «        || _        |j                  | _        |j
                  | _        | j                  | j                  z  | _        | j                  dz  | _        |j                  | _
        t        j                  | j                  | j                  d¬«      | _        t        j                  | j                  | j                  d¬«      | _        t        j                  | j                  | j                  d¬«      | _        t        j                  | j                  | j                  d¬«      | _        y )Ng      à¿F©Úbias)r6   r7   rC   Úhidden_sizeÚ	embed_dimÚnum_attention_headsÚ	num_headsr9   ÚscaleÚattention_dropoutÚdropoutr   ÚLinearÚk_projÚv_projÚq_projÚo_proj©rB   rC   rJ   s     €r,   r7   zPixtralAttention.__init__Š   sÐ   ø€ Ü‰ÑÔØˆŒØ×+Ñ+ˆŒØ×3Ñ3ˆŒØŸ™¨$¯.©.Ñ8ˆŒà—]‘] DÑ(ˆŒ
Ø×/Ñ/ˆŒä—i‘i §¡°·±ÀUÔKˆŒÜ—i‘i §¡°·±ÀUÔKˆŒÜ—i‘i §¡°·±ÀUÔKˆŒÜ—i‘i §¡°·±ÀUÔKˆ�r.   Úhidden_statesÚattention_maskÚposition_embeddingsÚoutput_attentionsÚreturnc                 ó>  — |j                  «       \  }}}| j                  |«      }| j                  |«      }	| j                  |«      }
|j	                  ||| j
                  | j                  «      j                  dd«      }|	j	                  ||| j
                  | j                  «      j                  dd«      }	|
j	                  ||| j
                  | j                  «      j                  dd«      }
|\  }}t        ||	||d¬«      \  }}	t        j                  ||	j                  dd«      «      | j                  z  }|�||z   }t        j                  j                  |dt        j                  ¬«      j!                  |j"                  «      }t        j                  j%                  || j$                  | j&                  ¬«      }t        j                  ||
«      }|j                  dd«      j)                  «       }|j+                  ||d«      }| j-                  |«      }||fS )	z#Input shape: Batch x Time x Channelr   r   r   )rl   r   r   )r   rP   )ÚpÚtraining)Úsizer   r}   r~   Úviewrx   r9   Ú	transposero   r   Úmatmulry   r   Ú
functionalÚsoftmaxÚfloat32rW   rP   r{   r‰   Ú
contiguousr   r€   )rB   r‚   rƒ   r„   r…   Ú
batch_sizeÚpatchesÚ_Úquery_statesÚ
key_statesÚvalue_statesrU   rV   Úattn_weightsÚattn_outputs                  r,   r[   zPixtralAttention.forward™   sÕ  € ð "/×!3Ñ!3Ó!5Ñˆ
�G˜Qà—{‘{ =Ó1ˆØ—[‘[ Ó/ˆ
Ø—{‘{ =Ó1ˆà#×(Ñ(¨°W¸d¿n¹nÈdÏmÉmÓ\×fÑfÐghÐjkÓlˆØ—_‘_ Z°¸$¿.¹.È$Ï-É-ÓX×bÑbÐcdÐfgÓhˆ
Ø#×(Ñ(¨°W¸d¿n¹nÈdÏmÉmÓ\×fÑfÐghÐjkÓlˆà&‰ˆˆSÜ#7¸ÀjÐRUÐWZÐjkÔ#lÑ ˆ�jä—|‘| L°*×2FÑ2FÀqÈ!Ó2LÓMÐPT×PZÑPZÑZˆàÐ%Ø'¨.Ñ8ˆLô —}‘}×,Ñ,¨\¸rÌÏÉÐ,ÓW×ZÑZÐ[g×[mÑ[mÓnˆÜ—}‘}×,Ñ,¨\¸T¿\¹\ÐTX×TaÑTaÐ,ÓbˆÜ—l‘l <°Ó>ˆà!×+Ñ+¨A¨qÓ1×<Ñ<Ó>ˆØ!×)Ñ)¨*°g¸rÓBˆà—k‘k +Ó.ˆà˜LÐ(Ð(r.   )NNF)r]   r^   r_   r`   r7   r   ÚTensorr   r   Úboolr[   rb   rc   s   @r,   rq   rq   ‡   s‹   ø„ ÙGôLð$ 26ØKOØ,1ñ%)à—|‘|ð%)ð ! §¡Ñ.ð%)ð & e¨E¯L©L¸%¿,¹,Ð,FÑ&GÑHð	%)ð
 $ D™>ð%)ð 
ˆu�|‰|˜X e§l¡lÑ3Ð3Ñ	4÷%)r.   rq   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )Ú
PixtralMLPc                 óÊ  •— t         ‰| �  «        || _        |j                  | _        |j                  | _        t        j                  | j                  | j                  d¬«      | _        t        j                  | j                  | j                  d¬«      | _        t        j                  | j                  | j                  d¬«      | _	        t        |j                     | _        y )NFrs   )r6   r7   rC   ru   Úintermediate_sizer   r|   Ú	gate_projÚup_projÚ	down_projr	   Ú
hidden_actÚact_fnr�   s     €r,   r7   zPixtralMLP.__init__Ã   s¦   ø€ Ü‰ÑÔØˆŒØ!×-Ñ-ˆÔØ!'×!9Ñ!9ˆÔÜŸ™ 4×#3Ñ#3°T×5KÑ5KÐRWÔXˆŒÜ—y‘y ×!1Ñ!1°4×3IÑ3IÐPUÔVˆŒÜŸ™ 4×#9Ñ#9¸4×;KÑ;KÐRWÔXˆŒÜ˜V×.Ñ.Ñ/ˆ�r.   c                 óˆ   — | j                  | j                  | j                  |«      «      | j                  |«      z  «      }|S r\   )r¢   r¤   r    r¡   )rB   rX   r¢   s      r,   r[   zPixtralMLP.forwardÍ   s6   € Ø—N‘N 4§;¡;¨t¯~©~¸aÓ/@Ó#AÀDÇLÁLÐQRÃOÑ#SÓTˆ	ØÐr.   )r]   r^   r_   r7   r[   rb   rc   s   @r,   r�   r�   Â   s   ø„ ô0ör.   r�   c                   ó,   ‡ — e Zd Zdˆ fd„	Zd„ Zd„ Zˆ xZS )ÚPixtralRMSNormc                 óŠ   •— t         ‰| �  «        t        j                  t	        j
                  |«      «      | _        || _        y)z=
        PixtralRMSNorm is equivalent to T5LayerNorm
        N)r6   r7   r   Ú	Parameterr   ÚonesÚweightÚvariance_epsilon)rB   ru   ÚepsrJ   s      €r,   r7   zPixtralRMSNorm.__init__Ô   s1   ø€ ô 	‰ÑÔÜ—l‘l¤5§:¡:¨kÓ#:Ó;ˆŒØ #ˆÕr.   c                 ó"  — |j                   }|j                  t        j                  «      }|j	                  d«      j                  dd¬«      }|t        j                  || j                  z   «      z  }| j                  |j                  |«      z  S )Nr   r   T)Úkeepdim)	rP   rW   r   r�   ÚpowÚmeanÚrsqrtr¬   r«   )rB   r‚   Úinput_dtypeÚvariances       r,   r[   zPixtralRMSNorm.forwardÜ   sy   € Ø#×)Ñ)ˆØ%×(Ñ(¬¯©Ó7ˆØ ×$Ñ$ QÓ'×,Ñ,¨R¸Ð,Ó>ˆØ%¬¯©°H¸t×?TÑ?TÑ4TÓ(UÑUˆØ�{‰{˜]×-Ñ-¨kÓ:Ñ:Ð:r.   c                 ó^   — t        | j                  j                  «      › d| j                  › �S )Nz, eps=)Útupler«   r   r¬   ©rB   s    r,   Ú
extra_reprzPixtralRMSNorm.extra_reprã   s*   € Ü˜Ÿ™×)Ñ)Ó*Ð+¨6°$×2GÑ2GÐ1HÐIÐIr.   )g�íµ ÷Æ°>)r]   r^   r_   r7   r[   r¸   rb   rc   s   @r,   r§   r§   Ó   s   ø„ õ$ò;öJr.   r§   c                   óÀ   ‡ — e Zd Zˆ fd„Z	 	 ddej
                  dej
                  deeej
                  ej
                  f      dee   deej                     f
d„Z
ˆ xZS )	ÚPixtralAttentionLayerc                 óÒ   •— t         ‰| �  «        t        |j                  d¬«      | _        t        |«      | _        t        |«      | _        t        |j                  d¬«      | _	        y )Nçñhãˆµøä>©r­   )
r6   r7   r§   ru   Úattention_normr�   Úfeed_forwardrq   Ú	attentionÚffn_normr�   s     €r,   r7   zPixtralAttentionLayer.__init__è   sP   ø€ Ü‰ÑÔÜ,¨V×-?Ñ-?ÀTÔJˆÔÜ& vÓ.ˆÔÜ)¨&Ó1ˆŒÜ& v×'9Ñ'9¸tÔDˆ�r.   r‚   rƒ   r„   r…   r†   c                 óÎ   — |}| j                  |«      }| j                  ||||¬«      \  }}||z   }|}| j                  |«      }| j                  |«      }||z   }|f}|r||fz  }|S )a=  
        Args:
            hidden_states (`torch.FloatTensor`):
                Input to the layer of shape `(batch, seq_len, embed_dim)`.
            attention_mask (`torch.FloatTensor`):
                Attention mask of shape `(batch, 1, q_len, k_v_seq_len)` where padding elements are indicated by very large negative values.
            output_attentions (`bool`, *optional*, defaults to `False`):
                Whether or not to return the attentions tensors of all attention layers. See `attentions` under
                returned tensors for more detail.
        )r‚   rƒ   r„   r…   )r¾   rÀ   rÁ   r¿   )rB   r‚   rƒ   r„   r…   Úresidualr˜   Úoutputss           r,   r[   zPixtralAttentionLayer.forwardï   s’   € ð" !ˆà×+Ñ+¨MÓ:ˆØ&*§n¡nØ'Ø)Ø 3Ø/ð	 '5ó '
Ñ#ˆ�|ð ! =Ñ0ˆà ˆØŸ™ mÓ4ˆØ×)Ñ)¨-Ó8ˆØ  =Ñ0ˆà Ð"ˆáØ˜�Ñ&ˆGØˆr.   )NN)r]   r^   r_   r7   r   rš   r   r   r›   ÚFloatTensorr[   rb   rc   s   @r,   rº   rº   ç   sv   ø„ ôEð LPØ,0ñ%à—|‘|ð%ð Ÿ™ð%ð & e¨E¯L©L¸%¿,¹,Ð,FÑ&GÑHð	%ð
 $ D™>ð%ð 
ˆu× Ñ Ñ	!÷%r.   rº   c                   ó¸   ‡ — e Zd Zˆ fd„Z	 	 	 	 	 d	deej                     deeej                  ej                  f      dee   dee   dee   de	ee
f   fd„Zˆ xZS )
ÚPixtralTransformerc                 ó   •— t         ‰| �  «        || _        t        j                  j                  «       | _        t        |j                  «      D ]&  }| j                  j                  t        |«      «       Œ( d| _        y )NF)r6   r7   rC   r   r   Ú
ModuleListÚlayersÚrangeÚnum_hidden_layersr    rº   Úgradient_checkpointing)rB   rC   r”   rJ   s      €r,   r7   zPixtralTransformer.__init__  sc   ø€ Ü‰ÑÔØˆŒÜ—h‘h×)Ñ)Ó+ˆŒÜ�v×/Ñ/Ó0ò 	>ˆAØ�K‰K×ÑÔ4°VÓ<Õ=ð	>à&+ˆÕ#r.   rƒ   r„   r…   Úoutput_hidden_statesÚreturn_dictr†   c                 óð  — |�|n| j                   j                  }|�|n| j                   j                  }|�|n| j                   j                  }|rdnd}|rdnd}|}	| j                  D ]_  }
|r||	fz   }| j
                  r,| j                  r | j                  |
j                  |	|||«      }n |
|	|||¬«      }|d   }	|sŒW||d   fz   }Œa |r||	fz   }|st        d„ |	||fD «       «      S t        |	||¬«      S )av  
        Args:
            inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
                Embeddings which serve as input to the Transformer.
            attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
                Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:

                - 1 for tokens that are **not masked**,
                - 0 for tokens that are **masked**.

                [What are attention masks?](../glossary#attention-mask)
            output_attentions (`bool`, *optional*):
                Whether or not to return the attentions tensors of all attention layers. See `attentions` under
                returned tensors for more detail.
            output_hidden_states (`bool`, *optional*):
                Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors
                for more detail.
            return_dict (`bool`, *optional*):
                Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
        N© )r„   r…   r   r   c              3   ó&   K  — | ]	  }|€Œ|–— Œ y ­wr\   rÑ   )Ú.0Úvs     r,   ú	<genexpr>z-PixtralTransformer.forward.<locals>.<genexpr>c  s   è ø€ Òe˜qÐWXÑWdœÑeùs   ‚Š)Úlast_hidden_stater‚   Ú
attentions)rC   r…   rÎ   Úuse_return_dictrÊ   rÍ   r‰   Ú_gradient_checkpointing_funcÚ__call__r¶   r
   )rB   Úinputs_embedsrƒ   r„   r…   rÎ   rÏ   Úencoder_statesÚall_attentionsr‚   Úencoder_layerÚlayer_outputss               r,   r[   zPixtralTransformer.forward   s=  € ð: 2CÐ1NÑ-ÐTX×T_ÑT_×TqÑTqÐà$8Ð$DÑ È$Ï+É+×JjÑJjð 	ð &1Ð%<‘kÀ$Ç+Á+×B]ÑB]ˆá3™¸ˆÙ0™°dˆà%ˆØ!Ÿ[™[ò 	FˆMÙ#Ø!/°=Ð2BÑ!B�Ø×*Ò*¨t¯}ª}Ø $× AÑ AØ!×*Ñ*Ø!Ø"Ø'Ø%ó!‘ñ !.Ø!Ø"Ø(;Ø&7ô	!�ð *¨!Ñ,ˆMâ Ø!/°=ÀÑ3CÐ2EÑ!E‘ð-	Fñ0  Ø+¨}Ð.>Ñ>ˆNáÜÑe ]°NÀNÐ$SÔeÓeÐeÜØ+¸>ÐVdô
ð 	
r.   )NNNNN)r]   r^   r_   r7   r   r   rš   r   r›   r   r
   r[   rb   rc   s   @r,   rÇ   rÇ     s—   ø„ ô,ð 26ØKOØ,0Ø/3Ø&*ñF
ð ! §¡Ñ.ðF
ð & e¨E¯L©L¸%¿,¹,Ð,FÑ&GÑHð	F
ð
 $ D™>ðF
ð ' t™nðF
ð ˜d‘^ðF
ð 
ˆu�oÐ%Ñ	&÷F
r.   rÇ   a[  
    This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
    library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
    etc.)

    This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
    Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
    and behavior.

    Parameters:
        config ([`PixtralVisionConfig`]):
            Model configuration class with all the parameters of the vision encoder. Initializing with a config file does not
            load the weights associated with the model, only the configuration. Check out the
            [`~PreTrainedModel.from_pretrained`] method to load the model weights.
c                   ó(   — e Zd ZeZdZdZdZdgZd„ Z	y)ÚPixtralPreTrainedModelÚmodelÚpixel_valuesTrº   c                 óŽ  — t        | j                  d«      r| j                  j                  n| j                  j                  }t        |t        j
                  t        j                  f«      rY|j                  j                  j                  d|¬«       |j                  �%|j                  j                  j                  «        y y t        |t        j                  «      rf|j                  j                  j                  d|¬«       |j                  �2|j                  j                  |j                     j                  «        y y y )NÚinitializer_rangeg        )r±   Ústd)ÚhasattrrC   rå   rQ   r   r|   ÚConv2dr«   ÚdataÚnormal_rt   Úzero_Ú	EmbeddingÚpadding_idx)rB   Úmoduleræ   s      r,   Ú_init_weightsz$PixtralPreTrainedModel._init_weights�  sñ   € ô �t—{‘{Ð$7Ô8ð �K‰K×)Ò)à—‘×.Ñ.ð 	ô �fœrŸy™y¬"¯)©)Ð4Ô5Ø�M‰M×Ñ×&Ñ&¨C°SÐ&Ô9Ø�{‰{Ð&Ø—‘× Ñ ×&Ñ&Õ(ð 'ä˜¤§¡Ô-Ø�M‰M×Ñ×&Ñ&¨C°SÐ&Ô9Ø×!Ñ!Ð-Ø—‘×"Ñ" 6×#5Ñ#5Ñ6×<Ñ<Õ>ð .ð .r.   N)
r]   r^   r_   r   Úconfig_classÚbase_model_prefixÚmain_input_nameÚsupports_gradient_checkpointingÚ_no_split_modulesrï   rÑ   r.   r,   rá   rá   z  s&   „ Ø&€LØÐØ$€OØ&*Ð#Ø0Ð1Ðó?r.   rá   a´  
    Args:
        pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`):
            Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See [`AutoImageProcessor.__call__`]
            for details.
        image_sizes (`torch.LongTensor` of shape `(batch_size, 2)`, *optional*):
            The sizes of the images in the batch, being (height, width) for each image.
        output_attentions (`bool`, *optional*):
            Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
            tensors for more detail.
        output_hidden_states (`bool`, *optional*):
            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
            more detail.
        return_dict (`bool`, *optional*):
            Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
c                 óü  — |j                   }|j                  }|j                  d   }t        j                  |«      j
                  }t        j                  ||f|||¬«      }t        j                  | «      j                  d«      }t        j                  dg| d d z   «      j                  d«      }t        ||«      D ]  \  }	}
d||	|
…|	|
…f<   Œ |d d d d …d d …f   j                  |j                  d   ddd«      }|S )Nr   )Ú
fill_valuerP   r3   r   r   )rP   r3   r   r   ÚfinfoÚminÚfullÚtensorÚcumsumÚzipÚexpand)r"   rú   rP   r3   Úseq_lenÚd_minÚcausal_maskÚblock_end_idxÚblock_start_idxÚstartÚends              r,   Úgenerate_block_attention_maskr  ¤  sö   € Ø�L‰L€EØ�]‰]€FØ�l‰l˜1‰o€GÜ�K‰K˜Ó×"Ñ"€EÜ—*‘*˜g wÐ/¸EÈÐW]Ô^€Kä—L‘LÐ!2Ó3×:Ñ:¸2Ó>€MÜ—l‘l A 3Ð):¸3¸BÐ)?Ñ#?Ó@×GÑGÈÓK€OÜ˜/¨=Ó9ò .‰
ˆˆsØ,-ˆ�E˜#�I˜u S˜yÐ(Ò)ð.ð ˜d Dª!ªQÐ.Ñ/×6Ñ6°v·|±|ÀA±ÈÈ2ÈrÓR€KØÐr.   z^The bare Pixtral vision encoder outputting raw hidden-states without any specific head on top.c                   óª   ‡ — e Zd ZdZˆ fd„Zd„ Z ee«      	 	 	 ddej                  dej                  de
e   de
e   de
e   d	eeef   fd
„«       Zˆ xZS )ÚPixtralVisionModelÚvision_encoderc                 óz  •— t         ‰| �  |«       || _        t        j                  |j
                  |j                  |j                  |j                  d¬«      | _        |j                  | _        t        |j                  d¬«      | _
        t        |«      | _        t        |«      | _        | j                  «        y )NF)Úin_channelsÚout_channelsÚkernel_sizeÚstridert   r¼   r½   )r6   r7   rC   r   rè   Únum_channelsru   r=   Ú
patch_convr§   Úln_prerÇ   Útransformerr0   Úpatch_positional_embeddingÚ	post_initr�   s     €r,   r7   zPixtralVisionModel.__init__»  s•   ø€ Ü‰Ñ˜Ô ØˆŒÜŸ)™)Ø×+Ñ+Ø×+Ñ+Ø×)Ñ)Ø×$Ñ$Øô
ˆŒð !×+Ñ+ˆŒÜ$ V×%7Ñ%7¸TÔBˆŒÜ-¨fÓ5ˆÔÜ*@ÀÓ*HˆÔ'à�‰Õr.   c                 ó   — | j                   S r\   )r  r·   s    r,   Úget_input_embeddingsz'PixtralVisionModel.get_input_embeddingsÌ  s   € Ø�‰Ðr.   rã   Úimage_sizesrÎ   r…   rÏ   r†   c           
      óÄ  — | j                  |«      }t        ||«      D �	�
cg c]1  \  }	}
|	dd|
d   | j                  z  …d|
d   | j                  z  …f   ‘Œ3 }}	}
t        j                  |D �cg c]  }|j                  d«      j                  ‘Œ c}d¬«      j                  d«      }| j                  |«      }t        || j                  j                  | j                  j                  z  ¬«      }| j                  ||«      }t        |D �cg c]!  }|j                  d   |j                  d   z  ‘Œ# c}|«      }| j                  ||||||¬	«      }|S c c}
}	w c c}w c c}w )
z�
        Returns:
            pixel_values: tensor of token features for
                all tokens of all images of shape (N_toks, D)
        .Nr   r   r   )r#   r   r   )rƒ   r„   rÎ   r…   rÏ   )r  rü   r=   r   r!   ÚflattenÚTri   r  r-   rC   r<   r  r  r   r  )rB   rã   r  rÎ   r…   rÏ   ÚargsÚkwargsÚpatch_embedsÚembedrŠ   r"   rˆ   rY   r„   rƒ   Úouts                    r,   r[   zPixtralVisionModel.forwardÏ  sc  € ð" —‘ |Ó4ˆô  # <°Ó=÷
á��tð �#Ð5˜$˜q™' T§_¡_Ñ4Ð5Ð7U¸$¸q¹'ÀTÇ_Á_Ñ:TÐ7UÐUÓVð
Ðñ 
ô —y‘yÐ:KÖ!L°Q !§)¡)¨A£,§.£.Ò!LÐRSÔT×^Ñ^Ð_`ÓaˆØ—{‘{ <Ó0ˆô 0Ø¨¯©×)?Ñ)?À4Ç;Á;×CYÑCYÑ)Yô
ˆð #×=Ñ=¸lÈLÓYÐä6Ø0AÖB¨1ˆQ�W‰W�R‰[˜1Ÿ7™7 2™;Ó&ÒBÀLó
ˆð ×ÑØØ)Ø 3Ø!5Ø/Ø#ð ó 
ˆð ˆ
ùó9
ùò "Mùò Cs   ¡6EÁ-"EÄ&E)NNN)r]   r^   r_   rñ   r7   r  r   ÚPIXTRAL_INPUTS_DOCSTRINGr   rš   r   r›   r   r   r
   r[   rb   rc   s   @r,   r  r  ´  s“   ø„ ð
 )Ðôò"ñ +Ð+CÓDð
 04Ø,0Ø&*ñ-à—l‘lð-ð —\‘\ð-ð ' t™nð	-ð
 $ D™>ð-ð ˜d‘^ð-ð 
ˆu�oÐ%Ñ	&ò-ó Eô-r.   r  )Nr   ))r`   Útypingr   r   r   r   Útorch.utils.checkpointr   Ú r   Úactivationsr	   Úmodeling_outputsr
   Úmodeling_rope_utilsr   Úutilsr   r   r   Úconfiguration_pixtralr   Ú
get_loggerr]   Úloggerr-   ÚModuler0   rg   ro   rq   r�   r§   rº   rÇ   ÚPIXTRAL_START_DOCSTRINGrá   r  r  r  Ú__all__rÑ   r.   r,   ú<module>r-     s  ðñ ç )Ñ )ã Û Ý å Ý !Ý /Ý 6÷ñ õ
 7ð 
ˆ×	Ñ	˜HÓ	%€ò ô0<˜RŸY™Yô 0<òh(óô67)�r—y‘yô 7)ôv�—‘ô ô"J�R—Y‘Yô Jô(-˜BŸI™Iô -ô`O
˜Ÿ™ô O
ðdÐ ô"?˜_ô ?ð0Ð ò$ñ  ØdØóôEÐ/ó Eó	ðEðP  Ð!9Ð
:�r.   