Ë
    T^(hÑ’  ã                   ó  — d Z ddlZddlmZmZmZmZ ddlZddlZddlm	Z	 ddl
mZ ddlmZ ddlmZmZ dd	lmZmZ dd
lmZ ddlmZmZmZmZ ddlmZ  ej:                  e«      ZdZ dZ!dZ"dZ# G d„ de	jH                  «      Z% G d„ de	jL                  «      Z' G d„ de	jL                  «      Z( G d„ de	jL                  «      Z) G d„ de«      Z* ede"«       G d„ de*«      «       Z+ ede"«       G d „ d!e*e«      «       Z,g d"¢Z-y)#zPyTorch XGLM model.é    N)ÚListÚOptionalÚTupleÚUnion)Únné   )ÚACT2FN)ÚGenerationMixin)Ú_prepare_4d_attention_maskÚ!_prepare_4d_causal_attention_mask)Ú)BaseModelOutputWithPastAndCrossAttentionsÚ!CausalLMOutputWithCrossAttentions)ÚPreTrainedModel)Úadd_code_sample_docstringsÚadd_start_docstringsÚ%add_start_docstrings_to_model_forwardÚloggingé   )Ú
XGLMConfigzfacebook/xglm-564Mr   aI  
    This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
    library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
    etc.)

    This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
    Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
    and behavior.

    Parameters:
        config ([`XGLMConfig`]):
            Model configuration class with all the parameters of the model. Initializing with a config file does not
            load the weights associated with the model, only the configuration. Check out the
            [`~PreTrainedModel.from_pretrained`] method to load the model weights.
a  
    Args:
        input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
            Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide
            it.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are input IDs?](../glossary#input-ids)
        attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
            Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:

            - 1 for tokens that are **not masked**,
            - 0 for tokens that are **masked**.

            [What are attention masks?](../glossary#attention-mask)
        position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
            config.max_position_embeddings - 1]`.

            [What are position IDs?](../glossary#position-ids)
        encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, encoder_sequence_length, hidden_size)`, *optional*):
            Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention of
            the decoder.
        encoder_attention_mask (`torch.LongTensor` of shape `(batch_size, encoder_sequence_length)`, *optional*):
            Mask to avoid performing cross-attention on padding tokens indices of encoder input_ids. Mask values
            selected in `[0, 1]`:

            - 1 for tokens that are **not masked**,
            - 0 for tokens that are **masked**.

            [What are attention masks?](../glossary#attention-mask)
        head_mask (`torch.Tensor` of shape `(num_layers, attention_heads)`, *optional*):
            Mask to nullify selected heads of the attention modules. Mask values selected in `[0, 1]`:

            - 1 indicates the head is **not masked**,
            - 0 indicates the head is **masked**.

        cross_attn_head_mask (`torch.Tensor` of shape `(num_layers, attention_heads)`, *optional*):
            Mask to nullify selected heads of the cross-attention modules. Mask values selected in `[0, 1]`:

            - 1 indicates the head is **not masked**,
            - 0 indicates the head is **masked**.

        past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
            Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape
            `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape
            `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`.

            Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention
            blocks) that can be used (see `past_key_values` input) to speed up sequential decoding.

            If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that
            don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all
            `decoder_input_ids` of shape `(batch_size, sequence_length)`.
        inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
            Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
            This is useful if you want more control over how to convert `input_ids` indices into associated vectors
            than the model's internal embedding lookup matrix.
        output_attentions (`bool`, *optional*):
            Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
            tensors for more detail.
        output_hidden_states (`bool`, *optional*):
            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
            more detail.
        return_dict (`bool`, *optional*):
            Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
c            
       ó`   ‡ — e Zd ZdZd	dedededee   fˆ fd„Zdej                  fˆ fd„Z
ˆ xZS )
ÚXGLMScaledWordEmbeddingz\
    This module overrides nn.Embeddings' forward by multiplying with embeddings scale.
    Únum_embeddingsÚembedding_dimÚpadding_idxÚembed_scalec                 ó6   •— t         ‰| �  |||«       || _        y ©N)ÚsuperÚ__init__r   )Úselfr   r   r   r   Ú	__class__s        €úd/var/www/skyplay_api_hub/venv/lib/python3.12/site-packages/transformers/models/xglm/modeling_xglm.pyr   z XGLMScaledWordEmbedding.__init__„   s   ø€ Ü‰Ñ˜¨¸ÔDØ&ˆÕó    Ú	input_idsc                 ó<   •— t         ‰| �  |«      | j                  z  S r   )r   Úforwardr   )r    r$   r!   s     €r"   r&   zXGLMScaledWordEmbedding.forwardˆ   s   ø€ Ü‰w‰˜yÓ)¨D×,<Ñ,<Ñ<Ð<r#   )ç      ð?)Ú__name__Ú
__module__Ú__qualname__Ú__doc__Úintr   Úfloatr   ÚtorchÚTensorr&   Ú__classcell__©r!   s   @r"   r   r      sE   ø„ ññ' sð '¸3ð 'ÈSð 'Ð_gÐhmÑ_nõ 'ð= §¡÷ =ñ =r#   r   c            	       óÎ   ‡ — e Zd ZdZddededee   fˆ fd„Zddededee   fd„Zeddededee   fd„«       Z	 e
j                  «       dd	ee
j                     d
efd„«       Zˆ xZS )Ú!XGLMSinusoidalPositionalEmbeddingzDThis module produces sinusoidal positional embeddings of any length.Únum_positionsr   r   c                 óŒ   •— t         ‰| �  «        d| _        || _        || _        | j                  || j                  z   ||«       y )Né   )r   r   Úoffsetr   r   Úmake_weights)r    r4   r   r   r!   s       €r"   r   z*XGLMSinusoidalPositionalEmbedding.__init__�   s@   ø€ Ü‰ÑÔØˆŒØ*ˆÔØ&ˆÔØ×Ñ˜-¨$¯+©+Ñ5°}ÀkÕRr#   r   c                 óà   — | j                  |||«      }t        | d«      r;|j                  | j                  j                  | j                  j
                  ¬«      }| j                  d|d¬«       y )NÚweights©ÚdtypeÚdeviceF)Ú
persistent)Úget_embeddingÚhasattrÚtor:   r<   r=   Úregister_buffer)r    r   r   r   Úemb_weightss        r"   r8   z.XGLMSinusoidalPositionalEmbedding.make_weights–   s[   € Ø×(Ñ(¨¸ÈÓTˆÜ�4˜Ô#à%Ÿ.™.¨t¯|©|×/AÑ/AÈ$Ï,É,×J]ÑJ]˜.Ó^ˆKà×Ñ˜Y¨ÀÐÕFr#   c                 óâ  — |dz  }t        j                  d«      |dz
  z  }t        j                  t        j                  |t        j
                  ¬«      j                  «       | z  «      }t        j                  | t        j
                  ¬«      j                  «       j                  d«      |j                  d«      z  }t        j                  t        j                  |«      t        j                  |«      gd¬«      j                  | d«      }|dz  dk(  r-t        j                  |t        j                  | d«      gd¬«      }|�	d||dd…f<   |j                  t        j                  «       «      S )	zÊ
        Build sinusoidal embeddings.

        This matches the implementation in tensor2tensor, but differs slightly from the description in Section 3.5 of
        "Attention Is All You Need".
        r6   i'  r   )r<   r   ©ÚdiméÿÿÿÿN)ÚmathÚlogr.   ÚexpÚarangeÚint64r-   Ú	unsqueezeÚcatÚsinÚcosÚviewÚzerosrA   Úget_default_dtype)r   r   r   Úhalf_dimÚembs        r"   r?   z/XGLMSinusoidalPositionalEmbedding.get_embeddingž   s  € ð ! AÑ%ˆÜ�h‰h�u‹o ¨A¡Ñ.ˆÜ�i‰iœŸ™ X´U·[±[ÔA×GÑGÓIÈSÈDÑPÓQˆÜ�l‰l˜>´·±Ô=×CÑCÓE×OÑOÐPQÓRÐUX×UbÑUbÐcdÓUeÑeˆÜ�i‰iœŸ™ 3›¬¯©°3«Ð8¸aÔ@×EÑEÀnÐVXÓYˆØ˜1Ñ Ò!ä—)‘)˜S¤%§+¡+¨n¸aÓ"@ÐAÀqÔIˆCØÐ"Ø"#ˆC�šQ�Ñà�v‰v”e×-Ñ-Ó/Ó0Ð0r#   Úposition_idsÚpast_key_values_lengthc                 ó¢  — |j                  «       \  }}|| j                  z  }d|z   |z   }|| j                  j                  d«      kD  r'| j                  || j                  | j
                  «       | j                  j                  d|j                  d«      «      j                  ||| j                  j                  d   «      j                  «       S )Nr6   r   rG   )
Úsizer7   r:   r8   r   r   Úindex_selectrQ   ÚshapeÚdetach)r    rV   rW   ÚbszÚseq_lenÚmax_poss         r"   r&   z)XGLMSinusoidalPositionalEmbedding.forward³   sµ   € à#×(Ñ(Ó*‰ˆˆWØ˜Ÿ™Ñ#ˆð �g‘+Ð 6Ñ6ˆØ�T—\‘\×&Ñ& qÓ)Ò)Ø×Ñ˜g t×'9Ñ'9¸4×;KÑ;KÔLà�|‰|×(Ñ(¨¨L×,=Ñ,=¸bÓ,AÓB×GÑGÈÈWÐVZ×VbÑVb×VhÑVhÐikÑVlÓm×tÑtÓvÐvr#   r   )Nr   )r(   r)   r*   r+   r,   r   r   r8   Ústaticmethodr?   r.   Úno_gradr/   r&   r0   r1   s   @r"   r3   r3   Œ   s´   ø„ ÙNñS cð S¸#ð SÈHÐUXÉMõ SñG¨3ð G¸sð GÐQYÐZ]ÑQ^ó Gð ñ1 cð 1¸#ð 1ÈHÐUXÉMò 1ó ð1ð( €U‡]�]ƒ_ñ	w H¨U¯\©\Ñ$:ð 	wÐ[^ò 	wó ô	wr#   r3   c                   ót  ‡ — e Zd ZdZ	 	 	 ddededededef
ˆ fd„Zdej                  d	ed
efd„Z
	 	 	 	 	 ddej                  deej                     deeej                        deej                     deej                     dedeej                  eej                     eeej                        f   fd„Zˆ xZS )ÚXGLMAttentionz=Multi-headed attention from 'Attention Is All You Need' paperÚ	embed_dimÚ	num_headsÚdropoutÚ
is_decoderÚbiasc                 óî  •— t         ‰| �  «        || _        || _        || _        ||z  | _        | j
                  |z  | j                  k7  rt        d| j                  › d|› d�«      ‚| j
                  dz  | _        || _        t        j                  |||¬«      | _        t        j                  |||¬«      | _        t        j                  |||¬«      | _        t        j                  |||¬«      | _        y )Nz;embed_dim must be divisible by num_heads (got `embed_dim`: z and `num_heads`: z).g      à¿©rh   )r   r   rd   re   rf   Úhead_dimÚ
ValueErrorÚscalingrg   r   ÚLinearÚk_projÚv_projÚq_projÚout_proj)r    rd   re   rf   rg   rh   r!   s         €r"   r   zXGLMAttention.__init__Ã   sÖ   ø€ ô 	‰ÑÔØ"ˆŒØ"ˆŒØˆŒØ! YÑ.ˆŒà�M‰M˜IÑ%¨$¯.©.Ò8ÜØMÈdÏnÉnÐM]Ø$ Y K¨rð3óð ð —}‘} dÑ*ˆŒØ$ˆŒä—i‘i 	¨9¸4Ô@ˆŒÜ—i‘i 	¨9¸4Ô@ˆŒÜ—i‘i 	¨9¸4Ô@ˆŒÜŸ	™	 )¨Y¸TÔBˆ�r#   Útensorr^   r]   c                 óŽ   — |j                  ||| j                  | j                  «      j                  dd«      j	                  «       S )Nr   r6   )rQ   re   rk   Ú	transposeÚ
contiguous)r    rs   r^   r]   s       r"   Ú_shapezXGLMAttention._shapeÞ   s7   € Ø�{‰{˜3 ¨¯©¸¿¹ÓG×QÑQÐRSÐUVÓW×bÑbÓdÐdr#   Úhidden_statesÚkey_value_statesÚpast_key_valueÚattention_maskÚlayer_head_maskÚoutput_attentionsÚreturnc                 óL  — |du}|j                  «       \  }}	}
| j                  |«      | j                  z  }|r|�|d   }|d   }�n
|rE| j                  | j	                  |«      d|«      }| j                  | j                  |«      d|«      }nÃ|�}| j                  | j	                  |«      d|«      }| j                  | j                  |«      d|«      }t        j                  |d   |gd¬«      }t        j                  |d   |gd¬«      }nD| j                  | j	                  |«      d|«      }| j                  | j                  |«      d|«      }| j                  r||f}|| j                  z  d| j                  f} | j                  ||	|«      j                  |Ž } |j                  |Ž } |j                  |Ž }|j                  d«      }t        j                  ||j                  dd«      «      }|j                  «       || j                  z  |	|fk7  r/t        d|| j                  z  |	|f› d|j                  «       › �«      ‚|�×|j                  «       |d|	|fk7  r#t        d	|d|	|f› d|j                  «       › �«      ‚|j                  || j                  |	|«      |z   }t        j                  |t        j                   t        j"                  |j$                  «      j&                  |j(                  ¬
«      «      }|j                  || j                  z  |	|«      }|j$                  t        j*                  k(  rNt,        j.                  j1                  |dt        j2                  ¬«      j5                  t        j*                  «      }n!t,        j.                  j1                  |d¬«      }|�›|j                  «       | j                  fk7  r*t        d| j                  f› d|j                  «       › �«      ‚|j                  dddd«      |j                  || j                  |	|«      z  }|j                  || j                  z  |	|«      }|r?|j                  || j                  |	|«      }|j                  || j                  z  |	|«      }nd}t,        j.                  j7                  || j6                  | j8                  ¬«      }t        j                  ||«      }|j                  «       || j                  z  |	| j                  fk7  r7t        d|| j                  |	| j                  f› d|j                  «       › �«      ‚|j                  || j                  |	| j                  «      }|j                  dd«      }|j;                  ||	| j<                  «      }| j?                  |«      }|||fS )z#Input shape: Batch x Time x ChannelNr   r   rG   r6   rE   z$Attention weights should be of size z	, but is z!Attention mask should be of size )r=   )rF   r<   z/Head mask for a single layer should be of size ©ÚpÚtrainingz `attn_output` should be of size ) rY   rq   rm   rw   ro   rp   r.   rN   rg   re   rk   rQ   Úbmmru   rl   Úmaxrs   Úfinfor<   Úminr=   Úfloat16r   Ú
functionalÚsoftmaxÚfloat32rA   rf   r‚   Úreshaperd   rr   )r    rx   ry   rz   r{   r|   r}   Úis_cross_attentionr]   Útgt_lenÚ_Úquery_statesÚ
key_statesÚvalue_statesÚ
proj_shapeÚsrc_lenÚattn_weightsÚattn_weights_reshapedÚ
attn_probsÚattn_outputs                       r"   r&   zXGLMAttention.forwardá   sV  € ð .°TÐ9Ðà'×,Ñ,Ó.‰ˆˆW�að —{‘{ =Ó1°D·L±LÑ@ˆá .Ð"<à'¨Ñ*ˆJØ)¨!Ñ,ŠLÙàŸ™ T§[¡[Ð1AÓ%BÀBÈÓLˆJØŸ;™; t§{¡{Ð3CÓ'DÀbÈ#ÓN‰LØÐ'àŸ™ T§[¡[°Ó%?ÀÀSÓIˆJØŸ;™; t§{¡{°=Ó'AÀ2ÀsÓKˆLÜŸ™ N°1Ñ$5°zÐ#BÈÔJˆJÜ Ÿ9™9 n°QÑ&7¸Ð%FÈAÔN‰Lð Ÿ™ T§[¡[°Ó%?ÀÀSÓIˆJØŸ;™; t§{¡{°=Ó'AÀ2ÀsÓKˆLà�?Š?ð )¨,Ð7ˆNà˜DŸN™NÑ*¨B°·±Ð>ˆ
ØC�t—{‘{ <°¸#Ó>×CÑCÀZÐPˆØ$�Z—_‘_ jÐ1ˆ
Ø(�|×(Ñ(¨*Ð5ˆà—/‘/ !Ó$ˆÜ—y‘y ¨z×/CÑ/CÀAÀqÓ/IÓJˆà×ÑÓ 3¨¯©Ñ#7¸À'Ð"JÒJÜØ6¸¸d¿n¹nÑ8LÈgÐW^Ð7_Ð6`ð aØ ×%Ñ%Ó'Ð(ð*óð ð
 Ð%Ø×"Ñ"Ó$¨¨a°¸'Ð(BÒBÜ Ø7¸¸aÀÈ'Ð8RÐ7SÐS\Ð]k×]pÑ]pÓ]rÐ\sÐtóð ð (×,Ñ,¨S°$·.±.À'È7ÓSÐVdÑdˆLÜ Ÿ9™9ØœeŸl™l¬5¯;©;°|×7IÑ7IÓ+J×+NÑ+NÐWc×WjÑWjÔkóˆLð (×,Ñ,¨S°4·>±>Ñ-AÀ7ÈGÓTˆLð ×Ñ¤§¡Ò.ÜŸ=™=×0Ñ0°À2ÌUÏ]É]Ð0Ó[×^Ñ^Ô_d×_lÑ_lÓm‰LäŸ=™=×0Ñ0°À2Ð0ÓFˆLàÐ&Ø×#Ñ#Ó%¨$¯.©.Ð):Ò:Ü ØEÀtÇ~Á~ÐFWÐEXð YØ'×,Ñ,Ó.Ð/ð1óð ð +×/Ñ/°°2°q¸!Ó<¸|×?PÑ?PÐQTÐVZ×VdÑVdÐfmÐovÓ?wÑwˆLØ'×,Ñ,¨S°4·>±>Ñ-AÀ7ÈGÓTˆLáð
 %1×$5Ñ$5°c¸4¿>¹>È7ÐT[Ó$\Ð!Ø0×5Ñ5°c¸D¿N¹NÑ6JÈGÐU\Ó]‰Là$(Ð!ä—]‘]×*Ñ*¨<¸4¿<¹<ÐRV×R_ÑR_Ð*Ó`ˆ
ä—i‘i 
¨LÓ9ˆà×ÑÓ #¨¯©Ñ"6¸ÀÇÁÐ!OÒOÜØ2°C¸¿¹ÈÐRV×R_ÑR_Ð3`Ð2að bØ×$Ñ$Ó&Ð'ð)óð ð
 "×&Ñ& s¨D¯N©N¸GÀTÇ]Á]ÓSˆØ!×+Ñ+¨A¨qÓ1ˆð "×)Ñ)¨#¨w¸¿¹ÓGˆà—m‘m KÓ0ˆàÐ1°>ÐAÐAr#   )ç        FT)NNNNF)r(   r)   r*   r+   r,   r-   Úboolr   r.   r/   rw   r   r   r&   r0   r1   s   @r"   rc   rc   À   s,  ø„ ÙGð Ø ØñCàðCð ðCð ð	Cð
 ðCð õCð6e˜UŸ\™\ð e°Cð e¸có eð 48Ø8<Ø15Ø26Ø"'ñvBà—|‘|ðvBð # 5§<¡<Ñ0ðvBð !  u§|¡|Ñ!4Ñ5ð	vBð
 ! §¡Ñ.ðvBð " %§,¡,Ñ/ðvBð  ðvBð 
ˆu�|‰|˜X e§l¡lÑ3°X¸eÀEÇLÁLÑ>QÑ5RÐRÑ	S÷vBr#   rc   c                   ó<  ‡ — e Zd Zdefˆ fd„Z	 	 	 	 	 	 	 	 ddej                  deej                     deej                     deej                     deej                     deej                     d	eeej                        d
ee	   dee	   dej                  fd„Z
ˆ xZS )ÚXGLMDecoderLayerÚconfigc                 ó,  •— t         ‰| �  «        |j                  | _        t	        | j                  |j
                  |j                  d¬«      | _        |j                  | _        t        |j                     | _        |j                  | _        |j                  rVt	        | j                  |j
                  |j                  d¬«      | _        t        j                   | j                  «      | _        t        j                   | j                  «      | _        t        j&                  | j                  |j(                  «      | _        t        j&                  |j(                  | j                  «      | _        t        j                   | j                  «      | _        y )NT)rd   re   rf   rg   )r   r   Úd_modelrd   rc   Úattention_headsÚattention_dropoutÚ	self_attnrf   r	   Úactivation_functionÚactivation_fnÚactivation_dropoutÚadd_cross_attentionÚencoder_attnr   Ú	LayerNormÚencoder_attn_layer_normÚself_attn_layer_normrn   Úffn_dimÚfc1Úfc2Úfinal_layer_norm©r    rœ   r!   s     €r"   r   zXGLMDecoderLayer.__init__[  s  ø€ Ü‰ÑÔØŸ™ˆŒä&Ø—n‘nØ×,Ñ,Ø×,Ñ,Øô	
ˆŒð —~‘~ˆŒÜ# F×$>Ñ$>Ñ?ˆÔØ"(×";Ñ";ˆÔà×%Ò%Ü -ØŸ.™.Ø ×0Ñ0Ø×0Ñ0Øô	!ˆDÔô ,.¯<©<¸¿¹Ó+GˆDÔ(ä$&§L¡L°·±Ó$@ˆÔ!Ü—9‘9˜TŸ^™^¨V¯^©^Ó<ˆŒÜ—9‘9˜VŸ^™^¨T¯^©^Ó<ˆŒÜ "§¡¨T¯^©^Ó <ˆÕr#   rx   r{   Úencoder_hidden_statesÚencoder_attention_maskr|   Úcross_attn_layer_head_maskrz   r}   Ú	use_cacher~   c
                 ót  — |}
| j                  |«      }|�|dd nd}| j                  |||||¬«      \  }}}t        j                  j	                  || j                  | j
                  ¬«      }|
|z   }d}d}|�w|}
| j                  |«      }|�|dd nd}| j                  ||||||¬«      \  }}}t        j                  j	                  || j                  | j
                  ¬«      }|
|z   }||z   }|}
| j                  |«      }| j                  | j                  |«      «      }t        j                  j	                  || j                  | j
                  ¬«      }| j                  |«      }t        j                  j	                  || j                  | j
                  ¬«      }|
|z   }|f}|r|||fz  }|	r||fz  }|S )a  
        Args:
            hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)`
            attention_mask (`torch.FloatTensor`): attention mask of size
                `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values.
            encoder_hidden_states (`torch.FloatTensor`):
                cross attention input to the layer of shape `(batch, seq_len, embed_dim)`
            encoder_attention_mask (`torch.FloatTensor`): encoder attention mask of size
                `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values.
            layer_head_mask (`torch.FloatTensor`): mask for attention heads in a given layer of size
                `(encoder_attention_heads,)`.
            cross_attn_layer_head_mask (`torch.FloatTensor`): mask for cross-attention heads in a given layer of
                size `(decoder_attention_heads,)`.
            past_key_value (`Tuple(torch.FloatTensor)`): cached past key and value projection states
            output_attentions (`bool`, *optional*):
                Whether or not to return the attentions tensors of all attention layers. See `attentions` under
                returned tensors for more detail.
        Nr6   )rx   rz   r{   r|   r}   r€   éþÿÿÿ)rx   ry   r{   r|   rz   r}   )r©   r¡   r   rˆ   rf   r‚   r¨   r¦   r­   r£   r«   r¤   r¬   )r    rx   r{   r¯   r°   r|   r±   rz   r}   r²   ÚresidualÚself_attn_past_key_valueÚself_attn_weightsÚpresent_key_valueÚcross_attn_present_key_valueÚcross_attn_weightsÚcross_attn_past_key_valueÚoutputss                     r"   r&   zXGLMDecoderLayer.forwardx  s  € ð< !ˆØ×1Ñ1°-Ó@ˆð :HÐ9S >°"°1Ñ#5ÐY]Ð à>B¿n¹nØ'Ø3Ø)Ø+Ø/ð ?Mó ?
Ñ;ˆÐ(Ð*;ô Ÿ™×-Ñ-¨m¸t¿|¹|ÐVZ×VcÑVcÐ-ÓdˆØ  =Ñ0ˆð (,Ð$Ø!ÐØ Ð,Ø$ˆHØ ×8Ñ8¸ÓGˆMð @NÐ?Y¨°r°sÑ(;Ð_cÐ%ØNR×N_ÑN_Ø+Ø!6Ø5Ø :Ø8Ø"3ð O`ó OÑKˆMÐ-Ð/Kô ŸM™M×1Ñ1°-À4Ç<Á<ÐZ^×ZgÑZgÐ1ÓhˆMØ$ }Ñ4ˆMð !2Ð4PÑ PÐð !ˆØ×-Ñ-¨mÓ<ˆØ×*Ñ*¨4¯8©8°MÓ+BÓCˆÜŸ™×-Ñ-¨m¸t×?VÑ?VÐae×anÑanÐ-ÓoˆØŸ™ Ó/ˆÜŸ™×-Ñ-¨m¸t¿|¹|ÐVZ×VcÑVcÐ-ÓdˆØ  =Ñ0ˆà Ð"ˆáØÐ)Ð+=Ð>Ñ>ˆGáØÐ)Ð+Ñ+ˆGàˆr#   )NNNNNNFT)r(   r)   r*   r   r   r.   r/   r   r   r™   r&   r0   r1   s   @r"   r›   r›   Z  sç   ø„ ð=˜zõ =ð@ 26Ø8<Ø9=Ø26Ø=AØ8<Ø,1Ø$(ñWà—|‘|ðWð ! §¡Ñ.ðWð  (¨¯©Ñ5ð	Wð
 !)¨¯©Ñ 6ðWð " %§,¡,Ñ/ðWð %-¨U¯\©\Ñ$:ðWð !  u§|¡|Ñ!4Ñ5ðWð $ D™>ðWð ˜D‘>ðWð 
�‰÷Wr#   r›   c                   ó$   — e Zd ZeZdZdZdgZd„ Zy)ÚXGLMPreTrainedModelÚmodelTr›   c                 ó  — | j                   j                  }t        |t        j                  «      rY|j
                  j                  j                  d|¬«       |j                  �%|j                  j                  j                  «        y y t        |t        j                  «      rf|j
                  j                  j                  d|¬«       |j                  �2|j
                  j                  |j                     j                  «        y y y )Nr˜   )ÚmeanÚstd)rœ   Úinit_stdÚ
isinstancer   rn   ÚweightÚdataÚnormal_rh   Úzero_Ú	Embeddingr   )r    ÚmodulerÂ   s      r"   Ú_init_weightsz!XGLMPreTrainedModel._init_weightsØ  sÃ   € Ø�k‰k×"Ñ"ˆÜ�fœbŸi™iÔ(Ø�M‰M×Ñ×&Ñ&¨C°SÐ&Ô9Ø�{‰{Ð&Ø—‘× Ñ ×&Ñ&Õ(ð 'ä˜¤§¡Ô-Ø�M‰M×Ñ×&Ñ&¨C°SÐ&Ô9Ø×!Ñ!Ð-Ø—‘×"Ñ" 6×#5Ñ#5Ñ6×<Ñ<Õ>ð .ð .r#   N)	r(   r)   r*   r   Úconfig_classÚbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesrË   © r#   r"   r¾   r¾   Ò  s!   „ Ø€LØÐØ&*Ð#Ø+Ð,Ðó	?r#   r¾   z^The bare XGLM Model transformer outputting raw hidden-states without any specific head on top.c            !       ó  ‡ — e Zd ZdZddedeej                     fˆ fd„Zd„ Z	d„ Z
 ee«       eeee¬«      	 	 	 	 	 	 	 	 	 	 	 	 	 ddeej$                     d	eej$                     d
eej$                     deej$                     deej$                     deej$                     deej$                     deeej(                        deej$                     dee   dee   dee   dee   deeej$                     ef   fd„«       «       Zˆ xZS )Ú	XGLMModelzÈ
    Transformer decoder consisting of *config.num_layers* layers. Each layer is a [`XGLMDecoderLayer`]

    Args:
        config: XGLMConfig
        embed_tokens (nn.Embedding): output embedding
    rœ   Úembed_tokensc                 óæ  •— t         ‰| �  |«       |j                  | _        |j                  | _        |j                  | _        |j                  | _        |j                  rt        j                  |j                  «      nd}|�|| _        n2t        |j                  |j                  | j
                  |¬«      | _        t        |j                  |j                  |j                  «      | _        t#        j$                  t'        |j(                  «      D �cg c]  }t+        |«      ‘Œ c}«      | _        t#        j.                  |j                  «      | _        d| _        | j5                  «        y c c}w )Nr'   )r   F)r   r   rf   Ú	layerdropÚpad_token_idr   Úmax_position_embeddingsÚmax_target_positionsÚscale_embeddingrH   Úsqrtrž   rÓ   r   Ú
vocab_sizer3   Úembed_positionsr   Ú
ModuleListÚrangeÚ
num_layersr›   Úlayersr§   Ú
layer_normÚgradient_checkpointingÚ	post_init)r    rœ   rÓ   r   rŽ   r!   s        €r"   r   zXGLMModel.__init__ñ  s  ø€ Ü‰Ñ˜Ô Ø—~‘~ˆŒØ×)Ñ)ˆŒØ!×.Ñ.ˆÔØ$*×$BÑ$BˆÔ!Ø39×3IÒ3I”d—i‘i §¡Ô/ÈsˆàÐ#Ø ,ˆDÕä 7Ø×!Ñ! 6§>¡>°4×3CÑ3CÐQ\ô!ˆDÔô  AØ×*Ñ*Ø�N‰NØ×Ñó 
ˆÔô
 —m‘mÄuÈV×M^ÑM^ÓG_Ö$`À!Ô%5°fÕ%=Ò$`ÓaˆŒÜŸ,™, v§~¡~Ó6ˆŒà&+ˆÔ#à�‰Õùò %as   ÄE.c                 ó   — | j                   S r   ©rÓ   ©r    s    r"   Úget_input_embeddingszXGLMModel.get_input_embeddings  s   € Ø× Ñ Ð r#   c                 ó   — || _         y r   rå   ©r    Úvalues     r"   Úset_input_embeddingszXGLMModel.set_input_embeddings  s
   € Ø!ˆÕr#   ©Ú
checkpointÚoutput_typerÌ   r$   r{   rV   r¯   r°   Ú	head_maskÚcross_attn_head_maskÚpast_key_valuesÚinputs_embedsr²   r}   Úoutput_hidden_statesÚreturn_dictr~   c                 óì  — |�|n| j                   j                  }|�|n| j                   j                  }|
�|
n| j                   j                  }
|�|n| j                   j                  }|�|	�t        d«      ‚|�8| j                  ||«       |j                  «       }|j                  d|d   «      }n!|	�|	j                  «       d d }nt        d«      ‚|�|d   d   j                  d   nd}|€Vt        j                  ||d   |z   t        j                  |�|j                  n|	j                  ¬«      }|j                  d«      }|	€| j                  |«      }	t!        |||	|«      }|�|�t#        ||	j$                  |d   ¬«      }|	| j'                  ||«      j)                  |	j                  «      z   }t*        j,                  j/                  |t1        | j.                  «      | j2                  ¬«      }| j4                  r%| j2                  r|
rt6        j9                  d	«       d
}
|rdnd }|rdnd }|r|�dnd }|
rdnd }t;        ||gddg«      D ]j  \  }}|€Œ	|j                  «       d   t=        | j>                  «      k7  sŒ3t        d|› dt=        | j>                  «      › d|j                  «       d   › d�«      ‚ tA        | j>                  «      D ]Ý  \  }}|r||fz  }| j2                  r%t        jB                  g «      }|| jD                  k  rŒ?|�||   nd }| j4                  r?| j2                  r3| jG                  |jH                  |||||�||   nd |�||   nd d ||
«
      }n ||||||�||   nd |�||   nd |||
¬«	      }|d   }|
r|||rdnd   fz  }|sŒÉ||d   fz  }|€ŒÕ||d   fz  }Œß | jK                  |«      }|r||fz  }|
r|nd }|stM        d„ |||||fD «       «      S tO        |||||¬«      S )NzDYou cannot specify both input_ids and inputs_embeds at the same timerG   z5You have to specify either input_ids or inputs_embedsr   r6   r;   )r�   r€   z_`use_cache = True` is incompatible with gradient checkpointing`. Setting `use_cache = False`...FrÐ   rï   rð   zThe `z` should be specified for z layers, but it is for ú.)r{   r¯   r°   r|   r±   rz   r}   r²   r   r   c              3   ó$   K  — | ]  }|�|–— Œ
 y ­wr   rÐ   )Ú.0Úvs     r"   ú	<genexpr>z$XGLMModel.forward.<locals>.<genexpr>§  s   è ø€ ò àØ�=ô ñùs   ‚)Úlast_hidden_staterñ   rx   Ú
attentionsÚcross_attentions)(rœ   r}   ró   r²   Úuse_return_dictrl   Ú%warn_if_padding_and_no_attention_maskrY   rQ   r[   r.   rK   Úlongr=   rM   rÓ   r   r   r<   rÜ   rA   r   rˆ   rf   r-   r‚   râ   ÚloggerÚwarning_onceÚzipÚlenrà   Ú	enumerateÚrandrÕ   Ú_gradient_checkpointing_funcÚ__call__rá   Útupler   )r    r$   r{   rV   r¯   r°   rï   rð   rñ   rò   r²   r}   ró   rô   Úinput_shaperW   rx   Úall_hidden_statesÚall_self_attnsÚall_cross_attentionsÚnext_decoder_cacheÚ	attn_maskÚ	mask_nameÚidxÚdecoder_layerÚdropout_probabilityrz   Úlayer_outputsÚ
next_caches                                r"   r&   zXGLMModel.forward  sÂ  € ð, 2CÐ1NÑ-ÐTX×T_ÑT_×TqÑTqÐà$8Ð$DÑ È$Ï+É+×JjÑJjð 	ð "+Ð!6‘I¸D¿K¹K×<QÑ<Qˆ	Ø%0Ð%<‘kÀ$Ç+Á+×B]ÑB]ˆð Ð  ]Ð%>ÜÐcÓdÐdØÐ"Ø×6Ñ6°yÀ.ÔQØ#Ÿ.™.Ó*ˆKØ!Ÿ™ r¨;°r©?Ó;‰IØÐ&Ø'×,Ñ,Ó.¨s°Ð3‰KäÐTÓUÐUàCRÐC^ °Ñ!3°AÑ!6×!<Ñ!<¸QÒ!?ÐdeÐàÐÜ Ÿ<™<Ø&Ø˜B‘Ð"8Ñ8Ü—j‘jØ+4Ð+@�y×'Ò'Àm×FZÑFZô	ˆLð (×1Ñ1°!Ó4ˆLàÐ Ø ×-Ñ-¨iÓ8ˆMä:Ø˜K¨Ð8Nó
ˆð
 !Ð,Ð1GÐ1Sä%?Ø&¨×(;Ñ(;À[ÐQSÁ_ô&Ð"ð &¨×(<Ñ(<¸\ÐKaÓ(b×(eÑ(eØ× Ñ ó)
ñ 
ˆô Ÿ™×-Ñ-¨m¼uÀTÇ\Á\Ó?RÐ]a×]jÑ]jÐ-Ókˆà×&Ò&¨4¯=ª=ÙÜ×#Ñ#Øuôð "�	ñ #7™B¸DÐÙ0™°dˆÙ&7Ð<QÐ<]™rÐdhÐÙ#,™R°$Ðô %(¨Ð4HÐ(IÈKÐYoÐKpÓ$qò 	Ñ ˆI�yØÑ$Ø—>‘>Ó# AÑ&¬#¨d¯k©kÓ*:Ó:Ü$Ø 	˜{Ð*DÄSÈÏÉÓEUÐDVð WØ%ŸN™NÓ,¨QÑ/Ð0°ð3óð ð	ô #,¨D¯K©KÓ"8ò /	@ÑˆC�á#Ø! mÐ%5Ñ5Ð!Ø�}Š}Ü&+§j¡j°£nÐ#Ø&¨¯©Ò7Øà5DÐ5P˜_¨SÒ1ÐVZˆNà×*Ò*¨t¯}ª}Ø $× AÑ AØ!×*Ñ*Ø!Ø"Ø)Ø*Ø&/Ð&;�I˜c’NÀØ1EÐ1QÐ(¨Ò-ÐW[ØØ%Øó!‘ñ !.Ø!Ø#1Ø*?Ø+AØ7@Ð7L Y¨s¢^ÐRVà5IÐ5UÐ,¨SÒ1Ð[_à#1Ø&7Ø'ô!�ð *¨!Ñ,ˆMáØ" }Ñ:K±QÐQRÑ'SÐ&UÑUÐ"â Ø =°Ñ#3Ð"5Ñ5�à(Ñ4Ø(¨]¸1Ñ-=Ð,?Ñ?Ñ(ð_/	@ðb Ÿ™¨Ó6ˆñ  Ø -Ð!1Ñ1Ðá+4Ñ'¸$ˆ
ÙÜñ à'¨Ð5FÈÐXlÐmôó ð ô
 9Ø+Ø&Ø+Ø%Ø1ô
ð 	
r#   r   )NNNNNNNNNNNNN)r(   r)   r*   r+   r   r   r   rÉ   r   rç   rë   r   ÚXGLM_INPUTS_DOCSTRINGr   Ú_CHECKPOINT_FOR_DOCr   Ú_CONFIG_FOR_DOCr.   r/   r   ÚFloatTensorr™   r   r   r&   r0   r1   s   @r"   rÒ   rÒ   ä  sž  ø„ ñ
ñ˜zð ¸À"Ç,Á,Ñ9Oõ ò6!ò"ñ +Ð+@ÓAÙØ&Ø=Ø$ôð -1Ø15Ø/3Ø8<Ø9=Ø,0Ø7;Ø=AØ04Ø$(Ø,0Ø/3Ø&*ñZ
à˜EŸL™LÑ)ðZ
ð ! §¡Ñ.ðZ
ð ˜uŸ|™|Ñ,ð	Z
ð
  (¨¯©Ñ5ðZ
ð !)¨¯©Ñ 6ðZ
ð ˜EŸL™LÑ)ðZ
ð ' u§|¡|Ñ4ðZ
ð " $ u×'8Ñ'8Ñ"9Ñ:ðZ
ð   §¡Ñ-ðZ
ð ˜D‘>ðZ
ð $ D™>ðZ
ð ' t™nðZ
ð ˜d‘^ðZ
ð 
ˆu�U—\‘\Ñ"Ð$MÐMÑ	NòZ
óó BôZ
r#   rÒ   z‡
    The XGLM Model transformer with a language modeling head on top (linear layer with weights tied to the input
    embeddings).
    c            #       ó*  ‡ — e Zd ZdZdgZˆ fd„Zd„ Zd„ Zd„ Zd„ Z	 e
e«       eeee¬«      	 	 	 	 	 	 	 	 	 	 	 	 	 	 dd	eej$                     d
eej$                     deej$                     deej$                     deej$                     deej$                     deej$                     deeej(                        deej$                     deej$                     dee   dee   dee   dee   deeej$                     ef   fd„«       «       Zed„ «       Zˆ xZS )ÚXGLMForCausalLMr¿   zlm_head.weightc                 óÆ   •— t         ‰| �  |«       t        |«      | _        t	        j
                  |j                  |j                  d¬«      | _        | j                  «        y )NFrj   )
r   r   rÒ   r¿   r   rn   Úhidden_sizerÛ   Úlm_headrã   r®   s     €r"   r   zXGLMForCausalLM.__init__À  sH   ø€ Ü‰Ñ˜Ô Ü˜vÓ&ˆŒ
Ü—y‘y ×!3Ñ!3°V×5FÑ5FÈUÔSˆŒð 	�‰Õr#   c                 ó.   — | j                   j                  S r   ©r¿   rÓ   ræ   s    r"   rç   z$XGLMForCausalLM.get_input_embeddingsÈ  s   € Ø�z‰z×&Ñ&Ð&r#   c                 ó&   — || j                   _        y r   r   ré   s     r"   rë   z$XGLMForCausalLM.set_input_embeddingsË  s   € Ø"'ˆ�
‰
Õr#   c                 ó   — | j                   S r   ©r  ræ   s    r"   Úget_output_embeddingsz%XGLMForCausalLM.get_output_embeddingsÎ  s   € Ø�|‰|Ðr#   c                 ó   — || _         y r   r#  )r    Únew_embeddingss     r"   Úset_output_embeddingsz%XGLMForCausalLM.set_output_embeddingsÑ  s	   € Ø%ˆ�r#   rì   r$   r{   rV   r¯   r°   rï   rð   rñ   rò   Úlabelsr²   r}   ró   rô   r~   c                 ó$  — |�|n| j                   j                  }|�|n| j                   j                  }|�|n| j                   j                  }| j	                  |||||||||	||||¬«      }| j                  |d   «      }d}|
�? | j                  ||
f| j                   j                  | j                   j                  dœ|¤Ž}|s|f|dd z   }|�|f|z   S |S t        |||j                  |j                  |j                  |j                  ¬«      S )a­  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
        N)r$   r{   rV   r¯   r°   rï   rð   rñ   rò   r²   r}   ró   rô   r   )rÛ   rÖ   r   )ÚlossÚlogitsrñ   rx   rü   rý   )rœ   r}   ró   rþ   r¿   r  Úloss_functionrÛ   rÖ   r   rñ   rx   rü   rý   )r    r$   r{   rV   r¯   r°   rï   rð   rñ   rò   r(  r²   r}   ró   rô   Úkwargsr¼   r+  r*  Úoutputs                       r"   r&   zXGLMForCausalLM.forwardÔ  sU  € ð> 2CÐ1NÑ-ÐTX×T_ÑT_×TqÑTqÐà$8Ð$DÑ È$Ï+É+×JjÑJjð 	ð &1Ð%<‘kÀ$Ç+Á+×B]ÑB]ˆð —*‘*ØØ)Ø%Ø"7Ø#9ØØ!5Ø+Ø'ØØ/Ø!5Ø#ð ó 
ˆð  —‘˜g a™jÓ)ˆàˆØÐØ%�4×%Ñ%ØØðð  Ÿ;™;×1Ñ1Ø!Ÿ[™[×5Ñ5ñ	ð
 ñˆDñ Ø�Y ¨¨ Ñ,ˆFØ'+Ð'7�D�7˜VÑ#ÐC¸VÐCä0ØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)Ø$×5Ñ5ô
ð 	
r#   c                 óJ   ‡— d}| D ]  }|t        ˆfd„|D «       «      fz  }Œ |S )NrÐ   c              3   ót   •K  — | ]/  }|j                  d ‰j                  |j                  «      «      –— Œ1 y­w)r   N)rZ   rA   r=   )rø   Ú
past_stateÚbeam_idxs     €r"   rú   z1XGLMForCausalLM._reorder_cache.<locals>.<genexpr>(  s.   øè ø€ ÒnÐU_�j×-Ñ-¨a°·±¸Z×=NÑ=NÓ1O×PÑnùs   ƒ58)r	  )rñ   r2  Úreordered_pastÚ
layer_pasts    `  r"   Ú_reorder_cachezXGLMForCausalLM._reorder_cache#  s=   ø€ àˆØ)ò 	ˆJØÜÓnÐcmÔnÓnðñ ‰Nð	ð Ðr#   )NNNNNNNNNNNNNN)r(   r)   r*   rÍ   Ú_tied_weights_keysr   rç   rë   r$  r'  r   r  r   r  r   r  r   r.   r/   r   r  r™   r   r   r&   r`   r5  r0   r1   s   @r"   r  r  µ  sÉ  ø„ ð  ÐØ*Ð+Ðôò'ò(òò&ñ +Ð+@ÓAÙØ&Ø5Ø$ôð -1Ø15Ø/3Ø8<Ø9=Ø,0Ø7;Ø=AØ04Ø)-Ø$(Ø,0Ø/3Ø&*ñG
à˜EŸL™LÑ)ðG
ð ! §¡Ñ.ðG
ð ˜uŸ|™|Ñ,ð	G
ð
  (¨¯©Ñ5ðG
ð !)¨¯©Ñ 6ðG
ð ˜EŸL™LÑ)ðG
ð ' u§|¡|Ñ4ðG
ð " $ u×'8Ñ'8Ñ"9Ñ:ðG
ð   §¡Ñ-ðG
ð ˜Ÿ™Ñ&ðG
ð ˜D‘>ðG
ð $ D™>ðG
ð ' t™nðG
ð ˜d‘^ðG
ð" 
ˆu�U—\‘\Ñ"Ð$EÐEÑ	Fò#G
óó BðG
ðR ñó ôr#   r  )r  rÒ   r¾   ).r+   rH   Útypingr   r   r   r   r.   Útorch.utils.checkpointr   Úactivationsr	   Ú
generationr
   Úmodeling_attn_mask_utilsr   r   Úmodeling_outputsr   r   Úmodeling_utilsr   Úutilsr   r   r   r   Úconfiguration_xglmr   Ú
get_loggerr(   r  r  r  ÚXGLM_START_DOCSTRINGr  rÉ   r   ÚModuler3   rc   r›   r¾   rÒ   r  Ú__all__rÐ   r#   r"   ú<module>rD     s   ðñ ã ß /Ó /ã Û Ý å !Ý )ß eß lÝ -ß uÓ uÝ *ð 
ˆ×	Ñ	˜HÓ	%€à*Ð Ø€ðÐ ð DÐ ôP
=˜bŸl™lô 
=ô1w¨¯	©	ô 1wôhWB�B—I‘Iô WBôtu�r—y‘yô uôp?˜/ô ?ñ$ ØdØóôJ
Ð#ó J
ó	ðJ
ñZ ðð óônÐ)¨?ó nóðnòb B�r#   