Ë
    S^(hø�  ã                   ó8  — d Z ddlZddlmZmZmZmZ ddlZddlm	c m
Z ddlZddlm	Z	 ddlmZ ddlmZ ddlmZ dd	lmZmZ dd
lmZ ddlmZmZmZmZ ddlmZ  ej>                  e «      Z!dZ"dZ# G d„ de	jH                  «      Z% G d„ de	jH                  «      Z& G d„ de	jH                  «      Z' G d„ de	jH                  «      Z( G d„ de	jH                  «      Z) G d„ de	jH                  «      Z* G d„ de	jH                  «      Z+ G d„ de	jH                  «      Z, G d „ d!e	jH                  «      Z- G d"„ d#e	jH                  «      Z. G d$„ d%e	jH                  «      Z/ G d&„ d'e«      Z0d(Z1d)Z2 ed*e1«       G d+„ d,e0«      «       Z3 ed-e1«       G d.„ d/e0e«      «       Z4g d0¢Z5y)1zPyTorch CPMAnté    N)ÚListÚOptionalÚTupleÚUnion)Únn)ÚCrossEntropyLossé   )ÚACT2FN)ÚGenerationMixin)ÚBaseModelOutputWithPastÚCausalLMOutputWithPast)ÚPreTrainedModel)Úadd_code_sample_docstringsÚadd_start_docstringsÚ%add_start_docstrings_to_model_forwardÚloggingé   )ÚCpmAntConfigzopenbmb/cpm-ant-10br   c                   óH   ‡ — e Zd ZdZdefˆ fd„Zdej                  fd„Zˆ xZ	S )ÚCpmAntLayerNormzv
    We use Root Mean Square (RMS) Layer Normalization, please see https://arxiv.org/abs/1910.07467 for details."
    Úconfigc                 óÔ   •— t         ‰| �  «        |j                  | _        |j                  | _        t        j                  t        j                  |j                  «      «      | _	        y ©N)
ÚsuperÚ__init__ÚepsÚhidden_sizeÚdim_normr   Ú	ParameterÚtorchÚemptyÚweight©Úselfr   Ú	__class__s     €úh/var/www/skyplay_api_hub/venv/lib/python3.12/site-packages/transformers/models/cpmant/modeling_cpmant.pyr   zCpmAntLayerNorm.__init__-   sE   ø€ Ü‰ÑÔà—:‘:ˆŒØ×*Ñ*ˆŒÜ—l‘l¤5§;¡;¨v×/AÑ/AÓ#BÓCˆ�ó    Úhidden_statesc                 óp  — |j                  d«      | j                  k7  rt        d«      ‚|j                  }|j	                  t
        j                  «      j                  d«      j                  dd¬«      }|t        j                  || j                  z   «      z  j	                  |«      | j                  z  }|S )úf
        Args:
            hidden_states (`torch.Tensor` of shape `(batch, seq_len, dim_in)`)
        éÿÿÿÿz'hidden_states.size(-1) != self.dim_normé   T)ÚdimÚkeepdim)Úsizer   ÚAssertionErrorÚdtypeÚtor    Úfloat32ÚpowÚmeanÚrsqrtr   r"   )r$   r(   Ú	old_dtypeÚvariances       r&   ÚforwardzCpmAntLayerNorm.forward4   sš   € ð
 ×Ñ˜bÓ! T§]¡]Ò2Ü Ð!JÓKÐKØ!×'Ñ'ˆ	Ø ×#Ñ#¤E§M¡MÓ2×6Ñ6°qÓ9×>Ñ>À2ÈtÐ>ÓTˆØ&¬¯©°XÀÇÁÑ5HÓ)IÑI×MÑMÈiÓXÐ[_×[fÑ[fÑfˆØÐr'   )
Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r   r    ÚTensorr9   Ú__classcell__©r%   s   @r&   r   r   (   s&   ø„ ñðD˜|õ Dð
 U§\¡\÷ 
r'   r   c                   óä   ‡ — e Zd Zdefˆ fd„Z	 	 	 ddej                  dej                  dej                  dej                  dee	   dee
ej                  ej                  f      d	ee	   fd
„Zˆ xZS )ÚCpmAntAttentionr   c                 óH  •— t         ‰| �  «        |j                  | _        |j                  | _        |j                  | _        t        j                  | j                  | j
                  | j                  z  d¬«      | _	        t        j                  | j                  | j
                  | j                  z  d¬«      | _
        t        j                  | j                  | j
                  | j                  z  d¬«      | _        t        j                  | j
                  | j                  z  | j                  d¬«      | _        t        j                  j                  d¬«      | _        |j                   �0t        j                  j#                  |j                   ¬«      | _        y d | _        y )NF©Úbiasr+   ©r-   )Úp)r   r   r   Ú	dim_modelÚnum_attention_headsÚ	num_headsÚdim_headr   ÚLinearÚ	project_qÚ	project_kÚ	project_vÚattention_outr    ÚSoftmaxÚsoftmaxÚ	dropout_pÚDropoutÚdropoutr#   s     €r&   r   zCpmAntAttention.__init__B   s  ø€ Ü‰ÑÔØ×+Ñ+ˆŒØ×3Ñ3ˆŒØŸ™ˆŒäŸ™ 4§>¡>°4·>±>ÀDÇMÁMÑ3QÐX]Ô^ˆŒÜŸ™ 4§>¡>°4·>±>ÀDÇMÁMÑ3QÐX]Ô^ˆŒÜŸ™ 4§>¡>°4·>±>ÀDÇMÁMÑ3QÐX]Ô^ˆŒäŸY™Y t§~¡~¸¿¹Ñ'EÀtÇ~Á~Ð\aÔbˆÔä—x‘x×'Ñ'¨BÐ'Ó/ˆŒà×ÑÐ'Ü Ÿ8™8×+Ñ+¨f×.>Ñ.>Ð+Ó?ˆD�LàˆD�Lr'   Úhidden_qÚ	hidden_kvÚattention_maskÚposition_biasÚoutput_attentionsÚpast_key_valuesÚ	use_cachec           	      óÂ  — |j                  d«      }|j                  d«      }	|j                  d«      }
| j                  |«      }| j                  |«      }| j                  |«      }|j	                  ||	| j
                  | j                  «      j                  dddd«      }|j	                  ||
| j
                  | j                  «      j                  dddd«      }|j	                  ||
| j
                  | j                  «      j                  dddd«      }|�It        j                  |d   |gd¬«      }t        j                  |d   |gd¬«      }|j                  d«      }
t        j                  ||j                  dd«      «      t        j                  | j                  «      z  }||z   }t        j                  ||j	                  |d|	|
«      t        j                  d	«      k(  t        j                   t#        d
«      |j$                  |j&                  ¬«      «      }| j)                  |«      }t        j                  ||j	                  |d|	|
«      t        j                  d	«      k(  t        j                   d|j$                  |j&                  ¬«      «      }|r|}nd}| j*                  �| j+                  |«      }t        j                  ||«      }|j	                  || j
                  |	| j                  «      j                  dddd«      }|j-                  «       j	                  ||	| j
                  | j                  z  «      }| j/                  |«      }d}|r||f}|||fS )a€  
        Args:
            hidden_q (`torch.Tensor`):
                Input of transformer block(self-attention block). It can be the raw embedding of a batch of sequences.
            hidden_kv (`torch.Tensor` of shape `(batch, len_k, dim_model)`)):
                Tensor *key_value* and *query* of shape `(batch, len_k, dim_model)`
            attention_mask (`torch.Tensor` of shape `(batch, len_seq, len_seq)`):
                Avoid invalid areas to participate in the calculation of self-attention.
            position_bias (`torch.Tensor` of shape `(batch, len_seq, len_seq)`):
                Provide positional information to self-attention block.
            output_attentions (`bool`, *optional*):
                Whether or not to return the attentions tensors of all attention layers.
            past_key_values (`Tuple[torch.Tensor, torch.Tensor]`, *optional*):
                Cached past key and value projection states.
            use_cache (`bool`, *optional*):
                If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding
                (see `past_key_values`).
        r   r   r,   r	   NéþÿÿÿrF   r+   Fz-inf)Údevicer1   )r/   rM   rN   rO   ÚviewrJ   rK   Úpermuter    ÚcatÚmatmulÚ	transposeÚmathÚsqrtÚmasked_fillÚtensorÚscalar_tensorÚfloatr_   r1   rR   rU   Ú
contiguousrP   )r$   rV   rW   rX   rY   rZ   r[   r\   Ú
batch_sizeÚlen_qÚlen_kÚqueryÚkeyÚvalueÚscoreÚattn_weightss                   r&   r9   zCpmAntAttention.forwardU   sß  € ð8 —]‘] 1Ó%ˆ
Ø—‘˜aÓ ˆØ—‘˜qÓ!ˆà—‘˜xÓ(ˆØ�n‰n˜YÓ'ˆØ—‘˜yÓ)ˆà—
‘
˜: u¨d¯n©n¸d¿m¹mÓL×TÑTÐUVÐXYÐ[\Ð^_Ó`ˆØ�h‰h�z 5¨$¯.©.¸$¿-¹-ÓH×PÑPÐQRÐTUÐWXÐZ[Ó\ˆØ—
‘
˜: u¨d¯n©n¸d¿m¹mÓL×TÑTÐUVÐXYÐ[\Ð^_Ó`ˆàÐ&Ü—)‘)˜_¨QÑ/°Ð5¸2Ô>ˆCÜ—I‘I˜¨qÑ1°5Ð9¸rÔBˆEØ—H‘H˜R“LˆEô —‘˜U C§M¡M°"°bÓ$9Ó:¼T¿Y¹YÀtÇ}Á}Ó=UÑUˆØ˜Ñ%ˆä×!Ñ!ØØ×Ñ 
¨A¨u°eÓ<ÄÇÁÈUÓ@SÑSÜ×Ñ¤ f£°e·l±lÈ%Ï+É+ÔVó
ˆð
 —‘˜UÓ#ˆä×!Ñ!ØØ×Ñ 
¨A¨u°eÓ<ÄÇÁÈUÓ@SÑSÜ×Ñ ¨%¯,©,¸e¿k¹kÔJó
ˆñ
 Ø ‰LàˆLà�<‰<Ð#Ø—L‘L Ó'ˆEô —‘˜U EÓ*ˆà—
‘
˜: t§~¡~°u¸d¿m¹mÓL×TÑTÐUVÐXYÐ[\Ð^_Ó`ˆØ× Ñ Ó"×'Ñ'¨
°E¸4¿>¹>ÈDÏMÉMÑ;YÓZˆà×"Ñ" 5Ó)ˆàˆÙØ" E˜lˆOà�l OÐ3Ð3r'   )FNN)r:   r;   r<   r   r   r    r>   Ú
BoolTensorr   Úboolr   r9   r?   r@   s   @r&   rB   rB   A   sœ   ø„ ð ˜|õ  ð2 -2ØGKØ$(ñQ4à—,‘,ðQ4ð —<‘<ðQ4ð ×(Ñ(ð	Q4ð
 —|‘|ðQ4ð $ D™>ðQ4ð " %¨¯©°e·l±lÐ(BÑ"CÑDðQ4ð ˜D‘>÷Q4r'   rB   c                   óÔ   ‡ — e Zd Zdefˆ fd„Z	 	 	 	 d
dej                  dej                  deej                     dee   dee	ej                  ej                  f      dee   fd	„Z
ˆ xZS )ÚCpmAntSelfAttentionBlockr   c                 óæ   •— t         ‰| �  «        t        |«      | _        t	        |«      | _        |j                  r/t        j                  j                  |j                  «      | _
        y d | _
        y r   )r   r   r   Úlayernorm_before_attentionrB   Úself_attentionrS   r    r   rT   rU   r#   s     €r&   r   z!CpmAntSelfAttentionBlock.__init__ª   sT   ø€ Ü‰ÑÔÜ*9¸&Ó*AˆÔ'Ü-¨fÓ5ˆÔØ×ÒÜ Ÿ8™8×+Ñ+¨F×,<Ñ,<Ó=ˆD�LàˆD�Lr'   r(   rX   rY   rZ   r[   r\   c           	      ó¬   — | j                  |«      }| j                  |||||||«      }|\  }}}	| j                  �| j                  |«      }||z   }|||	fS )a  
        Args:
            hidden_states (`torch.Tensor` of shape `(batch, len_seq, dim_model)`):
                Input of transformer block(self-attention block). It can be the raw embedding of a batch of sequences.
            attention_mask (`torch.Tensor` of shape `(batch, len_seq, len_seq)`):
                Avoid invalid areas to participate in the calculation of self-attention.
            position_bias (`torch.Tensor` of shape `(batch, len_seq, len_seq)`):
                Provide positional information to self-attention block.
            output_attentions (`bool`, *optional*):
                Whether or not to return the attentions tensors of all attention layers.
            past_key_values (`Tuple(torch.FloatTensor)`, *optional*):
                Cached past key and value projection states.
            use_cache (`bool`, *optional*):
                If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding
                (see `past_key_values`).
        )ry   rz   rU   )
r$   r(   rX   rY   rZ   r[   r\   Úoutputsrs   Úcurrent_key_values
             r&   r9   z CpmAntSelfAttentionBlock.forward³   su   € ð2 ×1Ñ1°-Ó@ˆØ×%Ñ%Ø�W˜n¨mÐ=NÐP_Ðajó
ˆð 4;Ñ0ˆ�Ð0à�<‰<Ð#Ø—l‘l 7Ó+ˆGØ%¨Ñ/ˆà˜lÐ,=Ð=Ð=r'   ©NFNN©r:   r;   r<   r   r   r    r>   r   ru   r   r9   r?   r@   s   @r&   rw   rw   ©   sŒ   ø„ ð ˜|õ  ð 15Ø,1ØGKØ$(ñ$>à—|‘|ð$>ð Ÿ™ð$>ð   §¡Ñ-ð	$>ð
 $ D™>ð$>ð " %¨¯©°e·l±lÐ(BÑ"CÑDð$>ð ˜D‘>÷$>r'   rw   c                   óD   ‡ — e Zd Zdefˆ fd„Zdej                  fd„Zˆ xZS )ÚCpmAntDenseGatedACTr   c                 ó,  •— t         ‰| �  «        t        j                  |j                  |j
                  d¬«      | _        t        j                  |j                  |j
                  d¬«      | _        t        j                  j                  «       | _
        y ©NFrD   )r   r   r   rL   r   Údim_ffÚw_0Úw_1r    ÚGELUÚactr#   s     €r&   r   zCpmAntDenseGatedACT.__init__Û   s[   ø€ Ü‰ÑÔÜ—9‘9˜V×/Ñ/°·±ÀUÔKˆŒÜ—9‘9˜V×/Ñ/°·±ÀUÔKˆŒÜ—8‘8—=‘=“?ˆ�r'   r(   c                 ór   — | j                  | j                  |«      «      }| j                  |«      }||z  }|S )z¼Transform an input tensor from one feature space to another via a nonlinear operation

        Args:
            hidden_states (`torch.Tensor` of shape `(batch, seq_len, dim_in)`)
        )rˆ   r…   r†   )r$   r(   Ú
gate_scores      r&   r9   zCpmAntDenseGatedACT.forwardá   s9   € ð —X‘X˜dŸh™h }Ó5Ó6ˆ
ØŸ™ Ó/ˆà" ]Ñ2ˆØÐr'   ©	r:   r;   r<   r   r   r    r>   r9   r?   r@   s   @r&   r�   r�   Ú   s   ø„ ð#˜|õ #ð
 U§\¡\÷ 
r'   r�   c                   óD   ‡ — e Zd Zdefˆ fd„Zdej                  fd„Zˆ xZS )ÚCpmAntFeedForwardr   c                 ó(  •— t         ‰| �  «        t        |«      | _        |j                  �/t
        j                  j                  |j                  «      | _        nd | _        t        j                  |j                  |j                  d¬«      | _        y rƒ   )r   r   r�   Úw_inrS   r    r   rT   rU   rL   r„   r   Úw_outr#   s     €r&   r   zCpmAntFeedForward.__init__ï   sg   ø€ Ü‰ÑÔÜ'¨Ó/ˆŒ	Ø×ÑÐ'Ü Ÿ8™8×+Ñ+¨F×,<Ñ,<Ó=ˆD�LàˆDŒLä—Y‘Y˜vŸ}™}¨f×.@Ñ.@ÀuÔMˆ�
r'   r(   c                 ó„   — | j                  |«      }| j                  �| j                  |«      }| j                  |«      }|S )r*   )r�   rU   r�   ©r$   r(   s     r&   r9   zCpmAntFeedForward.forwardù   s>   € ð
 Ÿ	™	 -Ó0ˆà�<‰<Ð#Ø ŸL™L¨Ó7ˆMàŸ
™
 =Ó1ˆàÐr'   r‹   r@   s   @r&   r�   r�   î   s!   ø„ ðN˜|õ Nð U§\¡\÷ r'   r�   c                   óD   ‡ — e Zd Zdefˆ fd„Zdej                  fd„Zˆ xZS )ÚCpmAntFFNBlockr   c                 óæ   •— t         ‰| �  «        t        |«      | _        t	        |«      | _        |j                  r/t        j                  j                  |j                  «      | _
        y d | _
        y r   )r   r   r   Úlayernorm_before_ffnr�   ÚffnrS   r    r   rT   rU   r#   s     €r&   r   zCpmAntFFNBlock.__init__	  sS   ø€ Ü‰ÑÔÜ$3°FÓ$;ˆÔ!Ü$ VÓ,ˆŒØ×ÒÜ Ÿ8™8×+Ñ+¨F×,<Ñ,<Ó=ˆD�LàˆD�Lr'   r(   c                 óŽ   — | j                  |«      }| j                  |«      }| j                  �| j                  |«      }||z   }|S )z£
        Args:
            hidden_states (`torch.Tensor` of shape `(batch, len_seq, dim_model)`):
                Hidden states before feed forward layer.
        )r–   r—   rU   )r$   r(   Ú
ln_outputsr|   s       r&   r9   zCpmAntFFNBlock.forward  sJ   € ð ×.Ñ.¨}Ó=ˆ
Ø—(‘(˜:Ó&ˆØ�<‰<Ð#Ø—l‘l 7Ó+ˆGØ%¨Ñ/ˆØÐr'   r‹   r@   s   @r&   r”   r”     s    ø„ ð ˜|õ  ðà—|‘|÷r'   r”   c                   óÔ   ‡ — e Zd Zdefˆ fd„Z	 	 	 	 d
dej                  dej                  deej                     dee   dee	ej                  ej                  f      dee   fd	„Z
ˆ xZS )ÚCpmAntTransformerBlockr   c                 ób   •— t         ‰| �  «        t        |«      | _        t	        |«      | _        y r   )r   r   rw   Úself_attr”   r—   r#   s     €r&   r   zCpmAntTransformerBlock.__init__$  s&   ø€ Ü‰ÑÔÜ0°Ó8ˆŒÜ! &Ó)ˆ�r'   r(   rX   rY   rZ   r[   r\   c                 óh   — | j                  ||||||¬«      }|\  }}}| j                  |«      }|||fS )a¤  
        Args:
            hidden_states (`torch.Tensor`):
                Input to the layer of shape `(batch, seq_len, dim_model)`
            attention_mask (`torch.Tensor`):
                Avoid invalid areas to participate in the calculation of shape `(batch, seq_len, seq_len)`
            position_bias (`torch.Tensor`):
                Provides position information to attention mechanism of shape `(num_heads, seq_len, seq_len)`
            output_attentions (`bool`, *optional*):
                Whether or not to return the attentions tensors of all attention layers.
            past_key_values (`Tuple[torch.Tensor, torch.Tensor])`, *optional*):
                Cached past key and value projection states
            use_cache (`bool`, *optional*):
                If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding
                (see `past_key_values`).
        )rX   rY   rZ   r[   r\   )r�   r—   )	r$   r(   rX   rY   rZ   r[   r\   rs   r}   s	            r&   r9   zCpmAntTransformerBlock.forward)  sU   € ð2 Ÿ™ØØ)Ø'Ø/Ø+Øð &ó 
ˆð :GÑ6ˆ�|Ð%6àŸ™ Ó/ˆà˜lÐ,=Ð=Ð=r'   r~   r   r@   s   @r&   r›   r›   #  sŒ   ø„ ð*˜|õ *ð 15Ø,1ØGKØ$(ñ&>à—|‘|ð&>ð Ÿ™ð&>ð   §¡Ñ-ð	&>ð
 $ D™>ð&>ð " %¨¯©°e·l±lÐ(BÑ"CÑDð&>ð ˜D‘>÷&>r'   r›   c                   óØ   ‡ — e Zd Zdefˆ fd„Z	 	 	 	 ddej                  dej                  dej                  dee   dee   dee	ej                  ej                  f      d	ee   fd
„Z
ˆ xZS )ÚCpmAntEncoderr   c                 óö   •— t         ‰| �  «        |j                  | _        t	        j
                  t        | j                  «      D �cg c]  }t        |«      ‘Œ c}«      | _        t        |«      | _
        y c c}w r   )r   r   Únum_hidden_layersÚ
num_layersr   Ú
ModuleListÚranger›   Úlayersr   Úoutput_layernorm)r$   r   Úithr%   s      €r&   r   zCpmAntEncoder.__init__S  s[   ø€ Ü‰ÑÔØ ×2Ñ2ˆŒÜ—m‘mÌuÐUY×UdÑUdÓOeÖ$fÈÔ%;¸FÕ%CÒ$fÓgˆŒä /°Ó 7ˆÕùò %gs   ÁA6r(   rX   rY   rZ   Úoutput_hidden_statesr[   r\   c           	      ó  — |rdnd}|rdnd}	|rdnd}
t        | j                  «      D ]9  \  }}|r||fz  } ||||||r||   nd|¬«      }|\  }}}|r|	|fz  }	|€Œ4|
|fz   }
Œ; | j                  |«      }|r||fz  }||
||	fS )a%  
        Args:
            hidden_states (`torch.Tensor`):
                Input to the layer of shape `(batch, seq_len, dim_model)`
            attention_mask (`torch.Tensor`):
                Avoid invalid areas to participate in the calculation of shape `(batch, seq_len, seq_len)`
            position_bias (`torch.Tensor`):
                Provides position information to attention mechanism of shape `(num_heads, seq_len, seq_len)`
            output_attentions (`bool`, *optional*):
                Whether or not to return the attentions tensors of all attention layers.
            output_hidden_states (`bool`, *optional*):
                Whether or not to return the hidden states of all layers.
            past_key_values (`Tuple[torch.Tensor, torch.Tensor])`, *optional*):
                Cached past key and value projection states
            use_cache (`bool`, *optional*):
                If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding
                (see `past_key_values`).
        © N)rZ   r[   r\   )Ú	enumerater¦   r§   )r$   r(   rX   rY   rZ   r©   r[   r\   Úall_hidden_statesÚall_self_attnsÚcurrent_key_valuesÚiÚlayerÚlayer_outputsrs   r}   s                   r&   r9   zCpmAntEncoder.forwardZ  sÝ   € ñ8 #7™B¸DÐÙ0™°dˆÙ#,™R°$Ðä! $§+¡+Ó.ò 	O‰HˆAˆuÙ#Ø! mÐ%5Ñ5Ð!Ù!ØØØØ"3Ù6E °Ò 2È4Ø#ôˆMð >KÑ:ˆM˜<Ð):Ù Ø < /Ñ1�Ø Ñ,Ø%7Ð;LÐ:NÑ%NÑ"ð	Oð" ×-Ñ-¨mÓ<ˆáØ -Ð!1Ñ1ÐàÐ0Ð2CÀ^ÐSÐSr'   )NNNNr   r@   s   @r&   r    r    R  s�   ø„ ð8˜|õ 8ð -1Ø/3ØGKØ$(ñ6Tà—|‘|ð6Tð Ÿ™ð6Tð —|‘|ð	6Tð
 $ D™>ð6Tð ' t™nð6Tð " %¨¯©°e·l±lÐ(BÑ"CÑDð6Tð ˜D‘>÷6Tr'   r    c                   óV   ‡ — e Zd Zˆ fd„Zdej
                  dej
                  fd„Zˆ xZS )ÚCpmAntIntermediatec                 ó  •— t         ‰| �  «        t        j                  |j                  |j
                  «      | _        t        |j                  t        «      rt        |j                     | _        y |j                  | _        y r   )r   r   r   rL   r   Úintermediate_sizeÚdenseÚ
isinstanceÚ
hidden_actÚstrr
   Úintermediate_act_fnr#   s     €r&   r   zCpmAntIntermediate.__init__•  s]   ø€ Ü‰ÑÔÜ—Y‘Y˜v×1Ñ1°6×3KÑ3KÓLˆŒ
Ü�f×'Ñ'¬Ô-Ü'-¨f×.?Ñ.?Ñ'@ˆDÕ$à'-×'8Ñ'8ˆDÕ$r'   r(   Úreturnc                 óJ   — | j                  |«      }| j                  |«      }|S r   )r·   r»   r’   s     r&   r9   zCpmAntIntermediate.forward�  s&   € ØŸ
™
 =Ó1ˆØ×0Ñ0°Ó?ˆØÐr'   ©r:   r;   r<   r   r    r>   r9   r?   r@   s   @r&   r´   r´   ”  s#   ø„ ô9ð U§\¡\ð °e·l±l÷ r'   r´   c                   óš   ‡ — e Zd Zdefˆ fd„Zdej                  dej                  dej                  dej                  fd„Zd„ Zd
d	„Z	ˆ xZ
S )ÚCpmAntSegmentPositionEmbeddingr   c                 ób  •— t         ‰| �  «        |j                  | _        |j                  | _        |j                  | _        |j                  | _	        t        j                  t        j                  |j                  |j                  z  |j                  z   |j                  «      «      | _        y r   )r   r   rI   rJ   Úposition_bias_num_bucketsÚnum_bucketsÚposition_bias_max_distanceÚmax_distanceÚsegment_typesÚnum_segmentsr   r   r    r!   Úrelative_attention_biasr#   s     €r&   r   z'CpmAntSegmentPositionEmbedding.__init__¤  sŠ   ø€ Ü‰ÑÔà×3Ñ3ˆŒØ!×;Ñ;ˆÔØ"×=Ñ=ˆÔØ"×0Ñ0ˆÔä')§|¡|Ü�K‰KØ×$Ñ$ v×';Ñ';Ñ;¸f×>^Ñ>^Ñ^Ø×*Ñ*óó(
ˆÕ$r'   Úkey_posÚ	query_posÚkey_segmentÚquery_segmentc           	      ó0  — t        j                  «       5  |j                  d«      }|j                  d«      }|j                  d«      }|j                  d«      |j                  d«      k7  r0t        d|j                  d«      › d|j                  d«      › d�«      ‚||j                  d«      k7  s||j                  d«      k7  r!t        d|› d|j                  d«      › d�«      ‚||j                  d«      k7  r!t        d|› d|j	                  d«      › d�«      ‚|j                  |d|«      }|j                  ||d«      }|j                  |d|«      }|j                  ||d«      }| j                  ||«      }|| j                  z   }| j                  t        j                  |t         j                  |j                  ¬	«      d d d …f   t        j                  |t         j                  |j                  ¬	«      d d …d f   z
  | j                  | j                  ¬
«      }	t        j                  ||k(  |	d d d …d d …f   |«      }d d d «       t        j                  | j                   «      }
|
j#                  dddd«      j%                  «       }
|
S # 1 sw Y   ŒMxY w)Nr   r   z>key_pos.size(0) should be equal to query_pos.size(0), but got z and ú!z7keylen should be equal to key_segment.size(1), but got z;querylen should be equal to query_segment.size(1), but got r+   ©r1   r_   )rÃ   rÅ   r	   r,   )r    Úno_gradr/   r0   Úszier`   Ú!_segment_relative_position_bucketrÃ   Ú_position_bucketÚarangeÚint32r_   rÅ   ÚwhereÚFÚ	embeddingrÈ   ra   rk   )r$   rÉ   rÊ   rË   rÌ   ÚbatchÚkeylenÚquerylenÚrelative_position_bucketÚabsolute_position_bucketÚembedss              r&   r9   z&CpmAntSegmentPositionEmbedding.forward³  s•  € ô �]‰]‹_ñ %	Ø—L‘L “OˆEØ—\‘\ !“_ˆFØ —~‘~ aÓ(ˆHà�|‰|˜A‹ )§.¡.°Ó"3Ò3Ü$ØTÐU\×UaÑUaÐbcÓUdÐTeÐejÐkt×kyÑkyÐz{Ók|Ðj}Ð}~Ðóð ð ˜×)Ñ)¨!Ó,Ò,°¸M×<NÑ<NÈqÓ<QÒ0QÜ$ØMÈfÈXÐUZÐ[f×[kÑ[kÐlmÓ[nÐZoÐopÐqóð ð ˜=×-Ñ-¨aÓ0Ò0Ü$ØQÐRZÐQ[Ð[`Ðan×asÑasÐtuÓavÐ`wÐwxÐyóð ð —l‘l 5¨"¨fÓ5ˆGØ!Ÿ™ u¨h¸Ó;ˆIØ%×*Ñ*¨5°"°fÓ=ˆKØ)×.Ñ.¨u°hÀÓCˆMà'+×'MÑ'MÈmÐ]hÓ'iÐ$Ø'?À$×BRÑBRÑ'RÐ$ð (,×'<Ñ'<Ü—‘˜V¬5¯;©;Ð?W×?^Ñ?^Ô_Ð`dÒfgÐ`gÑhÜ—,‘,˜x¬u¯{©{ÐC[×CbÑCbÔcÒdeÐgkÐdkÑlñmà ×,Ñ,Ø!×.Ñ.ð	 (=ó (Ð$ô (-§{¡{Ø Ñ-Ø(¨ªq²!¨Ñ4Ø(ó(Ð$÷C%	ôP —‘Ð5°t×7SÑ7SÓTˆà—‘  1 a¨Ó+×6Ñ6Ó8ˆØˆ÷W%	ð %	ús   •H+JÊJc                 ó&   — || j                   z  |z   S r   )rÇ   )r$   rÌ   rË   s      r&   rÒ   z@CpmAntSegmentPositionEmbedding._segment_relative_position_bucketç  s   € Ø˜t×0Ñ0Ñ0°;Ñ>Ð>r'   c                 ó.  — d}|dz  }|dkD  j                  t        j                  «      |z  }t        j                  |«      }|dz  }||k  }|t        j                  |j                  «       |z  «      t        j                  ||z  «      z  ||z
  z  j                  t        j                  «      z   }t        j                  |t        j                  ||dz
  «      «      }|t        j                  ||j                  t        j                  «      |«      z  }|S )Nr   r,   r   )
r2   r    rÕ   ÚabsÚlogrj   re   ÚminÚ	full_likerÖ   )r$   Úrelative_positionrÃ   rÅ   Úrelative_bucketsÚ	max_exactÚis_smallÚrelative_postion_if_larges           r&   rÓ   z/CpmAntSegmentPositionEmbedding._position_bucketê  s  € ØÐà˜ÑˆØ-°Ñ1×5Ñ5´e·k±kÓBÀ[ÑPÐÜ!ŸI™IÐ&7Ó8ÐØ 1Ñ$ˆ	Ø$ yÑ0ˆØ$-Ü�I‰IÐ'×-Ñ-Ó/°)Ñ;Ó<Ü�h‰h�| iÑ/Ó0ñ1à˜YÑ&ñ(÷ ‰"ŒU�[‰[‹/ñ	%Ð!ô
 %*§I¡IØ%Ü�O‰OÐ5°{ÀQ±ÓGó%
Ð!ð 	œEŸK™K¨Ð2C×2FÑ2FÄuÇ{Á{Ó2SÐUnÓoÑoÐØÐr'   )é    é€   )r:   r;   r<   r   r   r    r>   r9   rÒ   rÓ   r?   r@   s   @r&   rÀ   rÀ   £  sU   ø„ ð
˜|õ 
ð2à—‘ð2ð —<‘<ð2ð —\‘\ð	2ð
 —|‘|ó2òh?÷ r'   rÀ   c                   ón   ‡ — e Zd Zˆ fd„Zdej
                  dej
                  dej
                  fd„Zˆ xZS )ÚCpmAntOutputc                 ó(  •— t         ‰| �  «        t        j                  |j                  |j
                  «      | _        t        j                  |j
                  |j                  ¬«      | _        t        j                  |j                  «      | _        y )N)r   )r   r   r   rL   r¶   r   r·   Ú	LayerNormÚlayer_norm_epsrT   Úhidden_dropout_probrU   r#   s     €r&   r   zCpmAntOutput.__init__  s`   ø€ Ü‰ÑÔÜ—Y‘Y˜v×7Ñ7¸×9KÑ9KÓLˆŒ
ÜŸ™ f×&8Ñ&8¸f×>SÑ>SÔTˆŒÜ—z‘z &×"<Ñ"<Ó=ˆ�r'   r(   Úinput_tensorr¼   c                 ór   — | j                  |«      }| j                  |«      }| j                  ||z   «      }|S r   )r·   rU   rï   )r$   r(   rò   s      r&   r9   zCpmAntOutput.forward  s7   € ØŸ
™
 =Ó1ˆØŸ™ ]Ó3ˆØŸ™ }°|Ñ'CÓDˆØÐr'   r¾   r@   s   @r&   rí   rí      s1   ø„ ô>ð U§\¡\ð ÀÇÁð ÐRW×R^ÑR^÷ r'   rí   c                   ó   — e Zd ZdZeZdZd„ Zy)ÚCpmAntPreTrainedModelz†
    An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained
    models.
    Úcpmantc                 ó  — t        |t        j                  «      rm|j                  j                  j                  d| j                  j                  ¬«       |j                  �%|j                  j                  j                  «        yyt        |t        j                  «      rz|j                  j                  j                  d| j                  j                  ¬«       |j                  �2|j                  j                  |j                     j                  «        yyt        |t        j                  «      rJ|j                  j                  j                  «        |j                  j                  j                  d«       yt        |t        «      r&|j                  j                  j                  d«       yt        |t        «      r<|j                   j                  j                  d| j                  j                  ¬«       yy)zInitialize the weightsg        )r5   ÚstdNg      ð?)r¸   r   rL   r"   ÚdataÚnormal_r   Úinit_stdrE   Úzero_Ú	EmbeddingÚpadding_idxrï   Úfill_r   rÀ   rÈ   )r$   Úmodules     r&   Ú_init_weightsz#CpmAntPreTrainedModel._init_weights  s[  € ä�fœbŸi™iÔ(Ø�M‰M×Ñ×&Ñ&¨C°T·[±[×5IÑ5IÐ&ÔJØ�{‰{Ð&Ø—‘× Ñ ×&Ñ&Õ(ð 'ä˜¤§¡Ô-Ø�M‰M×Ñ×&Ñ&¨C°T·[±[×5IÑ5IÐ&ÔJØ×!Ñ!Ð-Ø—‘×"Ñ" 6×#5Ñ#5Ñ6×<Ñ<Õ>ð .ä˜¤§¡Ô-Ø�K‰K×Ñ×"Ñ"Ô$Ø�M‰M×Ñ×$Ñ$ SÕ)Ü˜¤Ô0Ø�M‰M×Ñ×$Ñ$ SÕ)Ü˜Ô >Ô?Ø×*Ñ*×/Ñ/×7Ñ7¸SÀdÇkÁk×FZÑFZÐ7Õ[ð @r'   N)r:   r;   r<   r=   r   Úconfig_classÚbase_model_prefixr  r«   r'   r&   rõ   rõ     s   „ ñð
  €LØ Ðó\r'   rõ   aB  
    This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class. Use
    it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and
    behavior.

    Parameters
        config ([`~CpmAntConfig`]): Model configuration class with all the parameters of the
            Initializing with a config file does not load the weights associated with the model, only the
            configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
a  
    Args:
        input_ids (`torch.Tensor` of shape `(batch_size, seq_len)`):
            Indices of input sequence tokens in the vocabulary.

            Indices can be obtained using [`CPMAntTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are input IDs?](../glossary#input-ids)
        past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
            Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention
            blocks) that can be used (see `past_key_values` input) to speed up sequential decoding.
        use_cache (`bool`, *optional*):
            If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see
            `past_key_values`).
        output_attentions (`bool`, *optional*):
            Whether or not to return the attentions tensors of all attention layers.
        output_hidden_states (`bool`, *optional*):
            Whether or not to return the hidden states of all layers.
        return_dict (`bool`, *optional*):
            Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
zTThe bare CPMAnt Model outputting raw hidden-states without any specific head on top.c                   ó  ‡ — e Zd Zdefˆ fd„Zd„ Zd„ Zd„ Z ee	«       e
eee¬«      	 	 	 	 	 	 ddeej                      dee   d	ee   d
eeeej                            dee   dee   deeej                      ef   fd„«       «       Zˆ xZS )ÚCpmAntModelr   c                 ó¸  •— t         ‰| �  |«       t        |«      | _        t	        j
                  |j                  |j                  «      | _        t	        j
                  |j                  |j                  |j                  z  z   |j                  «      | _        t        |«      | _        |j                  | _        |j                  | _	        | j                  «        y r   )r   r   r    Úencoderr   rý   rÆ   r   Úsegment_embeddingÚ
vocab_sizeÚprompt_typesÚprompt_lengthÚinput_embeddingrÀ   rY   Ú	post_initr#   s     €r&   r   zCpmAntModel.__init__R  s§   ø€ Ü‰Ñ˜Ô Ü$ VÓ,ˆŒÜ!#§¡¨f×.BÑ.BÀF×DVÑDVÓ!WˆÔÜ!Ÿ|™|Ø×Ñ × 3Ñ 3°f×6JÑ6JÑ JÑJÈF×L^ÑL^ó 
ˆÔô <¸FÓCˆÔØ#×1Ñ1ˆÔØ ×+Ñ+ˆŒà�‰Õr'   c                 ó   — | j                   S r   ©r  ©r$   s    r&   Úget_input_embeddingsz CpmAntModel.get_input_embeddings_  s   € Ø×#Ñ#Ð#r'   c                 ó   — || _         y r   r  )r$   Ú
embeddingsÚkwargss      r&   Úset_input_embeddingsz CpmAntModel.set_input_embeddingsb  s
   € Ø)ˆÕr'   c                 ó*  — |j                  d«      }|j                  d«      }|j                  }t        j                  ||¬«      t        j                  ||¬«      j	                  dd«      k  }|d d …d d d …f   |d d …d d …d f   j                  «       |j	                  d||«      z  z  }	|	|d d …d d d …f   |d d …d d …d f   k(  z  }	t        j                  t        t        || j                  z
  «      «      d d d…   |¬«      d d d …f   j                  |d«      |d d …d f   k  }
t        j                  t        j                  || j                  |¬«      j                  «       |
fd¬«      }
|
j	                  ||d«      |
j	                  |d|«      z  |	z  }	|	S )Nr   r   )r_   r+   rF   )r/   r_   r    rÔ   r`   Úlogical_notrh   Úlistr¥   r  Úrepeatrb   Úonesru   )r$   Ú	input_idsÚspanÚcontextÚlengthrÙ   Úseqlenr_   Údirectional_mask_2drX   Úmask_1ds              r&   Ú_prepare_attention_maskz#CpmAntModel._prepare_attention_maske  s‰  € Ø—‘˜qÓ!ˆØ—‘ Ó"ˆØ×!Ñ!ˆÜ#Ÿl™l¨6¸&ÔAÄUÇ\Á\ÐRXÐagÔEh×EmÑEmÐnpÐrsÓEtÑtÐØ ¢ Dª! Ñ,Ø’A’q˜$�JÑ×+Ñ+Ó-Ð0C×0HÑ0HÈÈFÐTZÓ0[Ñ[ñ
ˆð (¨4²°4º°
Ñ+;¸tÂAÂqÈ$ÀJÑ?OÑ+OÑPˆô �L‰Lœœe F¨T×-?Ñ-?Ñ$?Ó@ÓAÁ$ÀBÀ$ÑGÐPVÔWÐX\Ò^_ÐX_Ñ`×gÑgÐhmÐopÓqØ’Q˜�W‰oñð 	ô —)‘)œUŸZ™Z¨¨t×/AÑ/AÈ&ÔQ×VÑVÓXÐZaÐbÐhiÔjˆØ Ÿ™ e¨V°QÓ7¸'¿,¹,ÀuÈaÐQWÓ:XÑXÐ[iÑiˆØÐr'   ©Ú
checkpointÚoutput_typer  r  rZ   r©   r[   r\   Úreturn_dictr¼   c           	      óì  — |�|n| j                   j                  }|�|n| j                   j                  }|�|n| j                   j                  }|�|n| j                   j                  }|j
                  t        j                  k7  r|j                  t        j                  «      }|j
                  |j                  }	}t        j                  |dk7  dd«      j                  ||	¬«      }
|
dk7  j                  d«      j                  ||	¬«      }t        j                  t        j                  | j                  dz  | j                  z   | j                  dz  | j                  z   ||	¬«      j!                  |j#                  d«      d«      |fd¬«      }|j#                  «       \  }}t        j                  t        j$                  || j                  ||	¬«      |
fd¬«      }
t        j&                  ||fd||	¬«      }t        j                  |||	¬«      j!                  |d«      }t        j&                  ||fd||	¬«      }|€]d}t)        d g| j*                  j,                  z  «      }|j/                  «       }| j1                  |«      }| j3                  |
«      }||z   }nH|d   d   j#                  d«      }| j3                  |
«      }| j1                  |«      |d d …dd …d d …f   z   }| j5                  ||||«      }| j7                  |||
|
«      }|d d …|d …d d …f   }|d d …d d …|d …d d …f   }|d d …|d …d d …f   }| j+                  |||||||«      \  }}}}|dk(  rw|d d …| j                  d …d d …f   }|�4d	}|D ]+  }||d d …d d …| j                  d …| j                  d …f   fz  }Œ- |}|�'d	}|D ]  }||d d …| j                  d …d d …f   fz  }Œ  |}|st)        d
„ ||||fD «       «      S t9        ||||¬«      S )Nr   r,   rÏ   r+   r	   r   rF   r^   r«   c              3   ó&   K  — | ]	  }|€Œ|–— Œ y ­wr   r«   )Ú.0Úvs     r&   ú	<genexpr>z&CpmAntModel.forward.<locals>.<genexpr>Ò  s   è ø€ ò ØÐefÑer”ñùs   ‚Š)Úlast_hidden_stater[   r(   Ú
attentions)r   rZ   r©   Úuse_return_dictr\   r1   r    rÕ   r2   r_   rÖ   Úsumrb   rÔ   r  r	  r  r/   ÚzerosÚfullÚtupler  r£   rk   r  r  r"  rY   r   )r$   r  rZ   r©   r[   r\   r&  r  r1   r_   Úsegmentr  rÙ   Ú
seq_lengthr  Úpositionr  Úpast_lengthr(   Úsegment_statesrX   rY   Úpresent_key_valuesr­   Úall_attentionsÚnew_attentionsÚ	attentionÚnew_hidden_statesÚhidden_states                                r&   r9   zCpmAntModel.forwardw  s.  € ð  2CÐ1NÑ-ÐTX×T_ÑT_×TqÑTqÐà$8Ð$DÑ È$Ï+É+×JjÑJjð 	ð &1Ð%<‘kÀ$Ç+Á+×B]ÑB]ˆØ!*Ð!6‘I¸D¿K¹K×<QÑ<Qˆ	ð �?‰?œeŸk™kÒ)Ø!Ÿ™¤U§[¡[Ó1ˆIØ!Ÿ™¨×)9Ñ)9ˆvˆÜ—+‘+˜i¨1™n¨a°Ó3×6Ñ6¸UÈ6Ð6ÓRˆØ˜Q‘,×#Ñ# BÓ'×*Ñ*°¸vÐ*ÓFˆÜ—I‘Iä—‘Ø×&Ñ&¨Ñ*¨T¯_©_Ñ<Ø×&Ñ&¨Ñ*¨T¯_©_Ñ<ØØ!ô	÷
 ‘&˜Ÿ™¨Ó*¨AÓ.Øðð ô
ˆ	ð &ŸN™NÓ,ÑˆˆzÜ—)‘)œUŸ[™[¨°×0BÑ0BÈ%ÐX^Ô_ÐahÐiÐopÔqˆÜ—*‘*˜e ZÐ0°!¸5ÈÔPˆÜ—<‘< 
°%ÀÔG×NÑNÈuÐVWÓXˆÜ�z‰z˜5 *Ð-¨q¸ÀfÔMˆàÐ"ØˆKÜ# T F¨T¯\©\×-DÑ-DÑ$DÓEˆOØ!×,Ñ,Ó.ˆIØ ×0Ñ0°Ó;ˆMØ!×3Ñ3°GÓ<ˆNØ)¨NÑ:‰Mà)¨!Ñ,¨QÑ/×4Ñ4°RÓ8ˆKØ!×3Ñ3°GÓ<ˆNØ ×0Ñ0°Ó;¸nÊQÐPRÑPSÒUVÈYÑ>WÑWˆMà×5Ñ5°iÀÀwÐPVÓWˆØ×*Ñ*¨8°X¸wÈÓPˆà'ª¨;©<ºÐ(:Ñ;ˆØ%¢aª¨K©Lº!Ð&;Ñ<ˆØ%¢a¨©²qÐ&8Ñ9ˆàOSÏ|É|ØØØØØ ØØóP
ÑLˆÐ)Ð+<¸nð ˜!ÒØ)ª!¨T×-?Ñ-?Ñ-AÂ1Ð*DÑEˆMàÐ)Ø!#�Ø!/ò e�IØ" y²²A°t×7IÑ7IÑ7KÈT×M_ÑM_ÑMaÐ1aÑ'bÐ&dÑd‘Nðeà!/�Ø Ð,Ø$&Ð!Ø$5ò U�LØ%¨,²q¸$×:LÑ:LÑ:NÒPQÐ7QÑ*RÐ)TÑTÑ%ðUà$5Ð!áÜñ Ø)Ð+=Ð?PÐR`Ðaôó ð ô 'Ø+Ø.Ø+Ø%ô	
ð 	
r'   )NNNNNN)r:   r;   r<   r   r   r  r  r"  r   ÚCPMANT_INPUTS_DOCSTRINGr   Ú_CHECKPOINT_FOR_DOCr   Ú_CONFIG_FOR_DOCr   r    r>   ru   r   r   r9   r?   r@   s   @r&   r  r  M  sð   ø„ ð
˜|õ ò$ò*òñ$ +Ð+BÓCÙØ&Ø+Ø$ôð -1Ø,0Ø/3Ø@DØ$(Ø&*ñ^
à˜EŸL™LÑ)ð^
ð $ D™>ð^
ð ' t™nð	^
ð
 " %¨¨e¯l©lÑ(;Ñ"<Ñ=ð^
ð ˜D‘>ð^
ð ˜d‘^ð^
ð 
ˆu�U—\‘\Ñ"Ð$;Ð;Ñ	<ò^
óó Dô^
r'   r  zy
    The CPMAnt Model with a language modeling head on top (linear layer with weights tied to the input embeddings).
    c                   óf  ‡ — e Zd ZdgZdefˆ fd„Z ee«       ee	e
e¬«      	 	 	 	 	 	 	 	 ddeej                     deeeej                  ej                  f         dee   dee   d	ee   d
eej                     dee   deej                     deee
f   fd„«       «       Zd„ Zd„ Zd„ Zd„ Zd„ Zˆ xZS )ÚCpmAntForCausalLMzlm_head.weightr   c                 óú   •— t         ‰| �  |«       t        |«      | _        t	        j
                  |j                  |j                  |j                  |j                  z  z   d¬«      | _
        | j                  «        y rƒ   )r   r   r  rö   r   rL   r   r	  r
  r  Úlm_headr  r#   s     €r&   r   zCpmAntForCausalLM.__init__ç  sd   ø€ Ü‰Ñ˜Ô Ü! &Ó)ˆŒô —y‘yØ×Ñ × 1Ñ 1°F×4GÑ4GÈ&×J^ÑJ^Ñ4^Ñ ^Ðejô
ˆŒð 	�‰Õr'   r#  r  r[   r\   rZ   r©   Úlabelsr&  rX   r¼   c	                 óº  — |�|n| j                   j                  }| j                  ||||||«      }
|r|
j                  n|
d   }| j	                  |«      }d}|�At        «       } ||j                  d|j                  d«      «      |j                  d«      «      }|s|f|
dd z   }|�|f|z   S |S t        |||
j                  |
j                  |
j                  ¬«      S )u;
  
        Args:
            input_ids (`torch.Tensor` of shape `(batch_size, seq_len)`):
                Indices of input sequence tokens in the vocabulary.

                Indices can be obtained using [`CPMAntTokenizer`]. See [`PreTrainedTokenizer.encode`] and
                [`PreTrainedTokenizer.__call__`] for details.

                [What are input IDs?](../glossary#input-ids)
            past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
                Contains pre-computed hidden-states (key and values in the self-attention blocks and in the
                cross-attention blocks) that can be used (see `past_key_values` input) to speed up sequential decoding.
            use_cache (`bool`, *optional*):
                If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding
                (see `past_key_values`).
            output_attentions (`bool`, *optional*):
                Whether or not to return the attentions tensors of all attention layers.
            output_hidden_states (`bool`, *optional*):
                Whether or not to return the hidden states of all layers.
            labels (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
                Labels for computing the masked language modeling loss.
            return_dict (`bool`, *optional*):
                Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
            attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
                CPMAnt will process attention mask automatically, this parameter is a dummy parameter for
                text-generation pipeline.

        Example:

        Text Generation with CpmAntForCausalLM.
        ```python
        >>> from transformers import CPMAntTokenizer, CpmAntForCausalLM

        >>> texts = "ä»Šå¤©å¤©æ°”ä¸�é”™ï¼Œ"
        >>> model = CpmAntForCausalLM.from_pretrained("openbmb/cpm-ant-10b")
        >>> tokenizer = CPMAntTokenizer.from_pretrained("openbmb/cpm-ant-10b")
        >>> input_ids = tokenizer(texts, return_tensors="pt")
        >>> outputs = model.generate(**input_ids)
        >>> output_texts = tokenizer.batch_decode(outputs)
        >>> print(output_texts)
        ['ä»Šå¤©å¤©æ°”ä¸�é”™ï¼Œé˜³å…‰æ˜Žåªšï¼Œæˆ‘å’Œå¦ˆå¦ˆä¸€èµ·åŽ»è¶…å¸‚ä¹°ä¸œè¥¿ã€‚\nåœ¨è¶…å¸‚é‡Œï¼Œæˆ‘çœ‹åˆ°äº†ä¸€ä¸ªå¾ˆå¥½çŽ©çš„çŽ©å…·ï¼Œå®ƒçš„å��å­—å�«â€œæœºå™¨äººâ€�ã€‚å®ƒæœ‰ä¸€ä¸ªåœ†åœ†çš„è„‘è¢‹ï¼Œä¸¤å�ªåœ†åœ†çš„çœ¼ç�›ï¼Œè¿˜æœ‰ä¸€ä¸ªåœ†åœ†çš„']
        ```
        Nr   r+   r   )ÚlossÚlogitsr[   r(   r-  )r   r.  rö   r,  rD  r   r`   r/   r   r[   r(   r-  )r$   r  r[   r\   rZ   r©   rE  r&  rX   r  Úmodel_outputr(   rH  rG  Ú	loss_funcÚoutputs                   r&   r9   zCpmAntForCausalLM.forwardñ  sô   € ðz &1Ð%<‘kÀ$Ç+Á+×B]ÑB]ˆà—{‘{ØÐ(Ð*>ÀÐQZÐ\gó
ˆñ ;F˜×6Ò6È<ÐXYÉ?ˆà—‘˜mÓ,ˆàˆØÐÜ(Ó*ˆIÙ˜VŸ[™[¨¨V¯[©[¸«_Ó=¸v¿{¹{È2»ÓOˆDáØ�Y ¨a¨bÐ!1Ñ1ˆFØ)-Ð)9�T�G˜fÑ$ÐE¸vÐEä%ØØØ(×8Ñ8Ø&×4Ñ4Ø#×.Ñ.ô
ð 	
r'   c                 ó.   — | j                   j                  S r   ©rö   r  r  s    r&   r  z&CpmAntForCausalLM.get_input_embeddingsH  s   € Ø�{‰{×*Ñ*Ð*r'   c                 ó&   — || j                   _        y r   rM  )r$   r  s     r&   r  z&CpmAntForCausalLM.set_input_embeddingsK  s   € Ø&0ˆ�‰Õ#r'   c                 ó   — | j                   S r   ©rD  r  s    r&   Úget_output_embeddingsz'CpmAntForCausalLM.get_output_embeddingsN  s   € Ø�|‰|Ðr'   c                 ó   — || _         y r   rP  )r$   Únew_embeddingss     r&   Úset_output_embeddingsz'CpmAntForCausalLM.set_output_embeddingsQ  s	   € Ø%ˆ�r'   c                 ó‚   — |D �cg c]  }|�t        |«      n|‘Œ }}|D ]  }|d   |   |d<   |d   |   |d<   Œ |S c c}w )Nr   r   )r  )r$   r[   Úbeam_idxÚeachÚkey_value_layers        r&   Ú_reorder_cachez CpmAntForCausalLM._reorder_cacheT  sh   € ØP_Ö`È¨Ð)9œ4 œ:¸tÑCÐ`ˆÐ`Ø.ò 	>ˆOØ!0°Ñ!3°HÑ!=ˆO˜AÑØ!0°Ñ!3°HÑ!=ˆO˜AÒð	>ð Ðùò	 as   …<)NNNNNNNN)r:   r;   r<   Ú_tied_weights_keysr   r   r   r>  r   r?  r   r@  r   r    r>   r   r   ru   r   r9   r  r  rQ  rT  rY  r?   r@   s   @r&   rB  rB  Þ  s4  ø„ ð +Ð+Ðð˜|õ ñ +Ð+BÓCÙØ&Ø*Ø$ôð -1ØMQØ$(Ø,0Ø/3Ø)-Ø&*Ø15ñO
à˜EŸL™LÑ)ðO
ð " $ u¨U¯\©\¸5¿<¹<Ð-GÑ'HÑ"IÑJðO
ð ˜D‘>ð	O
ð
 $ D™>ðO
ð ' t™nðO
ð ˜Ÿ™Ñ&ðO
ð ˜d‘^ðO
ð ! §¡Ñ.ðO
ð 
ˆuÐ,Ð,Ñ	-òO
óó DðO
òb+ò1òò&ör'   rB  )rB  r  rõ   )6r=   re   Útypingr   r   r   r   r    Útorch.nn.functionalr   Ú
functionalr×   Útorch.utils.checkpointÚtorch.nnr   Úactivationsr
   Ú
generationr   Úmodeling_outputsr   r   Úmodeling_utilsr   Úutilsr   r   r   r   Úconfiguration_cpmantr   Ú
get_loggerr:   Úloggerr?  r@  ÚModuler   rB   rw   r�   r�   r”   r›   r    r´   rÀ   rí   rõ   ÚCPMANT_START_DOCSTRINGr>  r  rB  Ú__all__r«   r'   r&   ú<module>rk     s–  ðñ ã ß /Ó /ã ß Ð Û Ý Ý %å !Ý )ß OÝ -ß uÓ uÝ .ð 
ˆ×	Ñ	˜HÓ	%€à+Ð Ø €ô�b—i‘iô ô2e4�b—i‘iô e4ôP.>˜rŸy™yô .>ôb˜"Ÿ)™)ô ô(˜Ÿ	™	ô ô4�R—Y‘Yô ô6,>˜RŸY™Yô ,>ô^>T�B—I‘Iô >TôD˜Ÿ™ô ôY  R§Y¡Yô Y ôz�2—9‘9ô ô\˜Oô \ð8	Ð ðÐ ñ0 ØZØóôJ
Ð'ó J
ó	ðJ
ñZ ðð ó	ôuÐ-¨ó uóðuòp H�r'   