Ë
    T^(hD,  ã                   óº  — d Z ddlmZmZmZmZmZ ddlZddlZddlm	Z	 ddl
mZ ddlmZmZ ddlmZ dd	lmZ dd
lmZ ddlmZ ddlmZmZmZ ddlmZmZmZmZm Z m!Z!m"Z"m#Z# ddl$m%Z%  ejL                  e'«      Z(dZ)dZ* G d„ de	jV                  «      Z, G d„ de«      Z- G d„ de«      Z.dZ/ G d„ de!«      Z0 G d„ de«      Z1 G d„ de«      Z2 G d„ d e «      Z3g d!¢Z4y)"zPyTorch Starcoder2 model.é    )ÚCallableÚListÚOptionalÚTupleÚUnionN)Únné   )ÚACT2FN)ÚCacheÚDynamicCache)ÚFlashAttentionKwargs)ÚBaseModelOutputWithPast)ÚALL_ATTENTION_FUNCTIONS)ÚUnpack)Ú%add_start_docstrings_to_model_forwardÚcan_return_tupleÚloggingé   )ÚMistralAttentionÚMistralDecoderLayerÚMistralForCausalLMÚ MistralForSequenceClassificationÚMistralForTokenClassificationÚMistralModelÚapply_rotary_pos_embÚeager_attention_forwardé   )ÚStarcoder2Configr   zbigcode/starcoder2-7bc                   óh   ‡ — e Zd Zdefˆ fd„Zdeeej                        dej                  fd„Z	ˆ xZ
S )ÚStarcoder2MLPÚconfigc                 óP  •— t         ‰| �  «        |j                  }t        j                  ||j
                  |j                  ¬«      | _        t        j                  |j
                  ||j                  ¬«      | _        t        |j                     | _        |j                  | _        y ©N)Úbias)ÚsuperÚ__init__Úhidden_sizer   ÚLinearÚintermediate_sizeÚuse_biasÚc_fcÚc_projr
   Ú
hidden_actÚactÚresidual_dropout)Úselfr!   Ú	embed_dimÚ	__class__s      €úo/var/www/skyplay_api_hub/venv/lib/python3.12/site-packages/transformers/models/starcoder2/modular_starcoder2.pyr&   zStarcoder2MLP.__init__9   su   ø€ Ü‰ÑÔØ×&Ñ&ˆ	Ü—I‘I˜i¨×)AÑ)AÈÏÉÔXˆŒ	Ü—i‘i × 8Ñ 8¸)È&Ï/É/ÔZˆŒÜ˜&×+Ñ+Ñ,ˆŒØ &× 7Ñ 7ˆÕó    Úhidden_statesÚreturnc                 óØ   — | j                  |«      }| j                  |«      }| j                  |«      }t        j                  j                  || j                  | j                  ¬«      }|S )N©ÚpÚtraining)r+   r.   r,   r   Ú
functionalÚdropoutr/   r:   )r0   r5   s     r3   ÚforwardzStarcoder2MLP.forwardA   sZ   € ØŸ	™	 -Ó0ˆØŸ™ Ó/ˆØŸ™ MÓ2ˆÜŸ™×-Ñ-¨m¸t×?TÑ?TÐ_c×_lÑ_lÐ-ÓmˆØÐr4   )Ú__name__Ú
__module__Ú__qualname__r   r&   r   r   ÚtorchÚFloatTensorr=   Ú__classcell__©r2   s   @r3   r    r    8   s9   ø„ ð8Ð/õ 8ð X¨e°E×4EÑ4EÑ.FÑ%Gð ÈE×L]ÑL]÷ r4   r    c                   ó:  ‡ — e Zd Zddedee   fˆ fd„Z	 	 ddej                  de	ej                  ej                  f   deej                     dee
   deej                     d	ee   d
e	ej                  eej                     ee	ej                        f   fd„Zˆ xZS )ÚStarcoder2Attentionr!   Ú	layer_idxc                 ó„  •— t         ‰| �  «        |j                  | _        t        j                  |j
                  |j                  | j                  z  |j                  ¬«      | _	        t        j                  |j
                  |j                  | j                  z  |j                  ¬«      | _        t        j                  |j
                  |j                  | j                  z  |j                  ¬«      | _        t        j                  |j                  | j                  z  |j
                  |j                  ¬«      | _        y r#   )r%   r&   r/   r   r(   r'   Únum_attention_headsÚhead_dimr*   Úq_projÚnum_key_value_headsÚk_projÚv_projÚo_proj©r0   r!   rG   r2   s      €r3   r&   zStarcoder2Attention.__init__J   så   ø€ Ü‰ÑÔØ &× 7Ñ 7ˆÔÜ—i‘i × 2Ñ 2°F×4NÑ4NÐQU×Q^ÑQ^Ñ4^Ðek×etÑetÔuˆŒÜ—i‘i × 2Ñ 2°F×4NÑ4NÐQU×Q^ÑQ^Ñ4^Ðek×etÑetÔuˆŒÜ—i‘i × 2Ñ 2°F×4NÑ4NÐQU×Q^ÑQ^Ñ4^Ðek×etÑetÔuˆŒÜ—i‘i × :Ñ :¸T¿]¹]Ñ JÈF×L^ÑL^Ðek×etÑetÔuˆ�r4   r5   Úposition_embeddingsÚattention_maskÚpast_key_valueÚcache_positionÚkwargsr6   c           
      óN  — |j                   d d }g |¢d‘| j                  ‘­}| j                  |«      j                  |«      j	                  dd«      }	| j                  |«      j                  |«      j	                  dd«      }
| j                  |«      j                  |«      j	                  dd«      }|\  }}t        |	|
||«      \  }	}
|�'|||dœ}|j                  |
|| j                  |«      \  }
}t        }| j                  j                  dk7  r^| j                  j                  dk(  r(|j                  dd«      rt        j                  d	«       nt         | j                  j                     } || |	|
||f| j"                  sd
n| j$                  | j&                  t)        | j                  dd «      dœ|¤Ž\  }} |j*                  g |¢d‘­Ž j-                  «       }| j/                  |«      }t0        j2                  j5                  || j6                  | j"                  ¬«      }||fS )Néÿÿÿÿr   r   )ÚsinÚcosrT   ÚeagerÚsdpaÚoutput_attentionsFzã`torch.nn.functional.scaled_dot_product_attention` does not support `output_attentions=True`. Falling back to eager attention. This warning can be removed using the argument `attn_implementation="eager"` when loading the model.g        Úsliding_window)r<   Úscalingr]   r8   )ÚshaperJ   rK   ÚviewÚ	transposerM   rN   r   ÚupdaterG   r   r!   Ú_attn_implementationÚgetÚloggerÚwarning_oncer   r:   Úattention_dropoutr^   ÚgetattrÚreshapeÚ
contiguousrO   r   r;   r<   r/   )r0   r5   rQ   rR   rS   rT   rU   Úinput_shapeÚhidden_shapeÚquery_statesÚ
key_statesÚvalue_statesrY   rX   Úcache_kwargsÚattention_interfaceÚattn_outputÚattn_weightss                     r3   r=   zStarcoder2Attention.forwardR   s  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆØ—[‘[ Ó/×4Ñ4°\ÓB×LÑLÈQÐPQÓRˆ
Ø—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆà&‰ˆˆSÜ#7¸ÀjÐRUÐWZÓ#[Ñ ˆ�jàÐ%à#&¨sÀnÑUˆLØ'5×'<Ñ'<¸ZÈÐW[×WeÑWeÐgsÓ'tÑ$ˆJ˜ä(?ÐØ�;‰;×+Ñ+¨wÒ6Ø�{‰{×/Ñ/°6Ò9¸f¿j¹jÐI\Ð^cÔ>dÜ×#Ñ#ðLõô
 '>¸d¿k¹k×>^Ñ>^Ñ&_Ð#á$7ØØØØØð
%
ð  $Ÿ}š}‘C°$×2HÑ2HØ—L‘LÜ" 4§;¡;Ð0@À$ÓGñ
%
ð ñ
%
Ñ!ˆ�\ð *�k×)Ñ)Ð;¨;Ð;¸Ò;×FÑFÓHˆØ—k‘k +Ó.ˆÜ—m‘m×+Ñ+Ø˜4×0Ñ0¸4¿=¹=ð ,ó 
ˆð ˜LÐ(Ð(r4   )N)NN)r>   r?   r@   r   r   Úintr&   rA   ÚTensorr   r   Ú
LongTensorr   r   r=   rC   rD   s   @r3   rF   rF   I   sÊ   ø„ ñvÐ/ð v¸HÀS¹Mõ vð +/Ø59ñ4)à—|‘|ð4)ð # 5§<¡<°·±Ð#=Ñ>ð4)ð ! §¡Ñ.ð	4)ð
 ! ™ð4)ð ! ×!1Ñ!1Ñ2ð4)ð Ð-Ñ.ð4)ð 
ˆu�|‰|˜X e§l¡lÑ3°X¸eÀEÇLÁLÑ>QÑ5RÐRÑ	S÷4)r4   rF   c                   ó(   ‡ — e Zd Zdedefˆ fd„Zˆ xZS )ÚStarcoder2DecoderLayerr!   rG   c                 ó(  •— t         ‰| �  | «       t        ||¬«      | _        t	        |«      | _        t        j                  |j                  |j                  ¬«      | _
        t        j                  |j                  |j                  ¬«      | _        y )N)r!   rG   ©Úeps)r%   r&   rF   Ú	self_attnr    Úmlpr   Ú	LayerNormr'   Únorm_epsilonÚinput_layernormÚpost_attention_layernormrP   s      €r3   r&   zStarcoder2DecoderLayer.__init__Š   sh   ø€ Ü‰Ñ˜ÔÜ,°FÀiÔPˆŒÜ  Ó(ˆŒÜ!Ÿ|™|¨F×,>Ñ,>ÀF×DWÑDWÔXˆÔÜ(*¯©°V×5GÑ5GÈV×M`ÑM`Ô(aˆÕ%r4   )r>   r?   r@   r   rt   r&   rC   rD   s   @r3   rx   rx   ‰   s    ø„ ðbÐ/ð b¸C÷ bñ br4   rx   c                   óP  ‡ — e Zd Zdefˆ fd„Ze ee«      	 	 	 	 	 	 	 	 	 ddee	j                     dee	j                     dee	j                     deeeee	j                     f      dee	j                     dee   d	ee   d
ee   dee	j                     dee   defd„«       «       Zˆ xZS )ÚStarcoder2Modelr!   c           	      ó:  •— t         ‰| �  |«       t        j                  t	        |j
                  «      D �cg c]  }t        ||«      ‘Œ c}«      | _        t        j                  |j                  |j                  ¬«      | _        |j                  | _        y c c}w )Nrz   )r%   r&   r   Ú
ModuleListÚrangeÚnum_hidden_layersrx   Úlayersr~   r'   r   ÚnormÚembedding_dropoutrP   s      €r3   r&   zStarcoder2Model.__init__–   su   ø€ Ü‰Ñ˜Ô Ü—m‘mÜHMÈf×NfÑNfÓHgÖh¸9Ô# F¨IÕ6Òhó
ˆŒô —L‘L ×!3Ñ!3¸×9LÑ9LÔMˆŒ	Ø!'×!9Ñ!9ˆÕùò is   ·BÚ	input_idsrR   Úposition_idsÚpast_key_valuesÚinputs_embedsÚ	use_cacher\   Úoutput_hidden_statesrT   Úflash_attn_kwargsr6   c
                 óò  — |�|n| j                   j                  }|�|n| j                   j                  }|�|n| j                   j                  }|d u |d uz  rt	        d«      ‚| j
                  r%| j                  r|rt        j                  d«       d}|€| j                  |«      }|r|€
t        «       }|	€F|�|j                  «       nd}t        j                  |||j                  d   z   |j                  ¬«      }	|€|	j!                  d«      }| j#                  |||	||«      }|}t$        j&                  j)                  || j*                  | j                  ¬«      }| j-                  ||«      }|rdnd }|rdnd }| j.                  d | j                   j0                   D ],  }|r||fz  } ||f||||||	|d	œ|
¤Ž}|d   }|sŒ$||d   fz  }Œ. | j3                  |«      }|r||fz  }t5        ||r|nd ||¬
«      S )Nz:You must specify exactly one of input_ids or inputs_embedszX`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`.Fr   r   )Údevicer8   © )rR   rŒ   rS   r\   r�   rT   rQ   )Úlast_hidden_stater�   r5   Ú
attentions)r!   r\   r�   r�   Ú
ValueErrorÚgradient_checkpointingr:   re   rf   Úembed_tokensr   Úget_seq_lengthrA   Úaranger_   r“   Ú	unsqueezeÚ_update_causal_maskr   r;   r<   rŠ   Ú
rotary_embrˆ   r‡   r‰   r   )r0   r‹   rR   rŒ   r�   rŽ   r�   r\   r�   rT   r‘   Úpast_seen_tokensÚcausal_maskr5   rQ   Úall_hidden_statesÚall_self_attnsÚdecoder_layerÚlayer_outputss                      r3   r=   zStarcoder2Model.forwardž   s]  € ð 2CÐ1NÑ-ÐTX×T_ÑT_×TqÑTqÐà$8Ð$DÑ È$Ï+É+×JjÑJjð 	ð "+Ð!6‘I¸D¿K¹K×<QÑ<Qˆ	à˜Ð -°tÐ";Ò<ÜÐYÓZÐZà×&Ò&¨4¯=ª=¹YÜ×ÑØjôð ˆIàÐ Ø ×-Ñ-¨iÓ8ˆMá˜Ð0Ü*›nˆOàÐ!ØCRÐC^˜×=Ñ=Ô?ÐdeÐÜ"Ÿ\™\Ø Ð"2°]×5HÑ5HÈÑ5KÑ"KÐTa×ThÑThôˆNð ÐØ)×3Ñ3°AÓ6ˆLà×.Ñ.Ø˜M¨>¸?ÐL]ó
ˆð &ˆÜŸ™×-Ñ-Ø˜T×3Ñ3¸d¿m¹mð .ó 
ˆð
 #Ÿo™o¨m¸\ÓJÐñ #7™B¸DÐÙ0™°dˆà!Ÿ[™[Ð)H¨4¯;©;×+HÑ+HÐIò 	6ˆMÙ#Ø! mÐ%5Ñ5Ð!á)Øð
à*Ø)Ø.Ø"3Ø#Ø-Ø$7ñ
ð $ñ
ˆMð *¨!Ñ,ˆMâ Ø =°Ñ#3Ð"5Ñ5‘ð'	6ð* Ÿ	™	 -Ó0ˆñ  Ø -Ð!1Ñ1Ðä&Ø+Ù/8™O¸dØ+Ø%ô	
ð 	
r4   )	NNNNNNNNN)r>   r?   r@   r   r&   r   r   ÚSTARCODER2_INPUTS_DOCSTRINGr   rA   rv   ru   r   r   r   rB   Úboolr   r   r   r=   rC   rD   s   @r3   rƒ   rƒ   •   s*  ø„ ð:Ð/õ :ð Ù*Ð+FÓGð 15Ø15Ø37ØKOØ59Ø$(Ø,0Ø/3Ø59ñ[
à˜E×,Ñ,Ñ-ð[
ð ! §¡Ñ.ð[
ð ˜u×/Ñ/Ñ0ð	[
ð
 " %¨¨t°E×4EÑ4EÑ/FÐ(FÑ"GÑHð[
ð   × 1Ñ 1Ñ2ð[
ð ˜D‘>ð[
ð $ D™>ð[
ð ' t™nð[
ð ! ×!1Ñ!1Ñ2ð[
ð $Ð$8Ñ9ð[
ð 
!ò[
ó Hó ô[
r4   rƒ   c                   ó   — e Zd Zy)ÚStarcoder2ForCausalLMN©r>   r?   r@   r”   r4   r3   r¨   r¨   þ   ó   „ Ør4   r¨   c                   ó   — e Zd Zy)Ú#Starcoder2ForSequenceClassificationNr©   r”   r4   r3   r¬   r¬     rª   r4   r¬   c                   ó   — e Zd Zy)Ú Starcoder2ForTokenClassificationNr©   r”   r4   r3   r®   r®     rª   r4   r®   )r¨   rƒ   ÚStarcoder2PreTrainedModelr¬   r®   )5Ú__doc__Útypingr   r   r   r   r   rA   Útorch.utils.checkpointr   Úactivationsr
   Úcache_utilsr   r   Úmodeling_flash_attention_utilsr   Úmodeling_outputsr   Úmodeling_utilsr   Úprocessing_utilsr   Úutilsr   r   r   Úmistral.modeling_mistralr   r   r   r   r   r   r   r   Úconfiguration_starcoder2r   Ú
get_loggerr>   re   Ú_CONFIG_FOR_DOCÚ_CHECKPOINT_FOR_DOCÚModuler    rF   rx   r¥   rƒ   r¨   r¬   r®   Ú__all__r”   r4   r3   ú<module>rÁ      sÚ   ðñ(  ç 9Õ 9ã Û Ý å !ß .Ý Bõõ 6Ý &ß UÑ U÷	÷ 	ó 	õ 7ð 
ˆ×	Ñ	˜HÓ	%€à$€Ø-Ð ô�B—I‘Iô ô"=)Ð*ô =)ô@bÐ0ô bð #Ð ôf
�lô f
ôR	Ð.ô 	ô	Ð*Jô 	ô	Ð'Dô 	ò�r4   