Ë
    S^(h   ã                   ó  — d dl Z d dlmZ d dlZd dlmZ d dlmZmZ ddlm	Z	  e	j                  e«      Z G d„ dej                  «      Z G d	„ d
ej                  «      Z G d„ dej                  «      Z G d„ dej                  «      Z G d„ dej                  «      Z G d„ dej                  «      Z G d„ dej                  «      Z G d„ dej                  «      Z G d„ dej                  «      Z G d„ dej                  «      Z G d„ dej                  «      Z G d„ de«      Zi de“d ed!d"d#œf“d$e“d%e“d&ed'd(if“d)e“d*e“d+e“d,ej4                  “d-e“d.e“d/e“d0ej6                  “d1e“d2ej8                  “d3ej:                  “d4ej<                  “ej<                  ej>                  ej@                  d5œ¥Z! ee!«      Z"d6„ Z# e#d&«      Z$ e#d%«      Z% e#d«      Z& e#d$«      Z' e#d/«      Z( e#d4«      Z) e#d.«      Z* e#d-«      Z+y)7é    N)ÚOrderedDict)Úversion)ÚTensorÚnné   )Úloggingc                   ó2   ‡ — e Zd ZdZˆ fd„Zdedefd„Zˆ xZS )ÚPytorchGELUTanha  
    A fast C implementation of the tanh approximation of the GeLU activation function. See
    https://arxiv.org/abs/1606.08415.

    This implementation is equivalent to NewGELU and FastGELU but much faster. However, it is not an exact numerical
    match due to rounding errors.
    c                 óÎ   •— t         ‰| �  «        t        j                  t        j
                  «      t        j                  d«      k  rt        dt        j
                  › d�«      ‚y )Nz1.12.0zYou are using torch==zM, but torch>=1.12.0 is required to use PytorchGELUTanh. Please upgrade torch.)ÚsuperÚ__init__r   ÚparseÚtorchÚ__version__ÚImportError©ÚselfÚ	__class__s    €úV/var/www/skyplay_api_hub/venv/lib/python3.12/site-packages/transformers/activations.pyr   zPytorchGELUTanh.__init__%   sY   ø€ Ü‰ÑÔÜ�=‰=œ×*Ñ*Ó+¬g¯m©m¸HÓ.EÒEÜØ'¬×(9Ñ(9Ð':ð ;9ð 9óð ð Fó    ÚinputÚreturnc                 óD   — t         j                  j                  |d¬«      S )NÚtanh)Úapproximate)r   Ú
functionalÚgelu©r   r   s     r   ÚforwardzPytorchGELUTanh.forward-   s   € Ü�}‰}×!Ñ! %°VÐ!Ó<Ð<r   ©Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r   r   Ú__classcell__©r   s   @r   r
   r
      s    ø„ ñôð=˜Vð =¨÷ =r   r
   c                   ó    — e Zd ZdZdedefd„Zy)ÚNewGELUActivationzÆ
    Implementation of the GELU activation function currently in Google BERT repo (identical to OpenAI GPT). Also see
    the Gaussian Error Linear Units paper: https://arxiv.org/abs/1606.08415
    r   r   c                 óÀ   — d|z  dt        j                  t        j                  dt        j                  z  «      |dt        j
                  |d«      z  z   z  «      z   z  S )Nç      à?ç      ð?ç       @ç÷Hmâä¦?g      @)r   r   ÚmathÚsqrtÚpiÚpowr   s     r   r   zNewGELUActivation.forward7   sP   € Ø�U‰{˜c¤E§J¡J¬t¯y©y¸¼t¿w¹w¹Ó/GÈ5ÐS[Ô^c×^gÑ^gÐhmÐorÓ^sÑSsÑKsÑ/tÓ$uÑuÑvÐvr   N©r!   r"   r#   r$   r   r   © r   r   r(   r(   1   s   „ ñð
w˜Vð w¨ô wr   r(   c                   óJ   ‡ — e Zd ZdZddefˆ fd„Zdedefd„Zdedefd„Zˆ xZ	S )	ÚGELUActivationa«  
    Original Implementation of the GELU activation function in Google BERT repo when initially created. For
    information: OpenAI GPT's GELU is slightly different (and gives slightly different results): 0.5 * x * (1 +
    torch.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 * torch.pow(x, 3)))) This is now written in C in nn.functional
    Also see the Gaussian Error Linear Units paper: https://arxiv.org/abs/1606.08415
    Úuse_gelu_pythonc                 óˆ   •— t         ‰| �  «        |r| j                  | _        y t        j
                  j                  | _        y ©N)r   r   Ú_gelu_pythonÚactr   r   r   )r   r6   r   s     €r   r   zGELUActivation.__init__C   s/   ø€ Ü‰ÑÔÙØ×(Ñ(ˆD�Hä—}‘}×)Ñ)ˆD�Hr   r   r   c                 ój   — |dz  dt        j                  |t        j                  d«      z  «      z   z  S )Nr*   r+   r,   )r   Úerfr.   r/   r   s     r   r9   zGELUActivation._gelu_pythonJ   s,   € Ø�s‰{˜c¤E§I¡I¨e´d·i±iÀ³nÑ.DÓ$EÑEÑFÐFr   c                 ó$   — | j                  |«      S r8   ©r:   r   s     r   r   zGELUActivation.forwardM   ó   € Ø�x‰x˜‹Ðr   )F)
r!   r"   r#   r$   Úboolr   r   r9   r   r%   r&   s   @r   r5   r5   ;   s=   ø„ ññ*¨õ *ðG &ð G¨Vó Gð˜Vð ¨÷ r   r5   c                   ó    — e Zd ZdZdedefd„Zy)ÚFastGELUActivationz}
    Applies GELU approximation that is slower than QuickGELU but more accurate. See: https://github.com/hendrycks/GELUs
    r   r   c                 ó\   — d|z  dt        j                  |dz  dd|z  |z  z   z  «      z   z  S )Nr*   r+   g€ÑÓ3Eˆé?r-   )r   r   r   s     r   r   zFastGELUActivation.forwardV   s:   € Ø�U‰{˜c¤E§J¡J¨u°|Ñ/CÀsÈXÐX]ÑM]Ð`eÑMeÑGeÑ/fÓ$gÑgÑhÐhr   Nr2   r3   r   r   rB   rB   Q   s   „ ñði˜Vð i¨ô ir   rB   c                   ó    — e Zd ZdZdedefd„Zy)ÚQuickGELUActivationzr
    Applies GELU approximation that is fast but somewhat inaccurate. See: https://github.com/hendrycks/GELUs
    r   r   c                 ó8   — |t        j                  d|z  «      z  S )Ng¬Zd;û?)r   Úsigmoidr   s     r   r   zQuickGELUActivation.forward_   s   € Ø”u—}‘} U¨U¡]Ó3Ñ3Ð3r   Nr2   r3   r   r   rE   rE   Z   s   „ ñð4˜Vð 4¨ô 4r   rE   c                   ó<   ‡ — e Zd ZdZdedefˆ fd„Zdedefd„Zˆ xZS )ÚClippedGELUActivationa‚  
    Clip the range of possible GeLU outputs between [min, max]. This is especially useful for quantization purpose, as
    it allows mapping negatives values in the GeLU spectrum. For more information on this trick, please refer to
    https://arxiv.org/abs/2004.09602.

    Gaussian Error Linear Unit. Original Implementation of the gelu activation function in Google Bert repo when
    initially created.

    For information: OpenAI GPT's gelu is slightly different (and gives slightly different results): 0.5 * x * (1 +
    torch.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 * torch.pow(x, 3)))). See https://arxiv.org/abs/1606.08415
    ÚminÚmaxc                 ól   •— ||kD  rt        d|› d|› d�«      ‚t        ‰| �	  «        || _        || _        y )Nzmin should be < max (got min: z, max: ú))Ú
ValueErrorr   r   rJ   rK   )r   rJ   rK   r   s      €r   r   zClippedGELUActivation.__init__p   s>   ø€ Ø�Š9ÜÐ=¸c¸UÀ'È#ÈÈaÐPÓQÐQä‰ÑÔØˆŒØˆ�r   Úxr   c                 ój   — t        j                  t        |«      | j                  | j                  «      S r8   )r   Úclipr   rJ   rK   )r   rO   s     r   r   zClippedGELUActivation.forwardx   s!   € Ü�z‰zœ$˜q›' 4§8¡8¨T¯X©XÓ6Ð6r   )	r!   r"   r#   r$   Úfloatr   r   r   r%   r&   s   @r   rI   rI   c   s.   ø„ ñ
ð˜Eð ¨õ ð7˜ð 7 F÷ 7r   rI   c                   ó2   ‡ — e Zd ZdZˆ fd„Zdedefd„Zˆ xZS )ÚAccurateGELUActivationzÙ
    Applies GELU approximation that is faster than default and more accurate than QuickGELU. See:
    https://github.com/hendrycks/GELUs

    Implemented along with MEGA (Moving Average Equipped Gated Attention)
    c                 óx   •— t         ‰| �  «        t        j                  dt        j                  z  «      | _        y )Né   )r   r   r.   r/   r0   Úprecomputed_constantr   s    €r   r   zAccurateGELUActivation.__init__„   s'   ø€ Ü‰ÑÔÜ$(§I¡I¨a´$·'±'©kÓ$:ˆÕ!r   r   r   c                 óŒ   — d|z  dt        j                  | j                  |dt        j                  |d«      z  z   z  «      z   z  S )Nr*   r   r-   é   )r   r   rW   r1   r   s     r   r   zAccurateGELUActivation.forwardˆ   sE   € Ø�U‰{˜a¤%§*¡*¨T×-FÑ-FÈ%ÐRZÔ]b×]fÑ]fÐglÐnoÓ]pÑRpÑJpÑ-qÓ"rÑrÑsÐsr   r    r&   s   @r   rT   rT   |   s#   ø„ ñô;ðt˜Vð t¨÷ tr   rT   c                   óB   ‡ — e Zd ZdZˆ fd„Zdedefd„Zdedefd„Zˆ xZS )ÚMishActivationzÑ
    See Mish: A Self-Regularized Non-Monotonic Activation Function (Misra., https://arxiv.org/abs/1908.08681). Also
    visit the official repository for the paper: https://github.com/digantamisra98/Mish
    c                 óö   •— t         ‰| �  «        t        j                  t        j
                  «      t        j                  d«      k  r| j                  | _        y t        j                  j                  | _        y )Nz1.9.0)r   r   r   r   r   r   Ú_mish_pythonr:   r   r   Úmishr   s    €r   r   zMishActivation.__init__’   sL   ø€ Ü‰ÑÔÜ�=‰=œ×*Ñ*Ó+¬g¯m©m¸GÓ.DÒDØ×(Ñ(ˆD�Hä—}‘}×)Ñ)ˆD�Hr   r   r   c                 ól   — |t        j                  t        j                  j	                  |«      «      z  S r8   )r   r   r   r   Úsoftplusr   s     r   r]   zMishActivation._mish_python™   s%   € Ø”u—z‘z¤"§-¡-×"8Ñ"8¸Ó"?Ó@Ñ@Ð@r   c                 ó$   — | j                  |«      S r8   r>   r   s     r   r   zMishActivation.forwardœ   r?   r   )	r!   r"   r#   r$   r   r   r]   r   r%   r&   s   @r   r[   r[   Œ   s6   ø„ ñô
*ðA &ð A¨Vó Að˜Vð ¨÷ r   r[   c                   ó    — e Zd ZdZdedefd„Zy)ÚLinearActivationz[
    Applies the linear activation function, i.e. forwarding input directly to output.
    r   r   c                 ó   — |S r8   r3   r   s     r   r   zLinearActivation.forward¥   s   € Øˆr   Nr2   r3   r   r   rc   rc       s   „ ñð˜Vð ¨ô r   rc   c                   ó   — e Zd ZdZdd„Zy)ÚLaplaceActivationzó
    Applies elementwise activation based on Laplace function, introduced in MEGA as an attention activation. See
    https://arxiv.org/abs/2209.10655

    Inspired by squared relu, but with bounded range and gradient for better stability
    c                 óŒ   — ||z
  j                  |t        j                  d«      z  «      }ddt        j                  |«      z   z  S )Nr,   r*   r+   )Údivr.   r/   r   r<   )r   r   ÚmuÚsigmas       r   r   zLaplaceActivation.forward±   s<   € Ø˜‘× Ñ  ¬¯©°3«Ñ!7Ó8ˆØ�cœEŸI™I eÓ,Ñ,Ñ-Ð-r   N)g»¹øÛž æ?g ^×/ØÒ?©r!   r"   r#   r$   r   r3   r   r   rf   rf   ©   s   „ ñô.r   rf   c                   ó   — e Zd ZdZd„ Zy)ÚReLUSquaredActivationzX
    Applies the relu^2 activation introduced in https://arxiv.org/abs/2109.08668v2
    c                 ón   — t         j                  j                  |«      }t        j                  |«      }|S r8   )r   r   Úrelur   Úsquare)r   r   Úrelu_appliedÚsquareds       r   r   zReLUSquaredActivation.forward»   s)   € Ü—}‘}×)Ñ)¨%Ó0ˆÜ—,‘,˜|Ó,ˆØˆr   Nrk   r3   r   r   rm   rm   ¶   s   „ ñór   rm   c                   ó   ‡ — e Zd Zˆ fd„Zˆ xZS )ÚClassInstantierc                 ód   •— t         ‰| �  |«      }t        |t        «      r|n|i f\  }} |di |¤ŽS )Nr3   )r   Ú__getitem__Ú
isinstanceÚtuple)r   ÚkeyÚcontentÚclsÚkwargsr   s        €r   rv   zClassInstantier.__getitem__Â   s6   ø€ Ü‘'Ñ% cÓ*ˆÜ!+¨G´UÔ!;‘gÀ'È2À‰ˆˆVÙ‰}�V‰}Ðr   )r!   r"   r#   rv   r%   r&   s   @r   rt   rt   Á   s   ø„ ÷ð r   rt   r   Úgelu_10iöÿÿÿé
   )rJ   rK   Ú	gelu_fastÚgelu_newÚgelu_pythonr6   TÚgelu_pytorch_tanhÚgelu_accurateÚlaplaceÚ
leaky_reluÚlinearr^   Ú
quick_geluro   Úrelu2Úrelu6rG   Úsilu)Úswishr   Úpreluc           	      ó|   — | t         v r	t         |    S t        d| › dt        t         j                  «       «      › �«      ‚)Nz	function z not found in ACT2FN mapping )ÚACT2FNÚKeyErrorÚlistÚkeys)Úactivation_strings    r   Úget_activationr“   á   sB   € ØœFÑ"ÜÐ'Ñ(Ð(ä˜Ð#4Ð"5Ð5RÔSWÔX^×XcÑXcÓXeÓSfÐRgÐhÓiÐir   ),r.   Úcollectionsr   r   Ú	packagingr   r   r   Úutilsr   Ú
get_loggerr!   ÚloggerÚModuler
   r(   r5   rB   rE   rI   rT   r[   rc   rf   rm   rt   Ú	LeakyReLUÚReLUÚReLU6ÚSigmoidÚSiLUÚTanhÚPReLUÚACT2CLSrŽ   r“   r�   r€   r   r   r‡   rŠ   r^   Ú
linear_actr3   r   r   ú<module>r£      so  ðó Ý #ã Ý ß å ð 
ˆ×	Ñ	˜HÓ	%€ô=�b—i‘iô =ô*w˜Ÿ	™	ô wô�R—Y‘Yô ô,i˜Ÿ™ô iô4˜"Ÿ)™)ô 4ô7˜BŸI™Iô 7ô2t˜RŸY™Yô tô �R—Y‘Yô ô(�r—y‘yô ô
.˜Ÿ	™	ô 
.ô˜BŸI™Iô ô�kô ðØ
ˆNðàÐ%¨s¸2Ñ'>Ð?ðð Ð#ðð Ð!ð	ð
 �NÐ%6¸Ð$=Ð>ðð ˜ðð Ð+ðð Ð ðð �"—,‘,ðð Ððð ˆNðð Ð%ðð ˆB�G‰Gðð Ð"ðð ˆR�X‰Xðð  ˆr�z‰zð!ð" ˆB�G‰Gð#ð$ �W‰WØ�G‰GØ�X‰Xò)€ñ, 
˜Ó	!€òjñ ˜]Ó+€Ù˜*Ó%€Ù�fÓ€Ù˜;Ó'€	Ù˜LÓ)€
Ù�fÓ€Ù�fÓ€Ù˜HÓ%�
r   