Ë
    S^(h‰ô  ã                   óæ  — d dl mZmZmZ d dlZd dlmZ d dlZd dl	m
Z d dl
Zd dlmZmZmZ d dlmZmZ d dlmZ d dlmZ d dlmZmZ d dlmZ d	d
lmZmZmZmZm Z m!Z!m"Z"m#Z# d	dl$m%Z%m&Z&m'Z'm(Z(m)Z) d	dl*m+Z+m,Z,m-Z-m.Z. ddl/m0Z0  e.jb                  e2«      Z3dZ4dZ5ejl                  Z6ejn                  jp                   G d„ de+«      «       Z9dZ:dZ; G d„ dejx                  «      Z= G d„ dejx                  «      Z> G d„ dejx                  «      Z? G d„ dejx                  «      Z@ G d„ dejx                  «      ZA G d„ d ejx                  «      ZB G d!„ d"ejx                  «      ZC G d#„ d$ejx                  «      ZD G d%„ d&ejx                  «      ZE G d'„ d(ejx                  «      ZF G d)„ d*ejx                  «      ZG G d+„ d,e&«      ZH G d-„ d.ejx                  «      ZI e,d/e:«       G d0„ d1eH«      «       ZJ e'eJe4ee5«        G d2„ d3ejx                  «      ZK G d4„ d5ejx                  «      ZL e,d6e:«       G d7„ d8eH«      «       ZM e'eMe4ee5«        G d9„ d:ejx                  «      ZN e,d;e:«       G d<„ d=eH«      «       ZOd>ZP e)eOe;j£                  d?«      ePz   «        e(eOe9e5¬@«        G dA„ dBejx                  «      ZR e,dCe:«       G dD„ dEeH«      «       ZS e'eSe4e#e5«       dF„ ZT G dG„ dHejx                  «      ZU G dI„ dJejx                  «      ZV e,dKe:«       G dL„ dMeH«      «       ZW e)eWe;j£                  dN«      «        e'eWe4e e5«        G dO„ dPejx                  «      ZX e,dQe:«       G dR„ dSeH«      «       ZY e'eYe4e!e5«        G dT„ dUejx                  «      ZZ G dV„ dWejx                  «      Z[ e,dXe:«       G dY„ dZeH«      «       Z\ e'e\e4e"e5«        G d[„ d\ejx                  «      Z] e,d]e:«       G d^„ d_eH«      «       Z^ e'e^e4ee5«       g d`¢Z_y)aé    )ÚCallableÚOptionalÚTupleN)Ú
FrozenDictÚfreezeÚunfreeze)Úcombine_masksÚmake_causal_mask)Úpartitioning)Údot_product_attention_weights)Úflatten_dictÚunflatten_dict)Úlaxé   )ÚFlaxBaseModelOutputÚ-FlaxBaseModelOutputWithPastAndCrossAttentionsÚ%FlaxCausalLMOutputWithCrossAttentionsÚFlaxMaskedLMOutputÚFlaxMultipleChoiceModelOutputÚ FlaxQuestionAnsweringModelOutputÚFlaxSequenceClassifierOutputÚFlaxTokenClassifierOutput)ÚACT2FNÚFlaxPreTrainedModelÚappend_call_sample_docstringÚ append_replace_return_docstringsÚoverwrite_call_docstring)ÚModelOutputÚadd_start_docstringsÚ%add_start_docstrings_to_model_forwardÚloggingé   )ÚElectraConfigz"google/electra-small-discriminatorr#   c                   ó�   — e Zd ZU dZdZej                  ed<   dZe	e
ej                        ed<   dZe	e
ej                        ed<   y)ÚFlaxElectraForPreTrainingOutputaa  
    Output type of [`ElectraForPreTraining`].

    Args:
        logits (`jnp.ndarray` of shape `(batch_size, sequence_length, config.vocab_size)`):
            Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
        hidden_states (`tuple(jnp.ndarray)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
            Tuple of `jnp.ndarray` (one for the output of the embeddings + one for the output of each layer) of shape
            `(batch_size, sequence_length, hidden_size)`.

            Hidden-states of the model at the output of each layer plus the initial embedding outputs.
        attentions (`tuple(jnp.ndarray)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
            Tuple of `jnp.ndarray` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
            sequence_length)`.

            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
            heads.
    NÚlogitsÚhidden_statesÚ
attentions)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r&   ÚjnpÚndarrayÚ__annotations__r'   r   r   r(   © ó    úo/var/www/skyplay_api_hub/venv/lib/python3.12/site-packages/transformers/models/electra/modeling_flax_electra.pyr%   r%   ;   sG   … ñð& €FˆC�K‰KÓØ26€M�8˜E #§+¡+Ñ.Ñ/Ó6Ø/3€J�˜˜sŸ{™{Ñ+Ñ,Ô3r1   r%   a  

    This model inherits from [`FlaxPreTrainedModel`]. Check the superclass documentation for the generic methods the
    library implements for all its model (such as downloading, saving and converting weights from PyTorch models)

    This model is also a Flax Linen
    [flax.nn.Module](https://flax.readthedocs.io/en/latest/_autosummary/flax.nn.module.html) subclass. Use it as a
    regular Flax Module and refer to the Flax documentation for all matter related to general usage and behavior.

    Finally, this model supports inherent JAX features such as:

    - [Just-In-Time (JIT) compilation](https://jax.readthedocs.io/en/latest/jax.html#just-in-time-compilation-jit)
    - [Automatic Differentiation](https://jax.readthedocs.io/en/latest/jax.html#automatic-differentiation)
    - [Vectorization](https://jax.readthedocs.io/en/latest/jax.html#vectorization-vmap)
    - [Parallelization](https://jax.readthedocs.io/en/latest/jax.html#parallelization-pmap)

    Parameters:
        config ([`ElectraConfig`]): Model configuration class with all the parameters of the model.
            Initializing with a config file does not load the weights associated with the model, only the
            configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
a³  
    Args:
        input_ids (`numpy.ndarray` of shape `({0})`):
            Indices of input sequence tokens in the vocabulary.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are input IDs?](../glossary#input-ids)
        attention_mask (`numpy.ndarray` of shape `({0})`, *optional*):
            Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:

            - 1 for tokens that are **not masked**,
            - 0 for tokens that are **masked**.

            [What are attention masks?](../glossary#attention-mask)
        token_type_ids (`numpy.ndarray` of shape `({0})`, *optional*):
            Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
            1]`:

            - 0 corresponds to a *sentence A* token,
            - 1 corresponds to a *sentence B* token.

            [What are token type IDs?](../glossary#token-type-ids)
        position_ids (`numpy.ndarray` of shape `({0})`, *optional*):
            Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
            config.max_position_embeddings - 1]`.
        head_mask (`numpy.ndarray` of shape `({0})`, `optional):
            Mask to nullify selected heads of the attention modules. Mask values selected in `[0, 1]`:

            - 1 indicates the head is **not masked**,
            - 0 indicates the head is **masked**.

        return_dict (`bool`, *optional*):
            Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.

c                   óf   — e Zd ZU dZeed<   ej                  Zej                  ed<   d„ Z	dde
fd„Zy)	ÚFlaxElectraEmbeddingszGConstruct the embeddings from word, position and token_type embeddings.ÚconfigÚdtypec                 óÜ  — t        j                  | j                  j                  | j                  j                  t
        j                   j                  j                  | j                  j                  ¬«      ¬«      | _	        t        j                  | j                  j                  | j                  j                  t
        j                   j                  j                  | j                  j                  ¬«      ¬«      | _        t        j                  | j                  j                  | j                  j                  t
        j                   j                  j                  | j                  j                  ¬«      ¬«      | _        t        j                  | j                  j                  | j                   ¬«      | _        t        j"                  | j                  j$                  ¬«      | _        y )N)Ústddev)Úembedding_init©Úepsilonr6   ©Úrate)ÚnnÚEmbedr5   Ú
vocab_sizeÚembedding_sizeÚjaxÚinitializersÚnormalÚinitializer_rangeÚword_embeddingsÚmax_position_embeddingsÚposition_embeddingsÚtype_vocab_sizeÚtoken_type_embeddingsÚ	LayerNormÚlayer_norm_epsr6   ÚDropoutÚhidden_dropout_probÚdropout©Úselfs    r2   ÚsetupzFlaxElectraEmbeddings.setup˜   s5  € Ü!Ÿx™xØ�K‰K×"Ñ"Ø�K‰K×&Ñ&ÜŸ6™6×.Ñ.×5Ñ5¸T¿[¹[×=ZÑ=ZÐ5Ó[ô 
ˆÔô
 $&§8¡8Ø�K‰K×/Ñ/Ø�K‰K×&Ñ&ÜŸ6™6×.Ñ.×5Ñ5¸T¿[¹[×=ZÑ=ZÐ5Ó[ô$
ˆÔ ô
 &(§X¡XØ�K‰K×'Ñ'Ø�K‰K×&Ñ&ÜŸ6™6×.Ñ.×5Ñ5¸T¿[¹[×=ZÑ=ZÐ5Ó[ô&
ˆÔ"ô
 Ÿ™¨d¯k©k×.HÑ.HÐPT×PZÑPZÔ[ˆŒÜ—z‘z t§{¡{×'FÑ'FÔGˆ�r1   Údeterministicc                 ó  — | j                  |j                  d«      «      }| j                  |j                  d«      «      }| j                  |j                  d«      «      }||z   |z   }	| j	                  |	«      }	| j                  |	|¬«      }	|	S )NÚi4©rS   )rF   ÚastyperH   rJ   rK   rO   )
rQ   Ú	input_idsÚtoken_type_idsÚposition_idsÚattention_maskrS   Úinputs_embedsÚposition_embedsrJ   r'   s
             r2   Ú__call__zFlaxElectraEmbeddings.__call__¬   s�   € à×,Ñ,¨Y×-=Ñ-=¸dÓ-CÓDˆØ×2Ñ2°<×3FÑ3FÀtÓ3LÓMˆØ $× :Ñ :¸>×;PÑ;PÐQUÓ;VÓ WÐð &Ð(=Ñ=ÀÑOˆð Ÿ™ }Ó5ˆØŸ™ ]À-˜ÓPˆØÐr1   N©T©r)   r*   r+   r,   r#   r/   r-   Úfloat32r6   rR   Úboolr^   r0   r1   r2   r4   r4   ’   s0   … ÙQàÓØ—{‘{€Eˆ3�9‰9Ó"òHñ(Ð_cô r1   r4   c                   óÊ   — e Zd ZU eed<   dZeed<   ej                  Z	ej                  ed<   d„ Z
d„ Zd„ Zej                  d„ «       Z	 	 	 	 dd
eej"                     dedefd„Zy	)ÚFlaxElectraSelfAttentionr5   FÚcausalr6   c                 ó6  — | j                   j                  | j                   j                  z  | _        | j                   j                  | j                   j                  z  dk7  rt	        d«      ‚t        j                  | j                   j                  | j                  t        j
                  j                  j                  | j                   j                  «      ¬«      | _        t        j                  | j                   j                  | j                  t        j
                  j                  j                  | j                   j                  «      ¬«      | _        t        j                  | j                   j                  | j                  t        j
                  j                  j                  | j                   j                  «      ¬«      | _        | j                  r>t!        t#        j$                  d| j                   j&                  fd¬«      d¬«      | _        y y )Nr   z›`config.hidden_size`: {self.config.hidden_size} has to be a multiple of `config.num_attention_heads`                    : {self.config.num_attention_heads})r6   Úkernel_initr"   rb   ©r6   )r5   Úhidden_sizeÚnum_attention_headsÚhead_dimÚ
ValueErrorr>   ÚDenser6   rB   rC   rD   rE   ÚqueryÚkeyÚvaluere   r
   r-   ÚonesrG   Úcausal_maskrP   s    r2   rR   zFlaxElectraSelfAttention.setupÁ   si  € ØŸ™×/Ñ/°4·;±;×3RÑ3RÑRˆŒØ�;‰;×"Ñ" T§[¡[×%DÑ%DÑDÈÒIÜðIóð ô
 —X‘XØ�K‰K×#Ñ#Ø—*‘*ÜŸ™×+Ñ+×2Ñ2°4·;±;×3PÑ3PÓQô
ˆŒ
ô
 —8‘8Ø�K‰K×#Ñ#Ø—*‘*ÜŸ™×+Ñ+×2Ñ2°4·;±;×3PÑ3PÓQô
ˆŒô
 —X‘XØ�K‰K×#Ñ#Ø—*‘*ÜŸ™×+Ñ+×2Ñ2°4·;±;×3PÑ3PÓQô
ˆŒ
ð �;Š;Ü/Ü—‘˜!˜TŸ[™[×@Ñ@ÐAÈÔPÐX^ô ˆDÕð r1   c                 ó„   — |j                  |j                  d d | j                  j                  | j                  fz   «      S ©Né   )ÚreshapeÚshaper5   rj   rk   ©rQ   r'   s     r2   Ú_split_headsz%FlaxElectraSelfAttention._split_headsÞ   s;   € Ø×$Ñ$ ]×%8Ñ%8¸¸!Ð%<ÀÇÁ×@_Ñ@_Ðae×anÑanÐ?oÑ%oÓpÐpr1   c                 ón   — |j                  |j                  d d | j                  j                  fz   «      S rt   )rv   rw   r5   ri   rx   s     r2   Ú_merge_headsz%FlaxElectraSelfAttention._merge_headsá   s2   € Ø×$Ñ$ ]×%8Ñ%8¸¸!Ð%<ÀÇÁ×@WÑ@WÐ?YÑ%YÓZÐZr1   c                 ó(  — | j                  dd«      }| j                  ddt        j                  |j                  |j
                  «      }| j                  ddt        j                  |j                  |j
                  «      }| j                  ddd„ «      }|rø|j                  j                  �^ }	}
}}|j                  }dt        |	«      z  |ddfz   }t        j                  |j                  ||«      }t        j                  |j                  ||«      }||_        ||_        |j                  d   }|j                  |z   |_        t        j                  t        j                  |
«      ||z   k  t        |	«      d||
fz   «      }t        ||«      }|||fS )	a\  
        This function takes projected key, value states from a single input token and concatenates the states to cached
        states from previous steps. This function is slightly adapted from the official Flax repository:
        https://github.com/google/flax/blob/491ce18759622506588784b4fca0e4bf05f8c8cd/flax/linen/attention.py#L252
        ÚcacheÚ
cached_keyÚcached_valueÚcache_indexc                  óL   — t        j                  dt         j                  ¬«      S )Nr   rh   )r-   ÚarrayÚint32r0   r1   r2   ú<lambda>z@FlaxElectraSelfAttention._concatenate_to_cache.<locals>.<lambda>ð   s   € ÄCÇIÁIÈaÔWZ×W`ÑW`ÔDa€ r1   ©r   r   r"   )Úhas_variableÚvariabler-   Úzerosrw   r6   rp   Úlenr   Údynamic_update_sliceÚbroadcast_toÚarangeÚtupler	   )rQ   ro   rp   rn   r[   Úis_initializedr~   r   r€   Ú
batch_dimsÚ
max_lengthÚ	num_headsÚdepth_per_headÚ	cur_indexÚindicesÚnum_updated_cache_vectorsÚpad_masks                    r2   Ú_concatenate_to_cachez.FlaxElectraSelfAttention._concatenate_to_cacheä   sr  € ð ×*Ñ*¨7°LÓAˆØ—]‘] 7¨L¼#¿)¹)ÀSÇYÁYÐPS×PYÑPYÓZˆ
Ø—}‘} W¨n¼c¿i¹iÈÏÉÐV[×VaÑVaÓbˆØ—m‘m G¨]Ñ<aÓbˆáØAK×AQÑAQ×AWÑAWÑ>ˆZ˜ Y°à#×)Ñ)ˆIØœS ›_Ñ,°	¸1¸aÐ/@Ñ@ˆGÜ×*Ñ*¨:×+;Ñ+;¸SÀ'ÓJˆCÜ×,Ñ,¨\×-?Ñ-?ÀÈÓPˆEØ"ˆJÔØ!&ˆLÔØ(-¯©°A©Ð%Ø +× 1Ñ 1Ð4MÑ MˆKÔä×'Ñ'Ü—
‘
˜:Ó&¨Ð5NÑ)NÑNÜ�jÓ! QÐ(AÀ:Ð$NÑNóˆHô +¨8°^ÓDˆNØ�E˜>Ð)Ð)r1   NÚkey_value_statesÚ
init_cacheÚoutput_attentionsc                 ó4  — |d u}|j                   d   }	| j                  |«      }
|r#| j                  |«      }| j                  |«      }n"| j                  |«      }| j                  |«      }| j	                  |
«      }
| j	                  |«      }| j	                  |«      }| j
                  rÍ|
j                   d   |j                   d   }}| j                  dd«      r[| j                  d   d   }| j                  d   d   j                   d   }t        j                  | j                  dd|dfdd||f«      }n| j                  d d …d d …d |…d |…f   }t        j                  ||	f|j                   dd  z   «      }|�N| j
                  rBt        j                  t        j                  |d¬«      j                   «      }t        ||«      }n(| j
                  r}n|�t        j                  |d¬«      }| j
                  r,| j                  dd«      s|r| j                  |||
|«      \  }}}|�°t        j                   |dkD  t        j"                  |j                   d«      j%                  | j&                  «      t        j"                  |j                   t        j(                  | j&                  «      j*                  «      j%                  | j&                  «      «      }nd }d }|s*| j,                  j.                  dkD  r| j1                  d	«      }t3        |
|||| j,                  j.                  d
|| j&                  d ¬«	      }|�t        j4                  d||«      }t        j4                  d||«      }|j7                  |j                   d d dz   «      }|r||f}|S |f}|S )Nr   r"   r}   r~   r€   )éýÿÿÿéþÿÿÿ©Úaxisg        rO   T)ÚbiasÚdropout_rngÚdropout_rateÚbroadcast_dropoutrS   r6   Ú	precisionz...hqk,h->...hqkz...hqk,...khd->...qhdru   )éÿÿÿÿ)rw   rn   ro   rp   ry   re   r†   Ú	variablesr   Údynamic_slicerr   r-   r‹   Úexpand_dimsr	   r—   ÚselectÚfullrW   r6   ÚfinfoÚminr5   Úattention_probs_dropout_probÚmake_rngr   Úeinsumrv   )rQ   r'   r[   Úlayer_head_maskr˜   r™   rS   rš   Úis_cross_attentionÚ
batch_sizeÚquery_statesÚ
key_statesÚvalue_statesÚquery_lengthÚ
key_lengthÚ
mask_shiftÚmax_decoder_lengthrr   Úattention_biasr¡   Úattn_weightsÚattn_outputÚoutputss                          r2   r^   z!FlaxElectraSelfAttention.__call__  sk  € ð .°TÐ9ÐØ"×(Ñ(¨Ñ+ˆ
ð —z‘z -Ó0ˆáàŸ™Ð"2Ó3ˆJØŸ:™:Ð&6Ó7‰Lð Ÿ™ -Ó0ˆJØŸ:™: mÓ4ˆLà×(Ñ(¨Ó6ˆØ×&Ñ& zÓ2ˆ
Ø×(Ñ(¨Ó6ˆð �;Š;Ø'3×'9Ñ'9¸!Ñ'<¸j×>NÑ>NÈqÑ>Q˜*ˆLØ× Ñ  ¨,Ô7Ø!Ÿ^™^¨GÑ4°]ÑC�
Ø%)§^¡^°GÑ%<¸\Ñ%J×%PÑ%PÐQRÑ%SÐ"Ü!×/Ñ/Ø×$Ñ$ q¨!¨Z¸Ð&;¸aÀÀLÐRdÐ=eó‘ð #×.Ñ.ªq²!°]°l°]ÀKÀZÀKÐ/OÑP�Ü×*Ñ*¨;¸¸È×HYÑHYÐZ[ÐZ\ÐH]Ñ8]Ó^ˆKð Ð%¨$¯+ª+Ü ×-Ñ-¬c¯o©o¸nÐS[Ô.\Ð^i×^oÑ^oÓpˆNÜ*¨>¸;ÓG‰NØ�[Š[Ø(‰NØÐ'Ü Ÿ_™_¨^À(ÔKˆNð �;Š;˜D×-Ñ-¨g°|ÔDÉ
Ø7;×7QÑ7QØ˜L¨,¸ó8Ñ4ˆJ˜ nð
 Ð%ä ŸZ™ZØ Ñ"Ü—‘˜×-Ñ-¨sÓ3×:Ñ:¸4¿:¹:ÓFÜ—‘˜×-Ñ-¬s¯y©y¸¿¹Ó/D×/HÑ/HÓI×PÑPÐQU×Q[ÑQ[Ó\ó‰Nð "ˆNàˆÙ §¡×!IÑ!IÈCÒ!OØŸ-™-¨	Ó2ˆKä4ØØØØ#ØŸ™×AÑAØ"Ø'Ø—*‘*Øô

ˆð Ð&ÜŸ:™:Ð&8¸,ÈÓXˆLä—j‘jÐ!8¸,ÈÓUˆØ!×)Ñ)¨+×*;Ñ*;¸B¸QÐ*?À%Ñ*GÓHˆá1B�; Ð-ˆØˆð JUÈˆØˆr1   ©NFTF)r)   r*   r+   r#   r/   re   rb   r-   ra   r6   rR   ry   r{   r>   Úcompactr—   r   r.   r^   r0   r1   r2   rd   rd   ¼   sŒ   … ØÓØ€FˆDÓØ—{‘{€Eˆ3�9‰9Ó"òò:qò[ð ‡Z�Zñ*ó ð*ðH 37Ø ØØ"'ñ_ð
 # 3§;¡;Ñ/ð_ð ð_ð  ô_r1   rd   c                   ób   — e Zd ZU eed<   ej                  Zej                  ed<   d„ Zdde	fd„Z
y)ÚFlaxElectraSelfOutputr5   r6   c                 óÂ  — t        j                  | j                  j                  t        j                   j
                  j                  | j                  j                  «      | j                  ¬«      | _	        t        j                  | j                  j                  | j                  ¬«      | _
        t        j                  | j                  j                  ¬«      | _        y )N©rg   r6   r:   r<   )r>   rm   r5   ri   rB   rC   rD   rE   r6   ÚdenserK   rL   rM   rN   rO   rP   s    r2   rR   zFlaxElectraSelfOutput.setupl  s‡   € Ü—X‘XØ�K‰K×#Ñ#ÜŸ™×+Ñ+×2Ñ2°4·;±;×3PÑ3PÓQØ—*‘*ô
ˆŒ
ô
 Ÿ™¨d¯k©k×.HÑ.HÐPT×PZÑPZÔ[ˆŒÜ—z‘z t§{¡{×'FÑ'FÔGˆ�r1   rS   c                 óv   — | j                  |«      }| j                  ||¬«      }| j                  ||z   «      }|S ©NrV   ©rÄ   rO   rK   )rQ   r'   Úinput_tensorrS   s       r2   r^   zFlaxElectraSelfOutput.__call__u  s;   € ØŸ
™
 =Ó1ˆØŸ™ ]À-˜ÓPˆØŸ™ }°|Ñ'CÓDˆØÐr1   Nr_   ©r)   r*   r+   r#   r/   r-   ra   r6   rR   rb   r^   r0   r1   r2   rÁ   rÁ   h  s,   … ØÓØ—{‘{€Eˆ3�9‰9Ó"òHñÀ4ô r1   rÁ   c                   óx   — e Zd ZU eed<   dZeed<   ej                  Z	ej                  ed<   d„ Z
	 	 	 	 d	defd„Zy)
ÚFlaxElectraAttentionr5   Fre   r6   c                 ó²   — t        | j                  | j                  | j                  ¬«      | _        t        | j                  | j                  ¬«      | _        y )N©re   r6   rh   )rd   r5   re   r6   rQ   rÁ   ÚoutputrP   s    r2   rR   zFlaxElectraAttention.setup‚  s7   € Ü,¨T¯[©[ÀÇÁÐTX×T^ÑT^Ô_ˆŒ	Ü+¨D¯K©K¸t¿z¹zÔJˆ�r1   Nrš   c           	      ó„   — | j                  |||||||¬«      }|d   }	| j                  |	||¬«      }|f}
|r	|
|d   fz  }
|
S )N)r°   r˜   r™   rS   rš   r   rV   r"   )rQ   rÎ   )rQ   r'   r[   r°   r˜   r™   rS   rš   Úattn_outputsr¼   r½   s              r2   r^   zFlaxElectraAttention.__call__†  sl   € ð —y‘yØØØ+Ø-Ø!Ø'Ø/ð !ó 
ˆð # 1‘oˆØŸ™ K°Èm˜Ó\ˆà Ð"ˆáØ˜ Q™Ð)Ñ)ˆGàˆr1   r¾   )r)   r*   r+   r#   r/   re   rb   r-   ra   r6   rR   r^   r0   r1   r2   rË   rË   }  sG   … ØÓØ€FˆDÓØ—{‘{€Eˆ3�9‰9Ó"òKð ØØØ"'ñð  ôr1   rË   c                   óZ   — e Zd ZU eed<   ej                  Zej                  ed<   d„ Zd„ Z	y)ÚFlaxElectraIntermediater5   r6   c                 ó4  — t        j                  | j                  j                  t        j                   j
                  j                  | j                  j                  «      | j                  ¬«      | _	        t        | j                  j                     | _        y )NrÃ   )r>   rm   r5   Úintermediate_sizerB   rC   rD   rE   r6   rÄ   r   Ú
hidden_actÚ
activationrP   s    r2   rR   zFlaxElectraIntermediate.setup¬  s`   € Ü—X‘XØ�K‰K×)Ñ)ÜŸ™×+Ñ+×2Ñ2°4·;±;×3PÑ3PÓQØ—*‘*ô
ˆŒ
ô
 ! §¡×!7Ñ!7Ñ8ˆ�r1   c                 óJ   — | j                  |«      }| j                  |«      }|S ©N)rÄ   rÖ   rx   s     r2   r^   z FlaxElectraIntermediate.__call__´  s$   € ØŸ
™
 =Ó1ˆØŸ™¨Ó6ˆØÐr1   N©
r)   r*   r+   r#   r/   r-   ra   r6   rR   r^   r0   r1   r2   rÒ   rÒ   ¨  s$   … ØÓØ—{‘{€Eˆ3�9‰9Ó"ò9ór1   rÒ   c                   ób   — e Zd ZU eed<   ej                  Zej                  ed<   d„ Zdde	fd„Z
y)ÚFlaxElectraOutputr5   r6   c                 óÂ  — t        j                  | j                  j                  t        j                   j
                  j                  | j                  j                  «      | j                  ¬«      | _	        t        j                  | j                  j                  ¬«      | _        t        j                  | j                  j                  | j                  ¬«      | _        y )NrÃ   r<   r:   )r>   rm   r5   ri   rB   rC   rD   rE   r6   rÄ   rM   rN   rO   rK   rL   rP   s    r2   rR   zFlaxElectraOutput.setup¿  s‡   € Ü—X‘XØ�K‰K×#Ñ#ÜŸ™×+Ñ+×2Ñ2°4·;±;×3PÑ3PÓQØ—*‘*ô
ˆŒ
ô
 —z‘z t§{¡{×'FÑ'FÔGˆŒÜŸ™¨d¯k©k×.HÑ.HÐPT×PZÑPZÔ[ˆ�r1   rS   c                 óv   — | j                  |«      }| j                  ||¬«      }| j                  ||z   «      }|S rÆ   rÇ   )rQ   r'   Úattention_outputrS   s       r2   r^   zFlaxElectraOutput.__call__È  s<   € ØŸ
™
 =Ó1ˆØŸ™ ]À-˜ÓPˆØŸ™ }Ð7GÑ'GÓHˆØÐr1   Nr_   rÉ   r0   r1   r2   rÛ   rÛ   »  s,   … ØÓØ—{‘{€Eˆ3�9‰9Ó"ò\ñÀtô r1   rÛ   c                   ó°   — e Zd ZU eed<   ej                  Zej                  ed<   d„ Z	 	 	 	 	 dde	ej                     de	ej                     deded	ef
d
„Zy)ÚFlaxElectraLayerr5   r6   c                 óŽ  — t        | j                  | j                  j                  | j                  ¬«      | _        t        | j                  | j                  ¬«      | _        t        | j                  | j                  ¬«      | _        | j                  j                  r(t        | j                  d| j                  ¬«      | _
        y y )NrÍ   rh   F)rË   r5   Ú
is_decoderr6   Ú	attentionrÒ   ÚintermediaterÛ   rÎ   Úadd_cross_attentionÚcrossattentionrP   s    r2   rR   zFlaxElectraLayer.setupÔ  s‚   € Ü-¨d¯k©kÀ$Ç+Á+×BXÑBXÐ`d×`jÑ`jÔkˆŒÜ3°D·K±KÀtÇzÁzÔRˆÔÜ'¨¯©¸4¿:¹:ÔFˆŒØ�;‰;×*Ò*Ü"6°t·{±{È5ÐX\×XbÑXbÔ"cˆDÕð +r1   NÚencoder_hidden_statesÚencoder_attention_maskr™   rS   rš   c	                 óö   — | j                  ||||||¬«      }	|	d   }
|�| j                  |
|||||¬«      }|d   }
| j                  |
«      }| j                  ||
|¬«      }|f}|r||	d   fz  }|�	|d   fz  }|S )N)r°   r™   rS   rš   r   )r[   r°   r˜   rS   rš   rV   r"   )rã   ræ   rä   rÎ   )rQ   r'   r[   r°   rç   rè   r™   rS   rš   Úattention_outputsrÞ   Úcross_attention_outputsr½   s                r2   r^   zFlaxElectraLayer.__call__Û  s×   € ð !ŸN™NØØØ+Ø!Ø'Ø/ð +ó 
Ðð -¨QÑ/Ðð !Ð,Ø&*×&9Ñ&9Ø Ø5Ø /Ø!6Ø+Ø"3ð ':ó 'Ð#ð  7°qÑ9Ðà×)Ñ)Ð*:Ó;ˆØŸ™ MÐ3CÐS`˜Óaˆà Ð"ˆáØÐ)¨!Ñ,Ð.Ñ.ˆGØ$Ð0ØÐ3°AÑ6Ð8Ñ8�Øˆr1   )NNFTF)r)   r*   r+   r#   r/   r-   ra   r6   rR   r   r.   rb   r^   r0   r1   r2   rà   rà   Ð  sz   … ØÓØ—{‘{€Eˆ3�9‰9Ó"òdð 8<Ø8<Ø Ø"Ø"'ñ+ð
  (¨¯©Ñ4ð+ð !)¨¯©Ñ 5ð+ð ð+ð ð+ð  ô+r1   rà   c                   óÊ   — e Zd ZU eed<   ej                  Zej                  ed<   dZe	ed<   d„ Z
	 	 	 	 	 	 	 ddeej                     deej                     d	e	d
e	de	de	de	fd„Zy)ÚFlaxElectraLayerCollectionr5   r6   FÚgradient_checkpointingc           	      óº  — | j                   rjt        t        d¬«      }t        | j                  j
                  «      D �cg c]*  } || j                  t        |«      | j                  ¬«      ‘Œ, c}| _        y t        | j                  j
                  «      D �cg c]-  }t        | j                  t        |«      | j                  ¬«      ‘Œ/ c}| _        y c c}w c c}w )N)é   é   é   )Ústatic_argnums)Únamer6   )	rî   Úrematrà   Úranger5   Únum_hidden_layersÚstrr6   Úlayers)rQ   ÚFlaxElectraCheckpointLayerÚis      r2   rR   z FlaxElectraLayerCollection.setup  s¤   € Ø×&Ò&Ü).Ô/?ÐPYÔ)ZÐ&ô ˜tŸ{™{×<Ñ<Ó=öàñ +¨4¯;©;¼SÀ»VÈ4Ï:É:ÖVòˆD�Kô ˜tŸ{™{×<Ñ<Ó=öàô ! §¡´3°q³6ÀÇÁÖLòˆD�Kùòùò
s   ¿/CÂ2CNrç   rè   r™   rS   rš   Úoutput_hidden_statesÚreturn_dictc                 óî  — |rdnd }|	rdnd }|r|�dnd }|�W|j                   d   t        | j                  «      k7  r2t        dt        | j                  «      › d|j                   d   › d�«      ‚t	        | j                  «      D ]@  \  }}|	r||fz  } ||||�||   nd |||||«      }|d   }|sŒ,||d   fz  }|€Œ8||d   fz  }ŒB |	r||fz  }||||f}|
st        d„ |D «       «      S t        ||||¬	«      S )
Nr0   r   z&The head_mask should be specified for z/ layers, but it is for                         ú.r"   ru   c              3   ó&   K  — | ]	  }|€Œ|–— Œ y ­wrØ   r0   )Ú.0Úvs     r2   ú	<genexpr>z6FlaxElectraLayerCollection.__call__.<locals>.<genexpr>R  s   è ø€ Ò=˜q¨q©}œÑ=ùs   ‚Š)Úlast_hidden_stater'   r(   Úcross_attentions)rw   r‰   rù   rl   Ú	enumerater�   r   )rQ   r'   r[   Ú	head_maskrç   rè   r™   rS   rš   rü   rý   Úall_attentionsÚall_hidden_statesÚall_cross_attentionsrû   ÚlayerÚlayer_outputsr½   s                     r2   r^   z#FlaxElectraLayerCollection.__call__  sl  € ñ  1™°dˆÙ"6™B¸DÐÙ&7Ð<QÐ<]™rÐdhÐð Ð Ø�‰˜qÑ!¤c¨$¯+©+Ó&6Ò7Ü Ø<¼SÀÇÁÓ=MÐ<Nð OØ'Ÿo™o¨aÑ0Ð1°ð4óð ô
 " $§+¡+Ó.ò 	@‰HˆAˆuÙ#Ø! mÐ%5Ñ5Ð!á!ØØØ )Ð 5�	˜!’¸4Ø%Ø&ØØØ!ó	ˆMð *¨!Ñ,ˆMâ Ø =°Ñ#3Ð"5Ñ5�à(Ñ4Ø(¨]¸1Ñ-=Ð,?Ñ?Ñ(ð+	@ñ.  Ø -Ð!1Ñ1Ðà Ð"3°^ÐEYÐZˆáÜÑ= GÔ=Ó=Ð=ä<Ø+Ø+Ø%Ø1ô	
ð 	
r1   ©NNFTFFT©r)   r*   r+   r#   r/   r-   ra   r6   rî   rb   rR   r   r.   r^   r0   r1   r2   rí   rí   
  sž   … ØÓØ—{‘{€Eˆ3�9‰9Ó"Ø#(Ð˜DÓ(òð$ 8<Ø8<Ø Ø"Ø"'Ø%*Ø ñ=
ð
  (¨¯©Ñ4ð=
ð !)¨¯©Ñ 5ð=
ð ð=
ð ð=
ð  ð=
ð #ð=
ð ô=
r1   rí   c                   óÊ   — e Zd ZU eed<   ej                  Zej                  ed<   dZe	ed<   d„ Z
	 	 	 	 	 	 	 ddeej                     deej                     d	e	d
e	de	de	de	fd„Zy)ÚFlaxElectraEncoderr5   r6   Frî   c                 óf   — t        | j                  | j                  | j                  ¬«      | _        y )N©r6   rî   )rí   r5   r6   rî   r  rP   s    r2   rR   zFlaxElectraEncoder.setupb  s%   € Ü/Ø�K‰KØ—*‘*Ø#'×#>Ñ#>ô
ˆ�
r1   Nrç   rè   r™   rS   rš   rü   rý   c                 ó8   — | j                  |||||||||	|
¬«
      S )N)r  rç   rè   r™   rS   rš   rü   rý   )r  )rQ   r'   r[   r  rç   rè   r™   rS   rš   rü   rý   s              r2   r^   zFlaxElectraEncoder.__call__i  s8   € ð �z‰zØØØØ"7Ø#9Ø!Ø'Ø/Ø!5Ø#ð ó 
ð 	
r1   r  r  r0   r1   r2   r  r  ]  sž   … ØÓØ—{‘{€Eˆ3�9‰9Ó"Ø#(Ð˜DÓ(ò
ð 8<Ø8<Ø Ø"Ø"'Ø%*Ø ñ
ð
  (¨¯©Ñ4ð
ð !)¨¯©Ñ 5ð
ð ð
ð ð
ð  ð
ð #ð
ð ô
r1   r  c                   óZ   — e Zd ZU eed<   ej                  Zej                  ed<   d„ Zd„ Z	y)ÚFlaxElectraGeneratorPredictionsr5   r6   c                 óì   — t        j                  | j                  j                  | j                  ¬«      | _        t        j
                  | j                  j                  | j                  ¬«      | _        y )Nr:   rh   )r>   rK   r5   rL   r6   rm   rA   rÄ   rP   s    r2   rR   z%FlaxElectraGeneratorPredictions.setupˆ  sE   € ÜŸ™¨d¯k©k×.HÑ.HÐPT×PZÑPZÔ[ˆŒÜ—X‘X˜dŸk™k×8Ñ8ÀÇ
Á
ÔKˆ�
r1   c                 óŽ   — | j                  |«      }t        | j                  j                     |«      }| j	                  |«      }|S rØ   )rÄ   r   r5   rÕ   rK   rx   s     r2   r^   z(FlaxElectraGeneratorPredictions.__call__Œ  s=   € ØŸ
™
 =Ó1ˆÜ˜tŸ{™{×5Ñ5Ñ6°}ÓEˆØŸ™ }Ó5ˆØÐr1   NrÙ   r0   r1   r2   r  r  „  s%   … ØÓØ—{‘{€Eˆ3�9‰9Ó"òLór1   r  c                   ó^   — e Zd ZU dZeed<   ej                  Zej                  ed<   d„ Z	d„ Z
y)Ú#FlaxElectraDiscriminatorPredictionszEPrediction module for the discriminator, made up of two dense layers.r5   r6   c                 óÄ   — t        j                  | j                  j                  | j                  ¬«      | _        t        j                  d| j                  ¬«      | _        y )Nrh   r"   )r>   rm   r5   ri   r6   rÄ   Údense_predictionrP   s    r2   rR   z)FlaxElectraDiscriminatorPredictions.setup™  s9   € Ü—X‘X˜dŸk™k×5Ñ5¸T¿Z¹ZÔHˆŒ
Ü "§¡¨°$·*±*Ô =ˆÕr1   c                 ó¬   — | j                  |«      }t        | j                  j                     |«      }| j	                  |«      j                  d«      }|S )Nr¥   )rÄ   r   r5   rÕ   r  Úsqueezerx   s     r2   r^   z,FlaxElectraDiscriminatorPredictions.__call__�  sJ   € ØŸ
™
 =Ó1ˆÜ˜tŸ{™{×5Ñ5Ñ6°}ÓEˆØ×-Ñ-¨mÓ<×DÑDÀRÓHˆØÐr1   N)r)   r*   r+   r,   r#   r/   r-   ra   r6   rR   r^   r0   r1   r2   r  r  “  s'   … ÙOàÓØ—{‘{€Eˆ3�9‰9Ó"ò>ór1   r  c                   ó˜  ‡ — e Zd ZU dZeZdZdZej                  e
d<   ddej                  ddfd	ed
ededej                  dedefˆ fd„Zd„ Zddej(                  j*                  d
ededefd„Zd„ Z eej7                  d«      «      	 	 	 	 	 	 	 	 	 	 	 	 	 ddedej(                  j*                  dedee   dee   dee   defd„«       Zˆ xZS ) ÚFlaxElectraPreTrainedModelz†
    An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained
    models.
    ÚelectraNÚmodule_class)r"   r"   r   TFr5   Úinput_shapeÚseedr6   Ú_do_initrî   c                 ó\   •—  | j                   d|||dœ|¤Ž}t        ‰	| �	  ||||||¬«       y )N©r5   r6   rî   )r"  r#  r6   r$  r0   )r!  ÚsuperÚ__init__)
rQ   r5   r"  r#  r6   r$  rî   ÚkwargsÚmoduleÚ	__class__s
            €r2   r(  z#FlaxElectraPreTrainedModel.__init__®  sA   ø€ ð #�×"Ñ"Ðw¨&¸ÐVlÑwÐpvÑwˆÜ‰Ñ˜ °[ÀtÐSXÐckÐÕlr1   c                 ó^   — | j                  | j                  | j                  d¬«      | _        y )NTr&  )r!  r5   r6   Ú_modulerP   s    r2   Úenable_gradient_checkpointingz8FlaxElectraPreTrainedModel.enable_gradient_checkpointing¼  s*   € Ø×(Ñ(Ø—;‘;Ø—*‘*Ø#'ð )ó 
ˆ�r1   ÚrngÚparamsÚreturnc                 óÔ  — t        j                  |d¬«      }t        j                  |«      }t        j                  t        j                  t        j
                  |«      j                  d   «      |«      }t        j                  |«      }t        j                  | j                  j                  | j                  j                  f«      }t        j                  j                  |«      \  }	}
|	|
dœ}| j                  j                  rTt        j                  || j                  j                   fz   «      }|}| j"                  j%                  ||||||||d¬«	      }n"| j"                  j%                  ||||||d¬«      }|d   }|�dt'        t)        |«      «      }t'        t)        |«      «      }| j*                  D ]
  }||   ||<   Œ t-        «       | _        t/        t1        |«      «      S |S )NrU   rh   r¥   )r0  rO   F)rý   r0  )r-   rˆ   Ú
zeros_liker‹   rŒ   Ú
atleast_2drw   Ú	ones_likerq   r5   r÷   rj   rB   ÚrandomÚsplitrå   ri   r*  Úinitr   r   Ú_missing_keysÚsetr   r   )rQ   r/  r"  r0  rX   rY   rZ   r[   r  Ú
params_rngr¡   Úrngsrç   rè   Úmodule_init_outputsÚrandom_paramsÚmissing_keys                    r2   Úinit_weightsz'FlaxElectraPreTrainedModel.init_weightsÄ  s´  € ä—I‘I˜k°Ô6ˆ	ÜŸ™¨	Ó2ˆÜ×'Ñ'¬¯
©
´3·>±>À)Ó3L×3RÑ3RÐSUÑ3VÓ(WÐYdÓeˆÜŸ™ yÓ1ˆÜ—H‘H˜dŸk™k×;Ñ;¸T¿[¹[×=\Ñ=\Ð]Ó^ˆ	ä"%§*¡*×"2Ñ"2°3Ó"7Ñˆ
�KØ$°Ñ=ˆà�;‰;×*Ò*Ü$'§I¡I¨k¸T¿[¹[×=TÑ=TÐ<VÑ.VÓ$WÐ!Ø%3Ð"Ø"&§+¡+×"2Ñ"2ØØØØØØØ%Ø&Ø!ð #3ó 
#Ñð #'§+¡+×"2Ñ"2Ø�i °ÀÈyÐfkð #3ó #Ðð ,¨HÑ5ˆàÐÜ(¬°-Ó)@ÓAˆMÜ!¤(¨6Ó"2Ó3ˆFØ#×1Ñ1ò A�Ø&3°KÑ&@��{Ò#ðAä!$£ˆDÔÜœ.¨Ó0Ó1Ð1à Ð r1   c                 ó   — t        j                  ||fd¬«      }t        j                  |d¬«      }t        j                  t        j                  t        j
                  |«      j                  d   «      |j                  «      }| j                  j                  t        j                  j                  d«      |||dd¬«      }t        |d   «      S )	aW  
        Args:
            batch_size (`int`):
                batch_size used for fast auto-regressive decoding. Defines the batch size of the initialized cache.
            max_length (`int`):
                maximum possible length for auto-regressive decoding. Defines the sequence length of the initialized
                cache.
        rU   rh   r¥   r   FT)rý   r™   r}   )r-   rq   r5  r‹   rŒ   r4  rw   r*  r8  rB   r6  ÚPRNGKeyr   )rQ   r²   r�   rX   r[   rZ   Úinit_variabless          r2   r™   z%FlaxElectraPreTrainedModel.init_cacheï  s©   € ô —H‘H˜j¨*Ð5¸TÔBˆ	ÜŸ™ y¸Ô=ˆÜ×'Ñ'¬¯
©
´3·>±>À)Ó3L×3RÑ3RÐSUÑ3VÓ(WÐYb×YhÑYhÓiˆàŸ™×)Ñ)Ü�J‰J×Ñ˜qÓ! 9¨n¸lÐX]Ðjnð *ó 
ˆô ˜ wÑ/Ó0Ð0r1   úbatch_size, sequence_lengthr¡   Útrainrš   rü   rý   Úpast_key_valuesc                 óp  — |�|n| j                   j                  }|�|n| j                   j                  }|�|n| j                   j                  }|€t	        j
                  |«      }|€St	        j                  t	        j                  t	        j                  |«      j                  d   «      |j                  «      }|€t	        j
                  |«      }|€?t	        j                  | j                   j                  | j                   j                  f«      }i }|	�|	|d<   d|xs | j                  i}| j                   j                  râ|r	||d<   dg}nd}| j                  j!                  |t	        j"                  |d¬«      t	        j"                  |d¬«      t	        j"                  |d¬«      t	        j"                  |d¬«      t	        j"                  |d¬«      |||
 |||||¬«      }|�|r|\  }}t%        |d   «      |d	<   |S |�"|s |\  }}|d d
 t%        |d   «      fz   |d
d  z   }|S | j                  j!                  |t	        j"                  |d¬«      t	        j"                  |d¬«      t	        j"                  |d¬«      t	        j"                  |d¬«      t	        j"                  |d¬«      |
 ||||¬«      }|S )Nr¥   rO   r0  r}   FrU   rh   )rY   rZ   r  rç   rè   rS   rš   rü   rý   r<  ÚmutablerF  r"   )rY   rZ   r  rS   rš   rü   rý   r<  )r5   rš   rü   rý   r-   r5  r‹   rŒ   r4  rw   rq   r÷   rj   r0  rå   r*  Úapplyr‚   r   )rQ   rX   r[   rY   rZ   r  rç   rè   r0  r¡   rE  rš   rü   rý   rF  r<  ÚinputsrH  r½   s                      r2   r^   z#FlaxElectraPreTrainedModel.__call__  s¡  € ð$ 2CÐ1NÑ-ÐTX×T_ÑT_×TqÑTqÐà$8Ð$DÑ È$Ï+É+×JjÑJjð 	ð &1Ð%<‘kÀ$Ç+Á+×BYÑBYˆð Ð!Ü Ÿ]™]¨9Ó5ˆNàÐÜ×+Ñ+¬C¯J©J´s·~±~ÀiÓ7P×7VÑ7VÐWYÑ7ZÓ,[Ð]f×]lÑ]lÓmˆLàÐ!Ü Ÿ]™]¨9Ó5ˆNàÐÜŸ™ $§+¡+×"?Ñ"?ÀÇÁ×A`ÑA`Ð!aÓbˆIð ˆØÐ"Ø)ˆD�‰Oà˜FÒ1 d§k¡kÐ2ˆà�;‰;×*Ò*ñ Ø"1��w‘Ø"˜)‘à�à—k‘k×'Ñ'ØÜ—	‘	˜)¨4Ô0Ü—	‘	˜.°Ô5Ü"Ÿy™y¨¸tÔDÜ ŸY™Y |¸4Ô@ÜŸ)™) I°TÔ:Ø&;Ø'=Ø"'˜iØ"3Ø%9Ø'ØØð (ó ˆGð$ Ð*©{Ø+2Ñ(�˜Ü-5°oÀgÑ6NÓ-O�Ð)Ñ*Ø�Ø Ð,±[Ø+2Ñ(�˜Ø! " 1˜+¬°/À'Ñ2JÓ)KÐ(MÑMÐPWÐXYÐXZÐP[Ñ[�ð" ˆð —k‘k×'Ñ'ØÜ—	‘	˜)¨4Ô0Ü—	‘	˜.°Ô5Ü"Ÿy™y¨¸tÔDÜ ŸY™Y |¸4Ô@ÜŸ)™) I°TÔ:Ø"'˜iØ"3Ø%9Ø'Øð (ó ˆGð ˆr1   rØ   )NNNNNNNNFNNNN) r)   r*   r+   r,   r#   Úconfig_classÚbase_model_prefixr!  r>   ÚModuler/   r-   ra   r   Úintr6   rb   r(  r.  rB   r6  rB  r   r@  r™   r    ÚELECTRA_INPUTS_DOCSTRINGÚformatÚdictr   r^   Ú__classcell__)r+  s   @r2   r  r  ¤  so  ø… ñð
 !€LØ!ÐØ"€L�"—)‘)Ó"ð
 $ØØŸ;™;ØØ',ñmàðmð ðmð ð	mð
 �y‰yðmð ðmð !%õmò
ñ(! §
¡
× 2Ñ 2ð (!Àð (!ÐPZð (!Ðfpó (!òV1ñ& +Ð+C×+JÑ+JÐKhÓ+iÓjð ØØØØ"Ø#ØØ*.ØØ,0Ø/3Ø&*Ø $ñ^ð ð^ð —Z‘Z×'Ñ'ð^ð ð^ð $ D™>ð^ð ' t™nð^ð ˜d‘^ð^ð ò^ó kô^r1   r  c                   óê   — e Zd ZU eed<   ej                  Zej                  ed<   dZe	ed<   d„ Z
	 	 	 	 	 	 	 	 ddeej                     deej                     d	eej                     d
e	de	de	de	de	fd„Zy)ÚFlaxElectraModuler5   r6   Frî   c                 ó€  — t        | j                  | j                  ¬«      | _        | j                  j                  | j                  j
                  k7  r:t        j                  | j                  j
                  | j                  ¬«      | _        t        | j                  | j                  | j                  ¬«      | _        y )Nrh   r  )r4   r5   r6   Ú
embeddingsrA   ri   r>   rm   Úembeddings_projectr  rî   ÚencoderrP   s    r2   rR   zFlaxElectraModule.setupi  sv   € Ü/°·±À4Ç:Á:ÔNˆŒØ�;‰;×%Ñ%¨¯©×)@Ñ)@Ò@Ü&(§h¡h¨t¯{©{×/FÑ/FÈdÏjÉjÔ&YˆDÔ#Ü)Ø�K‰K˜tŸz™zÀ$×B]ÑB]ô
ˆ�r1   Nr  rç   rè   r™   rS   rš   rü   rý   c                 óž   — | j                  |||||	¬«      }t        | d«      r| j                  |«      }| j                  ||||	||||
||¬«
      S )NrV   rW  )r  rS   rç   rè   r™   rš   rü   rý   )rV  ÚhasattrrW  rX  )rQ   rX   r[   rY   rZ   r  rç   rè   r™   rS   rš   rü   rý   rV  s                 r2   r^   zFlaxElectraModule.__call__q  st   € ð —_‘_Ø�~ |°^ÐS`ð %ó 
ˆ
ô �4Ð-Ô.Ø×0Ñ0°Ó<ˆJà�|‰|ØØØØ'Ø"7Ø#9Ø!Ø/Ø!5Ø#ð ó 
ð 	
r1   )NNNFTFFT)r)   r*   r+   r#   r/   r-   ra   r6   rî   rb   rR   r   Únpr.   r^   r0   r1   r2   rT  rT  d  s´   … ØÓØ—{‘{€Eˆ3�9‰9Ó"Ø#(Ð˜DÓ(ò
ð +/Ø7;Ø8<Ø Ø"Ø"'Ø%*Ø ñ 
ð ˜BŸJ™JÑ'ð 
ð  (¨¯©Ñ4ð 
ð !)¨¯©Ñ 5ð 
ð ð 
ð ð 
ð  ð 
ð #ð 
ð ô 
r1   rT  zaThe bare Electra Model transformer outputting raw hidden-states without any specific head on top.c                   ó   — e Zd ZeZy)ÚFlaxElectraModelN)r)   r*   r+   rT  r!  r0   r1   r2   r]  r]  ”  s	   „ ð
 %�Lr1   r]  c                   óÆ   — e Zd ZU eed<   ej                  Zej                  ed<   dZe	j                  j                  j                  Zedej                   f   ed<   d„ Zd„ Zy)ÚFlaxElectraTiedDenserA   r6   N.Ú	bias_initc                 ó^   — | j                  d| j                  | j                  f«      | _        y )Nr    )Úparamr`  rA   r    rP   s    r2   rR   zFlaxElectraTiedDense.setup¥  s#   € Ø—J‘J˜v t§~¡~¸×8KÑ8KÐ7MÓNˆ�	r1   c                 óJ  — t        j                  || j                  «      }t        j                  || j                  «      }t        j                  |||j
                  dz
  fdfdf| j                  ¬«      }t        j                  | j                  | j                  «      }||z   S )Nr"   r…   )r0   r0   )r¤   )r-   Úasarrayr6   r   Údot_generalÚndimr¤   r    )rQ   ÚxÚkernelÚyr    s        r2   r^   zFlaxElectraTiedDense.__call__¨  s   € Ü�K‰K˜˜4Ÿ:™:Ó&ˆÜ—‘˜V T§Z¡ZÓ0ˆÜ�O‰OØØØ�v‰v˜‰zˆm˜TÐ" HÐ-Ø—n‘nô	
ˆô �{‰{˜4Ÿ9™9 d§j¡jÓ1ˆØ�4‰xˆr1   )r)   r*   r+   rN  r/   r-   ra   r6   r¤   rB   r>   rC   rˆ   r`  r   r[  r.   rR   r^   r0   r1   r2   r_  r_  Ÿ  sQ   … ØÓØ—{‘{€Eˆ3�9‰9Ó"Ø€IØ+.¯6©6×+>Ñ+>×+DÑ+D€Iˆx˜˜RŸZ™Z˜Ñ(ÓDòOó
r1   r_  c            	       óŒ   — e Zd ZU eed<   ej                  Zej                  ed<   dZe	ed<   d„ Z
	 	 	 	 	 	 	 	 dde	de	d	e	d
e	fd„Zy)ÚFlaxElectraForMaskedLMModuler5   r6   Frî   c                 ó´  — t        | j                  | j                  | j                  ¬«      | _        t        | j                  | j                  ¬«      | _        | j                  j                  r1t        | j                  j                  | j                  ¬«      | _
        y t        j                  | j                  j                  | j                  ¬«      | _
        y ©Nr&  ©r5   r6   rh   ©rT  r5   r6   rî   r   r  Úgenerator_predictionsÚtie_word_embeddingsr_  r@   Úgenerator_lm_headr>   rm   rP   s    r2   rR   z"FlaxElectraForMaskedLMModule.setupº  óŽ   € Ü(Ø—;‘; d§j¡jÈ×IdÑIdô
ˆŒô &EÈDÏKÉKÐ_c×_iÑ_iÔ%jˆÔ"Ø�;‰;×*Ò*Ü%9¸$¿+¹+×:PÑ:PÐX\×XbÑXbÔ%cˆDÕ"ä%'§X¡X¨d¯k©k×.DÑ.DÈDÏJÉJÔ%WˆDÕ"r1   NrS   rš   rü   rý   c
                 óˆ  — | j                  |||||||||	¬«	      }
|
d   }| j                  |«      }| j                  j                  r?| j                   j                  d   d   d   d   }| j                  ||j                  «      }n| j                  |«      }|	s	|f|
dd  z   S t        ||
j                  |
j                  ¬«      S )	N©rS   rš   rü   rý   r   r0  rV  rF   Ú	embeddingr"   ©r&   r'   r(   )
r   rp  r5   rq  r¦   rr  ÚTr   r'   r(   )rQ   rX   r[   rY   rZ   r  rS   rš   rü   rý   r½   r'   Úprediction_scoresÚshared_embeddings                 r2   r^   z%FlaxElectraForMaskedLMModule.__call__Ä  sç   € ð —,‘,ØØØØØØ'Ø/Ø!5Ø#ð ó 

ˆð   ™
ˆØ ×6Ñ6°}ÓEÐà�;‰;×*Ò*Ø#Ÿ|™|×5Ñ5°hÑ?ÀÑMÐN_Ñ`ÐalÑmÐØ $× 6Ñ 6Ð7HÐJZ×J\ÑJ\Ó ]Ñà $× 6Ñ 6Ð7HÓ IÐáØ%Ð'¨'°!°"¨+Ñ5Ð5ä!Ø$Ø!×/Ñ/Ø×)Ñ)ô
ð 	
r1   ©NNNNTFFT©r)   r*   r+   r#   r/   r-   ra   r6   rî   rb   rR   r^   r0   r1   r2   rk  rk  µ  sr   … ØÓØ—{‘{€Eˆ3�9‰9Ó"Ø#(Ð˜DÓ(òXð ØØØØ"Ø"'Ø%*Ø ñ'
ð ð'
ð  ð'
ð #ð'
ð ô'
r1   rk  z5Electra Model with a `language modeling` head on top.c                   ó   — e Zd ZeZy)ÚFlaxElectraForMaskedLMN)r)   r*   r+   rk  r!  r0   r1   r2   r~  r~  î  s   „ à/�Lr1   r~  c            	       óŒ   — e Zd ZU eed<   ej                  Zej                  ed<   dZe	ed<   d„ Z
	 	 	 	 	 	 	 	 dde	de	d	e	d
e	fd„Zy)ÚFlaxElectraForPreTrainingModuler5   r6   Frî   c                 ó²   — t        | j                  | j                  | j                  ¬«      | _        t        | j                  | j                  ¬«      | _        y ©Nr&  rn  )rT  r5   r6   rî   r   r  Údiscriminator_predictionsrP   s    r2   rR   z%FlaxElectraForPreTrainingModule.setupû  sC   € Ü(Ø—;‘; d§j¡jÈ×IdÑIdô
ˆŒô *MÐTX×T_ÑT_Ðgk×gqÑgqÔ)rˆÕ&r1   NrS   rš   rü   rý   c
                 ó¼   — | j                  |||||||||	¬«	      }
|
d   }| j                  |«      }|	s	|f|
dd  z   S t        ||
j                  |
j                  ¬«      S )Nru  r   r"   rw  )r   rƒ  r%   r'   r(   ©rQ   rX   r[   rY   rZ   r  rS   rš   rü   rý   r½   r'   r&   s                r2   r^   z(FlaxElectraForPreTrainingModule.__call__  s…   € ð —,‘,ØØØØØØ'Ø/Ø!5Ø#ð ó 

ˆð   ™
ˆà×/Ñ/°Ó>ˆáØ�9˜w q r˜{Ñ*Ð*ä.ØØ!×/Ñ/Ø×)Ñ)ô
ð 	
r1   r{  r|  r0   r1   r2   r€  r€  ö  sr   … ØÓØ—{‘{€Eˆ3�9‰9Ó"Ø#(Ð˜DÓ(òsð ØØØØ"Ø"'Ø%*Ø ñ#
ð ð#
ð  ð#
ð #ð#
ð ô#
r1   r€  zÊ
    Electra model with a binary classification head on top as used during pretraining for identifying generated tokens.

    It is recommended to load the discriminator checkpoint into that model.
    c                   ó   — e Zd ZeZy)ÚFlaxElectraForPreTrainingN)r)   r*   r+   r€  r!  r0   r1   r2   r‡  r‡  '  s	   „ ð 3�Lr1   r‡  aÎ  
    Returns:

    Example:

    ```python
    >>> from transformers import AutoTokenizer, FlaxElectraForPreTraining

    >>> tokenizer = AutoTokenizer.from_pretrained("google/electra-small-discriminator")
    >>> model = FlaxElectraForPreTraining.from_pretrained("google/electra-small-discriminator")

    >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="np")
    >>> outputs = model(**inputs)

    >>> prediction_logits = outputs.logits
    ```
rD  )Úoutput_typerK  c            	       óŒ   — e Zd ZU eed<   ej                  Zej                  ed<   dZe	ed<   d„ Z
	 	 	 	 	 	 	 	 dde	de	d	e	d
e	fd„Zy)Ú'FlaxElectraForTokenClassificationModuler5   r6   Frî   c                 ó’  — t        | j                  | j                  | j                  ¬«      | _        | j                  j
                  �| j                  j
                  n| j                  j                  }t        j                  |«      | _	        t        j                  | j                  j                  | j                  ¬«      | _        y ©Nr&  rh   )rT  r5   r6   rî   r   Úclassifier_dropoutrN   r>   rM   rO   rm   Ú
num_labelsÚ
classifier©rQ   r�  s     r2   rR   z-FlaxElectraForTokenClassificationModule.setupS  s‰   € Ü(Ø—;‘; d§j¡jÈ×IdÑIdô
ˆŒð
 �{‰{×-Ñ-Ð9ð �K‰K×*Ò*à—‘×0Ñ0ð 	ô
 —z‘zÐ"4Ó5ˆŒÜŸ(™( 4§;¡;×#9Ñ#9ÀÇÁÔLˆ�r1   NrS   rš   rü   rý   c
                 óâ   — | j                  |||||||||	¬«	      }
|
d   }| j                  ||¬«      }| j                  |«      }|	s	|f|
dd  z   S t        ||
j                  |
j
                  ¬«      S ©Nru  r   rV   r"   rw  )r   rO   r�  r   r'   r(   r…  s                r2   r^   z0FlaxElectraForTokenClassificationModule.__call___  s•   € ð —,‘,ØØØØØØ'Ø/Ø!5Ø#ð ó 

ˆð   ™
ˆàŸ™ ]À-˜ÓPˆØ—‘ Ó/ˆáØ�9˜w q r˜{Ñ*Ð*ä(ØØ!×/Ñ/Ø×)Ñ)ô
ð 	
r1   r{  r|  r0   r1   r2   rŠ  rŠ  N  sr   … ØÓØ—{‘{€Eˆ3�9‰9Ó"Ø#(Ð˜DÓ(ò
Mð ØØØØ"Ø"'Ø%*Ø ñ$
ð ð$
ð  ð$
ð #ð$
ð ô$
r1   rŠ  z‰
    Electra model with a token classification head on top.

    Both the discriminator and generator may be loaded into this model.
    c                   ó   — e Zd ZeZy)Ú!FlaxElectraForTokenClassificationN)r)   r*   r+   rŠ  r!  r0   r1   r2   r”  r”  †  s	   „ ð ;�Lr1   r”  c                 ó   — | S rØ   r0   )rg  r)  s     r2   Úidentityr–  š  s   € Ø€Hr1   c                   óf   — e Zd ZU dZeed<   ej                  Zej                  ed<   d„ Z	dde
fd„Zy)	ÚFlaxElectraSequenceSummaryaÜ  
    Compute a single vector summary of a sequence hidden states.

    Args:
        config ([`PretrainedConfig`]):
            The config used by the model. Relevant arguments in the config class of the model are (refer to the actual
            config class of your model for the default values it uses):

            - **summary_use_proj** (`bool`) -- Add a projection after the vector extraction.
            - **summary_proj_to_labels** (`bool`) -- If `True`, the projection outputs to `config.num_labels` classes
              (otherwise to `config.hidden_size`).
            - **summary_activation** (`Optional[str]`) -- Set to `"tanh"` to add a tanh activation to the output,
              another string or `None` will add no activation.
            - **summary_first_dropout** (`float`) -- Optional dropout probability before the projection and activation.
            - **summary_last_dropout** (`float`)-- Optional dropout probability after the projection and activation.
    r5   r6   c                 óš  — t         | _        t        | j                  d«      r®| j                  j                  r˜t        | j                  d«      rF| j                  j
                  r0| j                  j                  dkD  r| j                  j                  }n| j                  j                  }t        j                  || j                  ¬«      | _        t        | j                  dd «      }|r	t        |   nd„ | _        t         | _        t        | j                  d«      rG| j                  j                  dkD  r.t        j                   | j                  j                  «      | _        t         | _        t        | j                  d«      rI| j                  j$                  dkD  r/t        j                   | j                  j$                  «      | _        y y y )	NÚsummary_use_projÚsummary_proj_to_labelsr   rh   Úsummary_activationc                 ó   — | S rØ   r0   )rg  s    r2   r„   z2FlaxElectraSequenceSummary.setup.<locals>.<lambda>Á  s   € ÐXY€ r1   Úsummary_first_dropoutÚsummary_last_dropout)r–  ÚsummaryrZ  r5   rš  r›  rŽ  ri   r>   rm   r6   Úgetattrr   rÖ   Úfirst_dropoutrž  rM   Úlast_dropoutrŸ  )rQ   Únum_classesÚactivation_strings      r2   rR   z FlaxElectraSequenceSummary.setup³  s<  € ÜˆŒÜ�4—;‘;Ð 2Ô3¸¿¹×8TÒ8Tä˜Ÿ™Ð%=Ô>Ø—K‘K×6Ò6Ø—K‘K×*Ñ*¨QÒ.à"Ÿk™k×4Ñ4‘à"Ÿk™k×5Ñ5�ÜŸ8™8 K°t·z±zÔBˆDŒLä# D§K¡KÐ1EÀtÓLÐÙ7Hœ&Ð!2Ò3ÉkˆŒä%ˆÔÜ�4—;‘;Ð 7Ô8¸T¿[¹[×=^Ñ=^ÐabÒ=bÜ!#§¡¨D¯K©K×,MÑ,MÓ!NˆDÔä$ˆÔÜ�4—;‘;Ð 6Ô7¸D¿K¹K×<\Ñ<\Ð_`Ò<`Ü "§
¡
¨4¯;©;×+KÑ+KÓ LˆDÕð =aÐ7r1   NrS   c                 ó¨   — |dd…df   }| j                  ||¬«      }| j                  |«      }| j                  |«      }| j                  ||¬«      }|S )aZ  
        Compute a single vector summary of a sequence hidden states.

        Args:
            hidden_states (`jnp.ndarray` of shape `[batch_size, seq_len, hidden_size]`):
                The hidden states of the last layer.
            cls_index (`jnp.ndarray` of shape `[batch_size]` or `[batch_size, ...]` where ... are optional leading dimensions of `hidden_states`, *optional*):
                Used if `summary_type == "cls_index"` and takes the last token of the sequence as classification token.

        Returns:
            `jnp.ndarray`: The summary of the sequence hidden states.
        Nr   rV   )r¢  r   rÖ   r£  )rQ   r'   Ú	cls_indexrS   rÎ   s        r2   r^   z#FlaxElectraSequenceSummary.__call__Ë  s]   € ð šq !˜tÑ$ˆØ×#Ñ# F¸-Ð#ÓHˆØ—‘˜fÓ%ˆØ—‘ Ó(ˆØ×"Ñ" 6¸Ð"ÓGˆØˆr1   )NTr`   r0   r1   r2   r˜  r˜  ž  s3   … ñð" ÓØ—{‘{€Eˆ3�9‰9Ó"òMñ0ÀTô r1   r˜  c            	       óŒ   — e Zd ZU eed<   ej                  Zej                  ed<   dZe	ed<   d„ Z
	 	 	 	 	 	 	 	 dde	de	d	e	d
e	fd„Zy)Ú"FlaxElectraForMultipleChoiceModuler5   r6   Frî   c                 óþ   — t        | j                  | j                  | j                  ¬«      | _        t        | j                  | j                  ¬«      | _        t        j                  d| j                  ¬«      | _	        y )Nr&  rn  r"   rh   )
rT  r5   r6   rî   r   r˜  Úsequence_summaryr>   rm   r�  rP   s    r2   rR   z(FlaxElectraForMultipleChoiceModule.setupæ  sU   € Ü(Ø—;‘; d§j¡jÈ×IdÑIdô
ˆŒô !;À$Ç+Á+ÐUY×U_ÑU_Ô `ˆÔÜŸ(™( 1¨D¯J©JÔ7ˆ�r1   NrS   rš   rü   rý   c
                 ó<  — |j                   d   }
|�|j                  d|j                   d   «      nd }|�|j                  d|j                   d   «      nd }|�|j                  d|j                   d   «      nd }|�|j                  d|j                   d   «      nd }| j                  |||||||||	¬«	      }|d   }| j                  ||¬«      }| j	                  |«      }|j                  d|
«      }|	s	|f|dd  z   S t        ||j                  |j                  ¬«      S )Nr"   r¥   ru  r   rV   rw  )rw   rv   r   r«  r�  r   r'   r(   )rQ   rX   r[   rY   rZ   r  rS   rš   rü   rý   Únum_choicesr½   r'   Úpooled_outputr&   Úreshaped_logitss                   r2   r^   z+FlaxElectraForMultipleChoiceModule.__call__í  sK  € ð  —o‘o aÑ(ˆØBKÐBW�I×%Ñ% b¨)¯/©/¸"Ñ*=Ô>Ð]aˆ	ØQ_ÐQk˜×/Ñ/°°N×4HÑ4HÈÑ4LÔMÐquˆØQ_ÐQk˜×/Ñ/°°N×4HÑ4HÈÑ4LÔMÐquˆØKWÐKc�|×+Ñ+¨B°×0BÑ0BÀ2Ñ0FÔGÐimˆð —,‘,ØØØØØØ'Ø/Ø!5Ø#ð ó 

ˆð   ™
ˆØ×-Ñ-¨mÈ=Ð-ÓYˆØ—‘ Ó/ˆà Ÿ.™.¨¨[Ó9ˆáØ#Ð%¨°°¨Ñ3Ð3ä,Ø"Ø!×/Ñ/Ø×)Ñ)ô
ð 	
r1   r{  r|  r0   r1   r2   r©  r©  á  sq   … ØÓØ—{‘{€Eˆ3�9‰9Ó"Ø#(Ð˜DÓ(ò8ð ØØØØ"Ø"'Ø%*Ø ñ+
ð ð+
ð  ð+
ð #ð+
ð ô+
r1   r©  z¨
    ELECTRA Model with a multiple choice classification head on top (a linear layer on top of the pooled output and a
    softmax) e.g. for RocStories/SWAG tasks.
    c                   ó   — e Zd ZeZy)ÚFlaxElectraForMultipleChoiceN)r)   r*   r+   r©  r!  r0   r1   r2   r±  r±    s	   „ ð 6�Lr1   r±  z(batch_size, num_choices, sequence_lengthc            	       óŒ   — e Zd ZU eed<   ej                  Zej                  ed<   dZe	ed<   d„ Z
	 	 	 	 	 	 	 	 dde	de	d	e	d
e	fd„Zy)Ú%FlaxElectraForQuestionAnsweringModuler5   r6   Frî   c                 óÚ   — t        | j                  | j                  | j                  ¬«      | _        t        j                  | j                  j                  | j                  ¬«      | _        y rŒ  )	rT  r5   r6   rî   r   r>   rm   rŽ  Ú
qa_outputsrP   s    r2   rR   z+FlaxElectraForQuestionAnsweringModule.setup7  sE   € Ü(Ø—;‘; d§j¡jÈ×IdÑIdô
ˆŒô Ÿ(™( 4§;¡;×#9Ñ#9ÀÇÁÔLˆ�r1   NrS   rš   rü   rý   c
                 ób  — | j                  |||||||||	¬«	      }
|
d   }| j                  |«      }t        j                  || j                  j
                  d¬«      \  }}|j                  d«      }|j                  d«      }|	s
||f|
dd  z   S t        |||
j                  |
j                  ¬«      S )Nru  r   r¥   rž   r"   )Ústart_logitsÚ
end_logitsr'   r(   )
r   rµ  r-   r7  r5   rŽ  r  r   r'   r(   )rQ   rX   r[   rY   rZ   r  rS   rš   rü   rý   r½   r'   r&   r·  r¸  s                  r2   r^   z.FlaxElectraForQuestionAnsweringModule.__call__=  sË   € ð —,‘,ØØØØØØ'Ø/Ø!5Ø#ð ó 

ˆð   ™
ˆØ—‘ Ó/ˆÜ#&§9¡9¨V°T·[±[×5KÑ5KÐRTÔ#UÑ ˆ�jØ#×+Ñ+¨BÓ/ˆØ×'Ñ'¨Ó+ˆ
áØ  *Ð-°¸¸°Ñ;Ð;ä/Ø%Ø!Ø!×/Ñ/Ø×)Ñ)ô	
ð 	
r1   r{  r|  r0   r1   r2   r³  r³  2  sr   … ØÓØ—{‘{€Eˆ3�9‰9Ó"Ø#(Ð˜DÓ(òMð ØØØØ"Ø"'Ø%*Ø ñ&
ð ð&
ð  ð&
ð #ð&
ð ô&
r1   r³  zà
    ELECTRA Model with a span classification head on top for extractive question-answering tasks like SQuAD (a linear
    layers on top of the hidden-states output to compute `span start logits` and `span end logits`).
    c                   ó   — e Zd ZeZy)ÚFlaxElectraForQuestionAnsweringN)r)   r*   r+   r³  r!  r0   r1   r2   rº  rº  f  s	   „ ð 9�Lr1   rº  c                   óf   — e Zd ZU dZeed<   ej                  Zej                  ed<   d„ Z	dde
fd„Zy)	ÚFlaxElectraClassificationHeadz-Head for sentence-level classification tasks.r5   r6   c                 ó¤  — t        j                  | j                  j                  | j                  ¬«      | _        | j                  j                  �| j                  j                  n| j                  j                  }t        j                  |«      | _	        t        j                  | j                  j                  | j                  ¬«      | _        y )Nrh   )r>   rm   r5   ri   r6   rÄ   r�  rN   rM   rO   rŽ  Úout_projr�  s     r2   rR   z#FlaxElectraClassificationHead.setup  sˆ   € Ü—X‘X˜dŸk™k×5Ñ5¸T¿Z¹ZÔHˆŒ
ð �{‰{×-Ñ-Ð9ð �K‰K×*Ò*à—‘×0Ñ0ð 	ô
 —z‘zÐ"4Ó5ˆŒÜŸ™ §¡×!7Ñ!7¸t¿z¹zÔJˆ�r1   rS   c                 óÊ   — |d d …dd d …f   }| j                  ||¬«      }| j                  |«      }t        d   |«      }| j                  ||¬«      }| j                  |«      }|S )Nr   rV   Úgelu)rO   rÄ   r   r¾  )rQ   r'   rS   rg  s       r2   r^   z&FlaxElectraClassificationHead.__call__‰  sd   € Øš!˜Q¢˜'Ñ"ˆØ�L‰L˜¨-ˆLÓ8ˆØ�J‰J�q‹MˆÜ�6‰N˜1ÓˆØ�L‰L˜¨-ˆLÓ8ˆØ�M‰M˜!ÓˆØˆr1   Nr_   r`   r0   r1   r2   r¼  r¼  y  s/   … Ù7àÓØ—{‘{€Eˆ3�9‰9Ó"òKñ°Tô r1   r¼  c            	       óŒ   — e Zd ZU eed<   ej                  Zej                  ed<   dZe	ed<   d„ Z
	 	 	 	 	 	 	 	 dde	de	d	e	d
e	fd„Zy)Ú*FlaxElectraForSequenceClassificationModuler5   r6   Frî   c                 ó²   — t        | j                  | j                  | j                  ¬«      | _        t        | j                  | j                  ¬«      | _        y r‚  )rT  r5   r6   rî   r   r¼  r�  rP   s    r2   rR   z0FlaxElectraForSequenceClassificationModule.setup˜  s>   € Ü(Ø—;‘; d§j¡jÈ×IdÑIdô
ˆŒô 8¸t¿{¹{ÐRV×R\ÑR\Ô]ˆ�r1   NrS   rš   rü   rý   c
                 óÀ   — | j                  |||||||||	¬«	      }
|
d   }| j                  ||¬«      }|	s	|f|
dd  z   S t        ||
j                  |
j                  ¬«      S r’  )r   r�  r   r'   r(   r…  s                r2   r^   z3FlaxElectraForSequenceClassificationModule.__call__ž  s‡   € ð —,‘,ØØØØØØ'Ø/Ø!5Ø#ð ó 

ˆð   ™
ˆØ—‘ ¸m�ÓLˆáØ�9˜w q r˜{Ñ*Ð*ä+ØØ!×/Ñ/Ø×)Ñ)ô
ð 	
r1   r{  r|  r0   r1   r2   rÂ  rÂ  “  sr   … ØÓØ—{‘{€Eˆ3�9‰9Ó"Ø#(Ð˜DÓ(ò^ð ØØØØ"Ø"'Ø%*Ø ñ"
ð ð"
ð  ð"
ð #ð"
ð ô"
r1   rÂ  zŸ
    Electra Model transformer with a sequence classification/regression head on top (a linear layer on top of the
    pooled output) e.g. for GLUE tasks.
    c                   ó   — e Zd ZeZy)Ú$FlaxElectraForSequenceClassificationN)r)   r*   r+   rÂ  r!  r0   r1   r2   rÆ  rÆ  Ã  s	   „ ð >�Lr1   rÆ  c                   óJ  — e Zd ZU eed<   ej                  Zej                  ed<   dZe	ed<   d„ Z
	 	 	 	 	 	 	 	 	 	 	 ddeej                     deej                     d	eej                     d
eej                     deej                     deej                     de	de	de	de	de	fd„Zy)ÚFlaxElectraForCausalLMModuler5   r6   Frî   c                 ó´  — t        | j                  | j                  | j                  ¬«      | _        t        | j                  | j                  ¬«      | _        | j                  j                  r1t        | j                  j                  | j                  ¬«      | _
        y t        j                  | j                  j                  | j                  ¬«      | _
        y rm  ro  rP   s    r2   rR   z"FlaxElectraForCausalLMModule.setupÛ  rs  r1   Nr[   rY   rZ   r  rç   rè   r™   rS   rš   rü   rý   c                 ó¤  — | j                  |||||||||	|
||¬«      }|d   }| j                  |«      }| j                  j                  r?| j                   j                  d   d   d   d   }| j                  ||j                  «      }n| j                  |«      }|s	|f|dd  z   S t        ||j                  |j                  |j                  ¬«      S )	N)rç   rè   r™   rS   rš   rü   rý   r   r0  rV  rF   rv  r"   )r&   r'   r(   r  )r   rp  r5   rq  r¦   rr  rx  r   r'   r(   r  )rQ   rX   r[   rY   rZ   r  rç   rè   r™   rS   rš   rü   rý   r½   r'   ry  rz  s                    r2   r^   z%FlaxElectraForCausalLMModule.__call__å  sù   € ð —,‘,ØØØØØØ"7Ø#9Ø!Ø'Ø/Ø!5Ø#ð ó 
ˆð   ™
ˆØ ×6Ñ6°}ÓEÐà�;‰;×*Ò*Ø#Ÿ|™|×5Ñ5°hÑ?ÀÑMÐN_Ñ`ÐalÑmÐØ $× 6Ñ 6Ð7HÐJZ×J\ÑJ\Ó ]Ñà $× 6Ñ 6Ð7HÓ IÐáØ%Ð'¨'°!°"¨+Ñ5Ð5ä4Ø$Ø!×/Ñ/Ø×)Ñ)Ø$×5Ñ5ô	
ð 	
r1   )NNNNNNFTFFTr  r0   r1   r2   rÈ  rÈ  Ö  s÷   … ØÓØ—{‘{€Eˆ3�9‰9Ó"Ø#(Ð˜DÓ(òXð 15Ø04Ø.2Ø+/Ø7;Ø8<Ø Ø"Ø"'Ø%*Ø ñ.
ð ! §¡Ñ-ð.
ð ! §¡Ñ-ð	.
ð
 ˜sŸ{™{Ñ+ð.
ð ˜CŸK™KÑ(ð.
ð  (¨¯©Ñ4ð.
ð !)¨¯©Ñ 5ð.
ð ð.
ð ð.
ð  ð.
ð #ð.
ð ô.
r1   rÈ  z’
    Electra Model with a language modeling head on top (a linear layer on top of the hidden-states output) e.g for
    autoregressive tasks.
    c                   ó>   — e Zd ZeZddeej                     fd„Zd„ Z	y)ÚFlaxElectraForCausalLMNr[   c                 óH  — |j                   \  }}| j                  ||«      }t        j                  ||fd¬«      }|�-|j	                  d¬«      dz
  }t        j                  ||d«      }n4t        j                  t        j                  |d¬«      d d d …f   ||f«      }|||dœS )NrU   rh   r¥   rž   r"   )r   r   )rF  r[   rZ   )	rw   r™   r-   rq   Úcumsumr   rŠ   r‹   rŒ   )	rQ   rX   r�   r[   r²   Ú
seq_lengthrF  Úextended_attention_maskrZ   s	            r2   Úprepare_inputs_for_generationz4FlaxElectraForCausalLM.prepare_inputs_for_generation!  s±   € à!*§¡Ñˆ
�JàŸ/™/¨*°jÓAˆô #&§(¡(¨J¸
Ð+CÈ4Ô"PÐØÐ%Ø)×0Ñ0°bÐ0Ó9¸AÑ=ˆLÜ&)×&>Ñ&>Ð?VÐXfÐhnÓ&oÑ#ä×+Ñ+¬C¯J©J°zÈÔ,NÈtÒUVÈwÑ,WÐZdÐfpÐYqÓrˆLð  /Ø5Ø(ñ
ð 	
r1   c                 óL   — |j                   |d<   |d   d d …dd …f   dz   |d<   |S )NrF  rZ   r¥   r"   )rF  )rQ   Úmodel_outputsÚmodel_kwargss      r2   Úupdate_inputs_for_generationz3FlaxElectraForCausalLM.update_inputs_for_generation6  s8   € Ø*7×*GÑ*GˆÐ&Ñ'Ø'3°NÑ'CÂAÀrÁsÀFÑ'KÈaÑ'Oˆ�^Ñ$ØÐr1   rØ   )
r)   r*   r+   rÈ  r!  r   rB   ÚArrayrÑ  rÕ  r0   r1   r2   rÌ  rÌ    s'   „ ð 0€Lñ
ÐS[Ð\_×\eÑ\eÑSfó 
ó*r1   rÌ  )	rÌ  r~  r±  r‡  rº  rÆ  r”  r]  r  )`Útypingr   r   r   ÚflaxÚ
flax.linenÚlinenr>   rB   Ú	jax.numpyÚnumpyr-   r[  Úflax.core.frozen_dictr   r   r   r	   r
   r   Únn_partitioningÚflax.linen.attentionr   Úflax.traverse_utilr   r   r   Úmodeling_flax_outputsr   r   r   r   r   r   r   r   Úmodeling_flax_utilsr   r   r   r   r   Úutilsr   r   r    r!   Úconfiguration_electrar#   Ú
get_loggerr)   ÚloggerÚ_CHECKPOINT_FOR_DOCÚ_CONFIG_FOR_DOCrõ   ÚstructÚ	dataclassr%   ÚELECTRA_START_DOCSTRINGrO  rM  r4   rd   rÁ   rË   rÒ   rÛ   rà   rí   r  r  r  r  rT  r]  r_  rk  r~  r€  r‡  Ú&FLAX_ELECTRA_FOR_PRETRAINING_DOCSTRINGrP  rŠ  r”  r–  r˜  r©  r±  r³  rº  r¼  rÂ  rÆ  rÈ  rÌ  Ú__all__r0   r1   r2   ú<module>rî     s…  ð÷  -Ñ ,ã Ý Û 
Ý Û ß >Ñ >ß 6Ý 6Ý >ß ;Ý ÷	÷ 	ó 	÷õ ÷ gÓ fÝ 0ð 
ˆ×	Ñ	˜HÓ	%€à:Ð Ø!€à×Ñ€ð ‡�×Ñô4 kó 4ó ð4ð2Ð ð,$Ð ôN&˜BŸI™Iô &ôTh˜rŸy™yô hôX˜BŸI™Iô ô*'˜2Ÿ9™9ô 'ôV˜bŸi™iô ô&˜Ÿ	™	ô ô*6�r—y‘yô 6ôtO
 §¡ô O
ôf$
˜Ÿ™ô $
ôN b§i¡iô ô¨"¯)©)ô ô"}Ð!4ô }ô@-
˜Ÿ	™	ô -
ñ` ØgØóô%Ð1ó %ó	ð%ñ Ð-Ð/BÐDWÐYhÔ iô˜2Ÿ9™9ô ô,6
 2§9¡9ô 6
ñr ÐQÐSjÓkô0Ð7ó 0ó lð0ñ Ð3Ð5HÐJ\Ð^mÔ nô.
 b§i¡iô .
ñb ðð
 óô3Ð :ó 3óð3ð*Ð &ñ$ ØØ×#Ñ#Ð$AÓBÐEkÑkôñ !ØÐ+JÐYhõô
5
¨b¯i©iô 5
ñp ðð
 óô;Ð(Bó ;óð;ñ Ø%ØØØô	òô@ §¡ô @ôF7
¨¯©ô 7
ñt ðð óô6Ð#=ó 6óð6ñ
 Ø Ð":×"AÑ"AÐBlÓ"môñ Ø ØØ!Øô	ô1
¨B¯I©Iô 1
ñh ðð óô9Ð&@ó 9óð9ñ Ø#ØØ$Øô	ô B§I¡Iô ô4-
°·±ô -
ñ` ðð óô>Ð+Eó >óð>ñ Ø(ØØ Øô	ô=
 2§9¡9ô =
ñ@ ðð óôÐ7ó óðñ< ØØØ)Øô	ò
�r1   