Ë
    S^(hú  ã                   óº  — d dl mZmZmZ d dlZd dlmZ d dlZd dl	m
Z d dl
Zd dlmZmZmZ d dlmZmZ d dlmZ d dlmZ d dlmZmZ d dlmZ d	d
lmZmZmZmZm Z m!Z!m"Z"m#Z#m$Z$m%Z% d	dl&m'Z'm(Z(m)Z)m*Z*m+Z+ d	dl,m-Z-m.Z.m/Z/m0Z0 ddl1m2Z2  e0jf                  e4«      Z5dZ6dZ7ejp                  Z8ejr                  jt                   G d„ de-«      «       Z;dZ<dZ= G d„ dej|                  «      Z? G d„ dej|                  «      Z@ G d„ dej|                  «      ZA G d„ dej|                  «      ZB G d„ dej|                  «      ZC G d„ d ej|                  «      ZD G d!„ d"ej|                  «      ZE G d#„ d$ej|                  «      ZF G d%„ d&ej|                  «      ZG G d'„ d(ej|                  «      ZH G d)„ d*ej|                  «      ZI G d+„ d,ej|                  «      ZJ G d-„ d.ej|                  «      ZK G d/„ d0ej|                  «      ZL G d1„ d2ej|                  «      ZM G d3„ d4e(«      ZN G d5„ d6ej|                  «      ZO e.d7e<«       G d8„ d9eN«      «       ZP e)ePe6ee7«        G d:„ d;ej|                  «      ZQ e.d<e<«       G d=„ d>eN«      «       ZRd?ZS e+eRe=j©                  d@«      eSz   «        e*eRe;e7¬A«        G dB„ dCej|                  «      ZU e.dDe<«       G dE„ dFeN«      «       ZV e)eVe6e e7«        G dG„ dHej|                  «      ZW e.dIe<«       G dJ„ dKeN«      «       ZXdLZY e+eXe=j©                  d@«      eYz   «        e*eXe"e7¬A«        G dM„ dNej|                  «      ZZ e.dOe<«       G dP„ dQeN«      «       Z[ e)e[e6e$e7«        G dR„ dSej|                  «      Z\ e.dTe<«       G dU„ dVeN«      «       Z] e+e]e=j©                  dW«      «        e)e]e6e!e7«        G dX„ dYej|                  «      Z^ e.dZe<«       G d[„ d\eN«      «       Z_ e)e_e6e%e7«        G d]„ d^ej|                  «      Z` e.d_e<«       G d`„ daeN«      «       Za e)eae6e#e7«        G db„ dcej|                  «      Zb e.dde<«       G de„ dfeN«      «       Zc e)ece6ee7«       g dg¢Zdy)hé    )ÚCallableÚOptionalÚTupleN)Ú
FrozenDictÚfreezeÚunfreeze)Úcombine_masksÚmake_causal_mask)Úpartitioning)Údot_product_attention_weights)Úflatten_dictÚunflatten_dict)Úlaxé   )
Ú-FlaxBaseModelOutputWithPastAndCrossAttentionsÚFlaxBaseModelOutputWithPoolingÚ0FlaxBaseModelOutputWithPoolingAndCrossAttentionsÚ%FlaxCausalLMOutputWithCrossAttentionsÚFlaxMaskedLMOutputÚFlaxMultipleChoiceModelOutputÚFlaxNextSentencePredictorOutputÚ FlaxQuestionAnsweringModelOutputÚFlaxSequenceClassifierOutputÚFlaxTokenClassifierOutput)ÚACT2FNÚFlaxPreTrainedModelÚappend_call_sample_docstringÚ append_replace_return_docstringsÚoverwrite_call_docstring)ÚModelOutputÚadd_start_docstringsÚ%add_start_docstrings_to_model_forwardÚloggingé   )Ú
BertConfigzgoogle-bert/bert-base-uncasedr%   c                   ó²   — e Zd ZU dZdZej                  ed<   dZej                  ed<   dZ	e
eej                        ed<   dZe
eej                        ed<   y)ÚFlaxBertForPreTrainingOutputaI  
    Output type of [`BertForPreTraining`].

    Args:
        prediction_logits (`jnp.ndarray` of shape `(batch_size, sequence_length, config.vocab_size)`):
            Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
        seq_relationship_logits (`jnp.ndarray` of shape `(batch_size, 2)`):
            Prediction scores of the next sequence prediction (classification) head (scores of True/False continuation
            before SoftMax).
        hidden_states (`tuple(jnp.ndarray)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
            Tuple of `jnp.ndarray` (one for the output of the embeddings + one for the output of each layer) of shape
            `(batch_size, sequence_length, hidden_size)`.

            Hidden-states of the model at the output of each layer plus the initial embedding outputs.
        attentions (`tuple(jnp.ndarray)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
            Tuple of `jnp.ndarray` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
            sequence_length)`.

            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
            heads.
    NÚprediction_logitsÚseq_relationship_logitsÚhidden_statesÚ
attentions)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r(   ÚjnpÚndarrayÚ__annotations__r)   r*   r   r   r+   © ó    úi/var/www/skyplay_api_hub/venv/lib/python3.12/site-packages/transformers/models/bert/modeling_flax_bert.pyr'   r'   =   sW   … ñð, &*Ð�s—{‘{Ó)Ø+/Ð˜SŸ[™[Ó/Ø26€M�8˜E #§+¡+Ñ.Ñ/Ó6Ø/3€J�˜˜sŸ{™{Ñ+Ñ,Ô3r4   r'   aÜ
  

    This model inherits from [`FlaxPreTrainedModel`]. Check the superclass documentation for the generic methods the
    library implements for all its model (such as downloading, saving and converting weights from PyTorch models)

    This model is also a
    [flax.linen.Module](https://flax.readthedocs.io/en/latest/api_reference/flax.linen/module.html) subclass. Use it as
    a regular Flax linen Module and refer to the Flax documentation for all matter related to general usage and
    behavior.

    Finally, this model supports inherent JAX features such as:

    - [Just-In-Time (JIT) compilation](https://jax.readthedocs.io/en/latest/jax.html#just-in-time-compilation-jit)
    - [Automatic Differentiation](https://jax.readthedocs.io/en/latest/jax.html#automatic-differentiation)
    - [Vectorization](https://jax.readthedocs.io/en/latest/jax.html#vectorization-vmap)
    - [Parallelization](https://jax.readthedocs.io/en/latest/jax.html#parallelization-pmap)

    Parameters:
        config ([`BertConfig`]): Model configuration class with all the parameters of the model.
            Initializing with a config file does not load the weights associated with the model, only the
            configuration. Check out the [`~FlaxPreTrainedModel.from_pretrained`] method to load the model weights.
        dtype (`jax.numpy.dtype`, *optional*, defaults to `jax.numpy.float32`):
            The data type of the computation. Can be one of `jax.numpy.float32`, `jax.numpy.float16` (on GPUs) and
            `jax.numpy.bfloat16` (on TPUs).

            This can be used to enable mixed-precision training or half-precision inference on GPUs or TPUs. If
            specified all the computation will be performed with the given `dtype`.

            **Note that this only specifies the dtype of the computation and does not influence the dtype of model
            parameters.**

            If you wish to change the dtype of the model parameters, see [`~FlaxPreTrainedModel.to_fp16`] and
            [`~FlaxPreTrainedModel.to_bf16`].
        dtype (`jax.numpy.dtype`, *optional*, defaults to `jax.numpy.float32`):
            The data type of the computation. Can be one of `jax.numpy.float32`, `jax.numpy.float16` (on GPUs) and
            `jax.numpy.bfloat16` (on TPUs).

            This can be used to enable mixed-precision training or half-precision inference on GPUs or TPUs. If
            specified all the computation will be performed with the given `dtype`.

            **Note that this only specifies the dtype of the computation and does not influence the dtype of model
            parameters.**

            If you wish to change the dtype of the model parameters, see [`~FlaxPreTrainedModel.to_fp16`] and
            [`~FlaxPreTrainedModel.to_bf16`].

a³  
    Args:
        input_ids (`numpy.ndarray` of shape `({0})`):
            Indices of input sequence tokens in the vocabulary.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are input IDs?](../glossary#input-ids)
        attention_mask (`numpy.ndarray` of shape `({0})`, *optional*):
            Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:

            - 1 for tokens that are **not masked**,
            - 0 for tokens that are **masked**.

            [What are attention masks?](../glossary#attention-mask)
        token_type_ids (`numpy.ndarray` of shape `({0})`, *optional*):
            Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
            1]`:

            - 0 corresponds to a *sentence A* token,
            - 1 corresponds to a *sentence B* token.

            [What are token type IDs?](../glossary#token-type-ids)
        position_ids (`numpy.ndarray` of shape `({0})`, *optional*):
            Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
            config.max_position_embeddings - 1]`.
        head_mask (`numpy.ndarray` of shape `({0})`, `optional):
            Mask to nullify selected heads of the attention modules. Mask values selected in `[0, 1]`:

            - 1 indicates the head is **not masked**,
            - 0 indicates the head is **masked**.

        return_dict (`bool`, *optional*):
            Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.

c                   óf   — e Zd ZU dZeed<   ej                  Zej                  ed<   d„ Z	dde
fd„Zy)	ÚFlaxBertEmbeddingszGConstruct the embeddings from word, position and token_type embeddings.ÚconfigÚdtypec                 ó  — t        j                  | j                  j                  | j                  j                  t
        j                   j                  j                  | j                  j                  ¬«      | j                  ¬«      | _
        t        j                  | j                  j                  | j                  j                  t
        j                   j                  j                  | j                  j                  ¬«      | j                  ¬«      | _        t        j                  | j                  j                  | j                  j                  t
        j                   j                  j                  | j                  j                  ¬«      | j                  ¬«      | _        t        j                  | j                  j                   | j                  ¬«      | _        t        j"                  | j                  j$                  ¬«      | _        y )N)Ústddev)Úembedding_initr9   ©Úepsilonr9   ©Úrate)ÚnnÚEmbedr8   Ú
vocab_sizeÚhidden_sizeÚjaxÚinitializersÚnormalÚinitializer_ranger9   Úword_embeddingsÚmax_position_embeddingsÚposition_embeddingsÚtype_vocab_sizeÚtoken_type_embeddingsÚ	LayerNormÚlayer_norm_epsÚDropoutÚhidden_dropout_probÚdropout©Úselfs    r5   ÚsetupzFlaxBertEmbeddings.setup¸   sJ  € Ü!Ÿx™xØ�K‰K×"Ñ"Ø�K‰K×#Ñ#ÜŸ6™6×.Ñ.×5Ñ5¸T¿[¹[×=ZÑ=ZÐ5Ó[Ø—*‘*ô	 
ˆÔô $&§8¡8Ø�K‰K×/Ñ/Ø�K‰K×#Ñ#ÜŸ6™6×.Ñ.×5Ñ5¸T¿[¹[×=ZÑ=ZÐ5Ó[Ø—*‘*ô	$
ˆÔ ô &(§X¡XØ�K‰K×'Ñ'Ø�K‰K×#Ñ#ÜŸ6™6×.Ñ.×5Ñ5¸T¿[¹[×=ZÑ=ZÐ5Ó[Ø—*‘*ô	&
ˆÔ"ô Ÿ™¨d¯k©k×.HÑ.HÐPT×PZÑPZÔ[ˆŒÜ—z‘z t§{¡{×'FÑ'FÔGˆ�r4   Údeterministicc                 ó  — | j                  |j                  d«      «      }| j                  |j                  d«      «      }| j                  |j                  d«      «      }||z   |z   }	| j	                  |	«      }	| j                  |	|¬«      }	|	S )NÚi4©rV   )rI   ÚastyperK   rM   rN   rR   )
rT   Ú	input_idsÚtoken_type_idsÚposition_idsÚattention_maskrV   Úinputs_embedsÚposition_embedsrM   r*   s
             r5   Ú__call__zFlaxBertEmbeddings.__call__Î   s�   € à×,Ñ,¨Y×-=Ñ-=¸dÓ-CÓDˆØ×2Ñ2°<×3FÑ3FÀtÓ3LÓMˆØ $× :Ñ :¸>×;PÑ;PÐQUÓ;VÓ WÐð &Ð(=Ñ=ÀÑOˆð Ÿ™ }Ó5ˆØŸ™ ]À-˜ÓPˆØÐr4   N©T)r,   r-   r.   r/   r%   r2   r0   Úfloat32r9   rU   Úboolra   r3   r4   r5   r7   r7   ²   s0   … ÙQàÓØ—{‘{€Eˆ3�9‰9Ó"òHñ,Ð_cô r4   r7   c                   óÊ   — e Zd ZU eed<   dZeed<   ej                  Z	ej                  ed<   d„ Z
d„ Zd„ Zej                  d„ «       Z	 	 	 	 dd
eej"                     dedefd„Zy	)ÚFlaxBertSelfAttentionr8   FÚcausalr9   c                 ó6  — | j                   j                  | j                   j                  z  | _        | j                   j                  | j                   j                  z  dk7  rt	        d«      ‚t        j                  | j                   j                  | j                  t        j
                  j                  j                  | j                   j                  «      ¬«      | _        t        j                  | j                   j                  | j                  t        j
                  j                  j                  | j                   j                  «      ¬«      | _        t        j                  | j                   j                  | j                  t        j
                  j                  j                  | j                   j                  «      ¬«      | _        | j                  r>t!        t#        j$                  d| j                   j&                  fd¬«      d¬«      | _        y y )Nr   z›`config.hidden_size`: {self.config.hidden_size} has to be a multiple of `config.num_attention_heads`                    : {self.config.num_attention_heads})r9   Úkernel_initr$   rd   ©r9   )r8   rD   Únum_attention_headsÚhead_dimÚ
ValueErrorrA   ÚDenser9   rE   rF   rG   rH   ÚqueryÚkeyÚvaluerg   r
   r0   ÚonesrJ   Úcausal_maskrS   s    r5   rU   zFlaxBertSelfAttention.setupâ   si  € ØŸ™×/Ñ/°4·;±;×3RÑ3RÑRˆŒØ�;‰;×"Ñ" T§[¡[×%DÑ%DÑDÈÒIÜðIóð ô
 —X‘XØ�K‰K×#Ñ#Ø—*‘*ÜŸ™×+Ñ+×2Ñ2°4·;±;×3PÑ3PÓQô
ˆŒ
ô
 —8‘8Ø�K‰K×#Ñ#Ø—*‘*ÜŸ™×+Ñ+×2Ñ2°4·;±;×3PÑ3PÓQô
ˆŒô
 —X‘XØ�K‰K×#Ñ#Ø—*‘*ÜŸ™×+Ñ+×2Ñ2°4·;±;×3PÑ3PÓQô
ˆŒ
ð �;Š;Ü/Ü—‘˜!˜TŸ[™[×@Ñ@ÐAÈÔPÐX^ô ˆDÕð r4   c                 ó„   — |j                  |j                  d d | j                  j                  | j                  fz   «      S ©Né   )ÚreshapeÚshaper8   rk   rl   ©rT   r*   s     r5   Ú_split_headsz"FlaxBertSelfAttention._split_headsÿ   s;   € Ø×$Ñ$ ]×%8Ñ%8¸¸!Ð%<ÀÇÁ×@_Ñ@_Ðae×anÑanÐ?oÑ%oÓpÐpr4   c                 ón   — |j                  |j                  d d | j                  j                  fz   «      S ru   )rw   rx   r8   rD   ry   s     r5   Ú_merge_headsz"FlaxBertSelfAttention._merge_heads  s2   € Ø×$Ñ$ ]×%8Ñ%8¸¸!Ð%<ÀÇÁ×@WÑ@WÐ?YÑ%YÓZÐZr4   c                 ó(  — | j                  dd«      }| j                  ddt        j                  |j                  |j
                  «      }| j                  ddt        j                  |j                  |j
                  «      }| j                  ddd„ «      }|rø|j                  j                  �^ }	}
}}|j                  }dt        |	«      z  |ddfz   }t        j                  |j                  ||«      }t        j                  |j                  ||«      }||_        ||_        |j                  d   }|j                  |z   |_        t        j                  t        j                  |
«      ||z   k  t        |	«      d||
fz   «      }t        ||«      }|||fS )	a\  
        This function takes projected key, value states from a single input token and concatenates the states to cached
        states from previous steps. This function is slightly adapted from the official Flax repository:
        https://github.com/google/flax/blob/491ce18759622506588784b4fca0e4bf05f8c8cd/flax/linen/attention.py#L252
        ÚcacheÚ
cached_keyÚcached_valueÚcache_indexc                  óL   — t        j                  dt         j                  ¬«      S )Nr   rj   )r0   ÚarrayÚint32r3   r4   r5   ú<lambda>z=FlaxBertSelfAttention._concatenate_to_cache.<locals>.<lambda>  s   € ÄCÇIÁIÈaÔWZ×W`ÑW`ÔDa€ r4   )r   r   r$   )Úhas_variableÚvariabler0   Úzerosrx   r9   rq   Úlenr   Údynamic_update_sliceÚbroadcast_toÚarangeÚtupler	   )rT   rp   rq   ro   r^   Úis_initializedr   r€   r�   Ú
batch_dimsÚ
max_lengthÚ	num_headsÚdepth_per_headÚ	cur_indexÚindicesÚnum_updated_cache_vectorsÚpad_masks                    r5   Ú_concatenate_to_cachez+FlaxBertSelfAttention._concatenate_to_cache  sr  € ð ×*Ñ*¨7°LÓAˆØ—]‘] 7¨L¼#¿)¹)ÀSÇYÁYÐPS×PYÑPYÓZˆ
Ø—}‘} W¨n¼c¿i¹iÈÏÉÐV[×VaÑVaÓbˆØ—m‘m G¨]Ñ<aÓbˆáØAK×AQÑAQ×AWÑAWÑ>ˆZ˜ Y°à#×)Ñ)ˆIØœS ›_Ñ,°	¸1¸aÐ/@Ñ@ˆGÜ×*Ñ*¨:×+;Ñ+;¸SÀ'ÓJˆCÜ×,Ñ,¨\×-?Ñ-?ÀÈÓPˆEØ"ˆJÔØ!&ˆLÔØ(-¯©°A©Ð%Ø +× 1Ñ 1Ð4MÑ MˆKÔä×'Ñ'Ü—
‘
˜:Ó&¨Ð5NÑ)NÑNÜ�jÓ! QÐ(AÀ:Ð$NÑNóˆHô +¨8°^ÓDˆNØ�E˜>Ð)Ð)r4   NÚkey_value_statesÚ
init_cacheÚoutput_attentionsc                 ó4  — |d u}|j                   d   }	| j                  |«      }
|r#| j                  |«      }| j                  |«      }n"| j                  |«      }| j                  |«      }| j	                  |
«      }
| j	                  |«      }| j	                  |«      }| j
                  rÍ|
j                   d   |j                   d   }}| j                  dd«      r[| j                  d   d   }| j                  d   d   j                   d   }t        j                  | j                  dd|dfdd||f«      }n| j                  d d …d d …d |…d |…f   }t        j                  ||	f|j                   dd  z   «      }|�N| j
                  rBt        j                  t        j                  |d¬«      j                   «      }t        ||«      }n(| j
                  r}n|�t        j                  |d¬«      }| j
                  r,| j                  dd«      s|r| j                  |||
|«      \  }}}|�°t        j                   |dkD  t        j"                  |j                   d«      j%                  | j&                  «      t        j"                  |j                   t        j(                  | j&                  «      j*                  «      j%                  | j&                  «      «      }nd }d }|s*| j,                  j.                  dkD  r| j1                  d	«      }t3        |
|||| j,                  j.                  d
|| j&                  d ¬«	      }|�t        j4                  d||«      }t        j4                  d||«      }|j7                  |j                   d d dz   «      }|r||f}|S |f}|S )Nr   r$   r~   r   r�   )éýÿÿÿéþÿÿÿ©Úaxisg        rR   T)ÚbiasÚdropout_rngÚdropout_rateÚbroadcast_dropoutrV   r9   Ú	precisionz...hqk,h->...hqkz...hqk,...khd->...qhdrv   )éÿÿÿÿ)rx   ro   rp   rq   rz   rg   r†   Ú	variablesr   Údynamic_slicers   r0   r‹   Úexpand_dimsr	   r—   ÚselectÚfullrZ   r9   ÚfinfoÚminr8   Úattention_probs_dropout_probÚmake_rngr   Úeinsumrw   )rT   r*   r^   Úlayer_head_maskr˜   r™   rV   rš   Úis_cross_attentionÚ
batch_sizeÚquery_statesÚ
key_statesÚvalue_statesÚquery_lengthÚ
key_lengthÚ
mask_shiftÚmax_decoder_lengthrs   Úattention_biasr¡   Úattn_weightsÚattn_outputÚoutputss                          r5   ra   zFlaxBertSelfAttention.__call__&  sk  € ð .°TÐ9ÐØ"×(Ñ(¨Ñ+ˆ
ð —z‘z -Ó0ˆáàŸ™Ð"2Ó3ˆJØŸ:™:Ð&6Ó7‰Lð Ÿ™ -Ó0ˆJØŸ:™: mÓ4ˆLà×(Ñ(¨Ó6ˆØ×&Ñ& zÓ2ˆ
Ø×(Ñ(¨Ó6ˆð �;Š;Ø'3×'9Ñ'9¸!Ñ'<¸j×>NÑ>NÈqÑ>Q˜*ˆLØ× Ñ  ¨,Ô7Ø!Ÿ^™^¨GÑ4°]ÑC�
Ø%)§^¡^°GÑ%<¸\Ñ%J×%PÑ%PÐQRÑ%SÐ"Ü!×/Ñ/Ø×$Ñ$ q¨!¨Z¸Ð&;¸aÀÀLÐRdÐ=eó‘ð #×.Ñ.ªq²!°]°l°]ÀKÀZÀKÐ/OÑP�Ü×*Ñ*¨;¸¸È×HYÑHYÐZ[ÐZ\ÐH]Ñ8]Ó^ˆKð Ð%¨$¯+ª+Ü ×-Ñ-¬c¯o©o¸nÐS[Ô.\Ð^i×^oÑ^oÓpˆNÜ*¨>¸;ÓG‰NØ�[Š[Ø(‰NØÐ'Ü Ÿ_™_¨^À(ÔKˆNð �;Š;˜D×-Ñ-¨g°|ÔDÉ
Ø7;×7QÑ7QØ˜L¨,¸ó8Ñ4ˆJ˜ nð
 Ð%ä ŸZ™ZØ Ñ"Ü—‘˜×-Ñ-¨sÓ3×:Ñ:¸4¿:¹:ÓFÜ—‘˜×-Ñ-¬s¯y©y¸¿¹Ó/D×/HÑ/HÓI×PÑPÐQU×Q[ÑQ[Ó\ó‰Nð "ˆNàˆÙ §¡×!IÑ!IÈCÒ!OØŸ-™-¨	Ó2ˆKä4ØØØØ#ØŸ™×AÑAØ"Ø'Ø—*‘*Øô

ˆð Ð&ÜŸ:™:Ð&8¸,ÈÓXˆLä—j‘jÐ!8¸,ÈÓUˆØ!×)Ñ)¨+×*;Ñ*;¸B¸QÐ*?À%Ñ*GÓHˆá1B�; Ð-ˆØˆð JUÈˆØˆr4   ©NFTF)r,   r-   r.   r%   r2   rg   rd   r0   rc   r9   rU   rz   r|   rA   Úcompactr—   r   r1   ra   r3   r4   r5   rf   rf   Ý   sŒ   … ØÓØ€FˆDÓØ—{‘{€Eˆ3�9‰9Ó"òò:qò[ð ‡Z�Zñ*ó ð*ðH 37Ø ØØ"'ñ_ð
 # 3§;¡;Ñ/ð_ð ð_ð  ô_r4   rf   c                   ób   — e Zd ZU eed<   ej                  Zej                  ed<   d„ Zdde	fd„Z
y)ÚFlaxBertSelfOutputr8   r9   c                 óÂ  — t        j                  | j                  j                  t        j                   j
                  j                  | j                  j                  «      | j                  ¬«      | _	        t        j                  | j                  j                  | j                  ¬«      | _
        t        j                  | j                  j                  ¬«      | _        y )N©ri   r9   r=   r?   )rA   rn   r8   rD   rE   rF   rG   rH   r9   ÚdenserN   rO   rP   rQ   rR   rS   s    r5   rU   zFlaxBertSelfOutput.setupŒ  s‡   € Ü—X‘XØ�K‰K×#Ñ#ÜŸ™×+Ñ+×2Ñ2°4·;±;×3PÑ3PÓQØ—*‘*ô
ˆŒ
ô
 Ÿ™¨d¯k©k×.HÑ.HÐPT×PZÑPZÔ[ˆŒÜ—z‘z t§{¡{×'FÑ'FÔGˆ�r4   rV   c                 óv   — | j                  |«      }| j                  ||¬«      }| j                  ||z   «      }|S ©NrY   ©rÄ   rR   rN   )rT   r*   Úinput_tensorrV   s       r5   ra   zFlaxBertSelfOutput.__call__•  s;   € ØŸ
™
 =Ó1ˆØŸ™ ]À-˜ÓPˆØŸ™ }°|Ñ'CÓDˆØÐr4   Nrb   ©r,   r-   r.   r%   r2   r0   rc   r9   rU   rd   ra   r3   r4   r5   rÁ   rÁ   ˆ  s,   … ØÓØ—{‘{€Eˆ3�9‰9Ó"òHñÀ4ô r4   rÁ   c                   óx   — e Zd ZU eed<   dZeed<   ej                  Z	ej                  ed<   d„ Z
	 	 	 	 d	defd„Zy)
ÚFlaxBertAttentionr8   Frg   r9   c                 ó²   — t        | j                  | j                  | j                  ¬«      | _        t        | j                  | j                  ¬«      | _        y )N©rg   r9   rj   )rf   r8   rg   r9   rT   rÁ   ÚoutputrS   s    r5   rU   zFlaxBertAttention.setup¡  s7   € Ü)¨$¯+©+¸d¿k¹kÐQU×Q[ÑQ[Ô\ˆŒ	Ü(¨¯©¸D¿J¹JÔGˆ�r4   Nrš   c           	      ó„   — | j                  |||||||¬«      }|d   }	| j                  |	||¬«      }|f}
|r	|
|d   fz  }
|
S )N)r°   r˜   r™   rV   rš   r   rY   r$   )rT   rÎ   )rT   r*   r^   r°   r˜   r™   rV   rš   Úattn_outputsr¼   r½   s              r5   ra   zFlaxBertAttention.__call__¥  sl   € ð —y‘yØØØ+Ø-Ø!Ø'Ø/ð !ó 
ˆð # 1‘oˆØŸ™ K°Èm˜Ó\ˆà Ð"ˆáØ˜ Q™Ð)Ñ)ˆGàˆr4   r¾   )r,   r-   r.   r%   r2   rg   rd   r0   rc   r9   rU   ra   r3   r4   r5   rË   rË   œ  sG   … ØÓØ€FˆDÓØ—{‘{€Eˆ3�9‰9Ó"òHð ØØØ"'ñð  ôr4   rË   c                   óZ   — e Zd ZU eed<   ej                  Zej                  ed<   d„ Zd„ Z	y)ÚFlaxBertIntermediater8   r9   c                 ó4  — t        j                  | j                  j                  t        j                   j
                  j                  | j                  j                  «      | j                  ¬«      | _	        t        | j                  j                     | _        y ©NrÃ   )rA   rn   r8   Úintermediate_sizerE   rF   rG   rH   r9   rÄ   r   Ú
hidden_actÚ
activationrS   s    r5   rU   zFlaxBertIntermediate.setupÊ  s`   € Ü—X‘XØ�K‰K×)Ñ)ÜŸ™×+Ñ+×2Ñ2°4·;±;×3PÑ3PÓQØ—*‘*ô
ˆŒ
ô
 ! §¡×!7Ñ!7Ñ8ˆ�r4   c                 óJ   — | j                  |«      }| j                  |«      }|S ©N)rÄ   r×   ry   s     r5   ra   zFlaxBertIntermediate.__call__Ò  s$   € ØŸ
™
 =Ó1ˆØŸ™¨Ó6ˆØÐr4   N©
r,   r-   r.   r%   r2   r0   rc   r9   rU   ra   r3   r4   r5   rÒ   rÒ   Æ  s$   … ØÓØ—{‘{€Eˆ3�9‰9Ó"ò9ór4   rÒ   c                   ób   — e Zd ZU eed<   ej                  Zej                  ed<   d„ Zdde	fd„Z
y)ÚFlaxBertOutputr8   r9   c                 óÂ  — t        j                  | j                  j                  t        j                   j
                  j                  | j                  j                  «      | j                  ¬«      | _	        t        j                  | j                  j                  ¬«      | _        t        j                  | j                  j                  | j                  ¬«      | _        y )NrÃ   r?   r=   )rA   rn   r8   rD   rE   rF   rG   rH   r9   rÄ   rP   rQ   rR   rN   rO   rS   s    r5   rU   zFlaxBertOutput.setupÜ  s‡   € Ü—X‘XØ�K‰K×#Ñ#ÜŸ™×+Ñ+×2Ñ2°4·;±;×3PÑ3PÓQØ—*‘*ô
ˆŒ
ô
 —z‘z t§{¡{×'FÑ'FÔGˆŒÜŸ™¨d¯k©k×.HÑ.HÐPT×PZÑPZÔ[ˆ�r4   rV   c                 óv   — | j                  |«      }| j                  ||¬«      }| j                  ||z   «      }|S rÆ   rÇ   )rT   r*   Úattention_outputrV   s       r5   ra   zFlaxBertOutput.__call__å  s<   € ØŸ
™
 =Ó1ˆØŸ™ ]À-˜ÓPˆØŸ™ }Ð7GÑ'GÓHˆØÐr4   Nrb   rÉ   r3   r4   r5   rÜ   rÜ   Ø  s,   … ØÓØ—{‘{€Eˆ3�9‰9Ó"ò\ñÀtô r4   rÜ   c                   ó°   — e Zd ZU eed<   ej                  Zej                  ed<   d„ Z	 	 	 	 	 dde	ej                     de	ej                     deded	ef
d
„Zy)ÚFlaxBertLayerr8   r9   c                 óŽ  — t        | j                  | j                  j                  | j                  ¬«      | _        t        | j                  | j                  ¬«      | _        t        | j                  | j                  ¬«      | _        | j                  j                  r(t        | j                  d| j                  ¬«      | _
        y y )NrÍ   rj   F)rË   r8   Ú
is_decoderr9   Ú	attentionrÒ   ÚintermediaterÜ   rÎ   Úadd_cross_attentionÚcrossattentionrS   s    r5   rU   zFlaxBertLayer.setupð  s‚   € Ü*¨4¯;©;¸t¿{¹{×?UÑ?UÐ]a×]gÑ]gÔhˆŒÜ0°·±ÀDÇJÁJÔOˆÔÜ$ T§[¡[¸¿
¹
ÔCˆŒØ�;‰;×*Ò*Ü"3°D·K±KÈÐUY×U_ÑU_Ô"`ˆDÕð +r4   NÚencoder_hidden_statesÚencoder_attention_maskr™   rV   rš   c	                 óö   — | j                  ||||||¬«      }	|	d   }
|�| j                  |
|||||¬«      }|d   }
| j                  |
«      }| j                  ||
|¬«      }|f}|r||	d   fz  }|�	|d   fz  }|S )N)r°   r™   rV   rš   r   )r^   r°   r˜   rV   rš   rY   r$   )rä   rç   rå   rÎ   )rT   r*   r^   r°   rè   ré   r™   rV   rš   Úattention_outputsrß   Úcross_attention_outputsr½   s                r5   ra   zFlaxBertLayer.__call__÷  s×   € ð !ŸN™NØØØ+Ø!Ø'Ø/ð +ó 
Ðð -¨QÑ/Ðð !Ð,Ø&*×&9Ñ&9Ø Ø5Ø /Ø!6Ø+Ø"3ð ':ó 'Ð#ð  7°qÑ9Ðà×)Ñ)Ð*:Ó;ˆØŸ™ MÐ3CÐS`˜Óaˆà Ð"ˆáØÐ)¨!Ñ,Ð.Ñ.ˆGØ$Ð0ØÐ3°AÑ6Ð8Ñ8�Øˆr4   )NNFTF)r,   r-   r.   r%   r2   r0   rc   r9   rU   r   r1   rd   ra   r3   r4   r5   rá   rá   ì  sz   … ØÓØ—{‘{€Eˆ3�9‰9Ó"òað 8<Ø8<Ø Ø"Ø"'ñ+ð
  (¨¯©Ñ4ð+ð !)¨¯©Ñ 5ð+ð ð+ð ð+ð  ô+r4   rá   c                   óÊ   — e Zd ZU eed<   ej                  Zej                  ed<   dZe	ed<   d„ Z
	 	 	 	 	 	 	 ddeej                     deej                     d	e	d
e	de	de	de	fd„Zy)ÚFlaxBertLayerCollectionr8   r9   FÚgradient_checkpointingc           	      óº  — | j                   rjt        t        d¬«      }t        | j                  j
                  «      D �cg c]*  } || j                  t        |«      | j                  ¬«      ‘Œ, c}| _        y t        | j                  j
                  «      D �cg c]-  }t        | j                  t        |«      | j                  ¬«      ‘Œ/ c}| _        y c c}w c c}w )N)é   é   é   )Ústatic_argnums)Únamer9   )	rï   Úrematrá   Úranger8   Únum_hidden_layersÚstrr9   Úlayers)rT   ÚFlaxBertCheckpointLayerÚis      r5   rU   zFlaxBertLayerCollection.setup*  s¤   € Ø×&Ò&Ü&+¬MÈ)Ô&TÐ#ô ˜tŸ{™{×<Ñ<Ó=öàñ (¨¯©¼#¸a»&ÈÏ
É
ÖSòˆD�Kô TYÐY]×YdÑYd×YvÑYvÓSwöØNO”˜dŸk™k´°A³¸d¿j¹jÖIòˆD�Kùòùò
s   ¿/CÂ2CNrè   ré   r™   rV   rš   Úoutput_hidden_statesÚreturn_dictc                 óî  — |rdnd }|	rdnd }|r|�dnd }|�W|j                   d   t        | j                  «      k7  r2t        dt        | j                  «      › d|j                   d   › d�«      ‚t	        | j                  «      D ]@  \  }}|	r||fz  } ||||�||   nd |||||«      }|d   }|sŒ,||d   fz  }|€Œ8||d   fz  }ŒB |	r||fz  }||||f}|
st        d„ |D «       «      S t        ||||¬	«      S )
Nr3   r   z&The head_mask should be specified for z/ layers, but it is for                         ú.r$   rv   c              3   ó&   K  — | ]	  }|€Œ|–— Œ y ­wrÙ   r3   )Ú.0Úvs     r5   ú	<genexpr>z3FlaxBertLayerCollection.__call__.<locals>.<genexpr>l  s   è ø€ Ò=˜q¨q©}œÑ=ùs   ‚Š)Úlast_hidden_stater*   r+   Úcross_attentions)rx   r‰   rú   rm   Ú	enumerater�   r   )rT   r*   r^   Ú	head_maskrè   ré   r™   rV   rš   rý   rþ   Úall_attentionsÚall_hidden_statesÚall_cross_attentionsrü   ÚlayerÚlayer_outputsr½   s                     r5   ra   z FlaxBertLayerCollection.__call__6  sl  € ñ  1™°dˆÙ"6™B¸DÐÙ&7Ð<QÐ<]™rÐdhÐð Ð Ø�‰˜qÑ!¤c¨$¯+©+Ó&6Ò7Ü Ø<¼SÀÇÁÓ=MÐ<Nð OØ'Ÿo™o¨aÑ0Ð1°ð4óð ô
 " $§+¡+Ó.ò 	@‰HˆAˆuÙ#Ø! mÐ%5Ñ5Ð!á!ØØØ )Ð 5�	˜!’¸4Ø%Ø&ØØØ!ó	ˆMð *¨!Ñ,ˆMâ Ø =°Ñ#3Ð"5Ñ5�à(Ñ4Ø(¨]¸1Ñ-=Ð,?Ñ?Ñ(ð+	@ñ.  Ø -Ð!1Ñ1Ðà Ð"3°^ÐEYÐZˆáÜÑ= GÔ=Ó=Ð=ä<Ø+Ø+Ø%Ø1ô	
ð 	
r4   ©NNFTFFT©r,   r-   r.   r%   r2   r0   rc   r9   rï   rd   rU   r   r1   ra   r3   r4   r5   rî   rî   %  sž   … ØÓØ—{‘{€Eˆ3�9‰9Ó"Ø#(Ð˜DÓ(ò
ð" 8<Ø8<Ø Ø"Ø"'Ø%*Ø ñ=
ð
  (¨¯©Ñ4ð=
ð !)¨¯©Ñ 5ð=
ð ð=
ð ð=
ð  ð=
ð #ð=
ð ô=
r4   rî   c                   óÊ   — e Zd ZU eed<   ej                  Zej                  ed<   dZe	ed<   d„ Z
	 	 	 	 	 	 	 ddeej                     deej                     d	e	d
e	de	de	de	fd„Zy)ÚFlaxBertEncoderr8   r9   Frï   c                 óf   — t        | j                  | j                  | j                  ¬«      | _        y )N©r9   rï   )rî   r8   r9   rï   r  rS   s    r5   rU   zFlaxBertEncoder.setup{  s%   € Ü,Ø�K‰KØ—*‘*Ø#'×#>Ñ#>ô
ˆ�
r4   Nrè   ré   r™   rV   rš   rý   rþ   c                 ó8   — | j                  |||||||||	|
¬«
      S )N)r  rè   ré   r™   rV   rš   rý   rþ   )r  )rT   r*   r^   r  rè   ré   r™   rV   rš   rý   rþ   s              r5   ra   zFlaxBertEncoder.__call__‚  s8   € ð �z‰zØØØØ"7Ø#9Ø!Ø'Ø/Ø!5Ø#ð ó 
ð 	
r4   r  r  r3   r4   r5   r  r  v  sž   … ØÓØ—{‘{€Eˆ3�9‰9Ó"Ø#(Ð˜DÓ(ò
ð 8<Ø8<Ø Ø"Ø"'Ø%*Ø ñ
ð
  (¨¯©Ñ4ð
ð !)¨¯©Ñ 5ð
ð ð
ð ð
ð  ð
ð #ð
ð ô
r4   r  c                   óZ   — e Zd ZU eed<   ej                  Zej                  ed<   d„ Zd„ Z	y)ÚFlaxBertPoolerr8   r9   c                 óð   — t        j                  | j                  j                  t        j                   j
                  j                  | j                  j                  «      | j                  ¬«      | _	        y rÔ   )
rA   rn   r8   rD   rE   rF   rG   rH   r9   rÄ   rS   s    r5   rU   zFlaxBertPooler.setup¡  sH   € Ü—X‘XØ�K‰K×#Ñ#ÜŸ™×+Ñ+×2Ñ2°4·;±;×3PÑ3PÓQØ—*‘*ô
ˆ�
r4   c                 ó`   — |d d …df   }| j                  |«      }t        j                  |«      S )Nr   )rÄ   rA   Útanh)rT   r*   Úcls_hidden_states      r5   ra   zFlaxBertPooler.__call__¨  s1   € Ø(ª¨A¨Ñ.ÐØŸ:™:Ð&6Ó7ÐÜ�w‰wÐ'Ó(Ð(r4   NrÚ   r3   r4   r5   r  r  �  s$   … ØÓØ—{‘{€Eˆ3�9‰9Ó"ò
ó)r4   r  c                   óZ   — e Zd ZU eed<   ej                  Zej                  ed<   d„ Zd„ Z	y)ÚFlaxBertPredictionHeadTransformr8   r9   c                 ó0  — t        j                  | j                  j                  | j                  ¬«      | _        t        | j                  j                     | _        t        j                  | j                  j                  | j                  ¬«      | _	        y )Nrj   r=   )rA   rn   r8   rD   r9   rÄ   r   rÖ   r×   rN   rO   rS   s    r5   rU   z%FlaxBertPredictionHeadTransform.setup²  s[   € Ü—X‘X˜dŸk™k×5Ñ5¸T¿Z¹ZÔHˆŒ
Ü  §¡×!7Ñ!7Ñ8ˆŒÜŸ™¨d¯k©k×.HÑ.HÐPT×PZÑPZÔ[ˆ�r4   c                 óh   — | j                  |«      }| j                  |«      }| j                  |«      S rÙ   )rÄ   r×   rN   ry   s     r5   ra   z(FlaxBertPredictionHeadTransform.__call__·  s-   € ØŸ
™
 =Ó1ˆØŸ™¨Ó6ˆØ�~‰~˜mÓ,Ð,r4   NrÚ   r3   r4   r5   r  r  ®  s%   … ØÓØ—{‘{€Eˆ3�9‰9Ó"ò\ó
-r4   r  c                   óÄ   — e Zd ZU eed<   ej                  Zej                  ed<   ej                  j                  j                  Zedej                  f   ed<   d„ Zdd„Zy)	ÚFlaxBertLMPredictionHeadr8   r9   .Ú	bias_initc                 ó4  — t        | j                  | j                  ¬«      | _        t	        j
                  | j                  j                  | j                  d¬«      | _        | j                  d| j                  | j                  j                  f«      | _
        y )Nrj   F)r9   Úuse_biasr    )r  r8   r9   Ú	transformrA   rn   rC   ÚdecoderÚparamr!  r    rS   s    r5   rU   zFlaxBertLMPredictionHead.setupÂ  s`   € Ü8¸¿¹ÈDÏJÉJÔWˆŒÜ—x‘x §¡× 6Ñ 6¸d¿j¹jÐSXÔYˆŒØ—J‘J˜v t§~¡~¸¿¹×8NÑ8NÐ7PÓQˆ�	r4   Nc                 ó  — | j                  |«      }|�+| j                  j                  dd|j                  ii|«      }n| j                  |«      }t	        j
                  | j                  | j                  «      }||z  }|S )NÚparamsÚkernel)r$  r%  ÚapplyÚTr0   Úasarrayr    r9   )rT   r*   Úshared_embeddingr    s       r5   ra   z!FlaxBertLMPredictionHead.__call__Ç  st   € ØŸ™ }Ó5ˆàÐ'Ø ŸL™L×.Ñ.°¸8ÐEU×EWÑEWÐ:XÐ/YÐ[hÓi‰Mà ŸL™L¨Ó7ˆMä�{‰{˜4Ÿ9™9 d§j¡jÓ1ˆØ˜ÑˆØÐr4   rÙ   )r,   r-   r.   r%   r2   r0   rc   r9   rE   rA   rF   rˆ   r!  r   Únpr1   rU   ra   r3   r4   r5   r   r   ½  sL   … ØÓØ—{‘{€Eˆ3�9‰9Ó"Ø+.¯6©6×+>Ñ+>×+DÑ+D€Iˆx˜˜RŸZ™Z˜Ñ(ÓDòRô

r4   r   c                   ó\   — e Zd ZU eed<   ej                  Zej                  ed<   d„ Zdd„Z	y)ÚFlaxBertOnlyMLMHeadr8   r9   c                 óP   — t        | j                  | j                  ¬«      | _        y )Nrj   )r   r8   r9   ÚpredictionsrS   s    r5   rU   zFlaxBertOnlyMLMHead.setupØ  s   € Ü3°D·K±KÀtÇzÁzÔRˆÕr4   Nc                 ó,   — | j                  ||¬«      }|S ©N©r-  )r2  )rT   r*   r-  s      r5   ra   zFlaxBertOnlyMLMHead.__call__Û  s   € Ø×(Ñ(¨ÐIYÐ(ÓZˆØÐr4   rÙ   rÚ   r3   r4   r5   r0  r0  Ô  s%   … ØÓØ—{‘{€Eˆ3�9‰9Ó"òSôr4   r0  c                   óP   — e Zd ZU ej                  Zej
                  ed<   d„ Zd„ Zy)ÚFlaxBertOnlyNSPHeadr9   c                 óP   — t        j                  d| j                  ¬«      | _        y )Nrv   rj   )rA   rn   r9   Úseq_relationshiprS   s    r5   rU   zFlaxBertOnlyNSPHead.setupã  s   € Ü "§¡¨°$·*±*Ô =ˆÕr4   c                 ó$   — | j                  |«      S rÙ   )r9  )rT   Úpooled_outputs     r5   ra   zFlaxBertOnlyNSPHead.__call__æ  s   € Ø×$Ñ$ ]Ó3Ð3r4   N)	r,   r-   r.   r0   rc   r9   r2   rU   ra   r3   r4   r5   r7  r7  à  s   … Ø—{‘{€Eˆ3�9‰9Ó"ò>ó4r4   r7  c                   ó\   — e Zd ZU eed<   ej                  Zej                  ed<   d„ Zdd„Z	y)ÚFlaxBertPreTrainingHeadsr8   r9   c                 óœ   — t        | j                  | j                  ¬«      | _        t	        j
                  d| j                  ¬«      | _        y )Nrj   rv   )r   r8   r9   r2  rA   rn   r9  rS   s    r5   rU   zFlaxBertPreTrainingHeads.setupî  s0   € Ü3°D·K±KÀtÇzÁzÔRˆÔÜ "§¡¨°$·*±*Ô =ˆÕr4   Nc                 óR   — | j                  ||¬«      }| j                  |«      }||fS r4  )r2  r9  )rT   r*   r;  r-  Úprediction_scoresÚseq_relationship_scores         r5   ra   z!FlaxBertPreTrainingHeads.__call__ò  s6   € Ø ×,Ñ,¨]ÐM]Ð,Ó^ÐØ!%×!6Ñ!6°}Ó!EÐØ Ð"8Ð8Ð8r4   rÙ   rÚ   r3   r4   r5   r=  r=  ê  s$   … ØÓØ—{‘{€Eˆ3�9‰9Ó"ò>ô9r4   r=  c                   ó˜  ‡ — e Zd ZU dZeZdZdZej                  e
d<   ddej                  ddfd	ed
ededej                  dedefˆ fd„Zd„ Zddej(                  j*                  d
ededefd„Zd„ Z eej7                  d«      «      	 	 	 	 	 	 	 	 	 	 	 	 	 ddedej(                  j*                  dedee   dee   dee   defd„«       Zˆ xZS ) ÚFlaxBertPreTrainedModelz†
    An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained
    models.
    ÚbertNÚmodule_class)r$   r$   r   TFr8   Úinput_shapeÚseedr9   Ú_do_initrï   c                 ó\   •—  | j                   d|||dœ|¤Ž}t        ‰	| �	  ||||||¬«       y )N©r8   r9   rï   )rF  rG  r9   rH  r3   )rE  ÚsuperÚ__init__)
rT   r8   rF  rG  r9   rH  rï   ÚkwargsÚmoduleÚ	__class__s
            €r5   rL  z FlaxBertPreTrainedModel.__init__  sM   ø€ ð #�×"Ñ"ð 
ØØØ#9ñ
ð ñ	
ˆô 	‰Ñ˜ °[ÀtÐSXÐckÐÕlr4   c                 ó^   — | j                  | j                  | j                  d¬«      | _        y )NTrJ  )rE  r8   r9   Ú_modulerS   s    r5   Úenable_gradient_checkpointingz5FlaxBertPreTrainedModel.enable_gradient_checkpointing  s*   € Ø×(Ñ(Ø—;‘;Ø—*‘*Ø#'ð )ó 
ˆ�r4   Úrngr(  Úreturnc                 óÔ  — t        j                  |d¬«      }t        j                  |«      }t        j                  t        j                  t        j
                  |«      j                  d   «      |«      }t        j                  |«      }t        j                  | j                  j                  | j                  j                  f«      }t        j                  j                  |«      \  }	}
|	|
dœ}| j                  j                  rTt        j                  || j                  j                   fz   «      }|}| j"                  j%                  ||||||||d¬«	      }n"| j"                  j%                  ||||||d¬«      }|d   }|�dt'        t)        |«      «      }t'        t)        |«      «      }| j*                  D ]
  }||   ||<   Œ t-        «       | _        t/        t1        |«      «      S |S )NrX   rj   r¥   )r(  rR   F)rþ   r(  )r0   rˆ   Ú
zeros_liker‹   rŒ   Ú
atleast_2drx   Ú	ones_likerr   r8   rø   rk   rE   ÚrandomÚsplitræ   rD   rN  Úinitr   r   Ú_missing_keysÚsetr   r   )rT   rS  rF  r(  r[   r\   r]   r^   r  Ú
params_rngr¡   Úrngsrè   ré   Úmodule_init_outputsÚrandom_paramsÚmissing_keys                    r5   Úinit_weightsz$FlaxBertPreTrainedModel.init_weights  s´  € ä—I‘I˜k°Ô6ˆ	ÜŸ™¨	Ó2ˆÜ×'Ñ'¬¯
©
´3·>±>À)Ó3L×3RÑ3RÐSUÑ3VÓ(WÐYdÓeˆÜŸ™ yÓ1ˆÜ—H‘H˜dŸk™k×;Ñ;¸T¿[¹[×=\Ñ=\Ð]Ó^ˆ	ä"%§*¡*×"2Ñ"2°3Ó"7Ñˆ
�KØ$°Ñ=ˆà�;‰;×*Ò*Ü$'§I¡I¨k¸T¿[¹[×=TÑ=TÐ<VÑ.VÓ$WÐ!Ø%3Ð"Ø"&§+¡+×"2Ñ"2ØØØØØØØ%Ø&Ø!ð #3ó 
#Ñð #'§+¡+×"2Ñ"2Ø�i °ÀÈyÐfkð #3ó #Ðð ,¨HÑ5ˆàÐÜ(¬°-Ó)@ÓAˆMÜ!¤(¨6Ó"2Ó3ˆFØ#×1Ñ1ò A�Ø&3°KÑ&@��{Ò#ðAä!$£ˆDÔÜœ.¨Ó0Ó1Ð1à Ð r4   c                 ó   — t        j                  ||fd¬«      }t        j                  |d¬«      }t        j                  t        j                  t        j
                  |«      j                  d   «      |j                  «      }| j                  j                  t        j                  j                  d«      |||dd¬«      }t        |d   «      S )	aW  
        Args:
            batch_size (`int`):
                batch_size used for fast auto-regressive decoding. Defines the batch size of the initialized cache.
            max_length (`int`):
                maximum possible length for auto-regressive decoding. Defines the sequence length of the initialized
                cache.
        rX   rj   r¥   r   FT)rþ   r™   r~   )r0   rr   rX  r‹   rŒ   rW  rx   rN  r[  rE   rY  ÚPRNGKeyr   )rT   r²   r�   r[   r^   r]   Úinit_variabless          r5   r™   z"FlaxBertPreTrainedModel.init_cacheF  s©   € ô —H‘H˜j¨*Ð5¸TÔBˆ	ÜŸ™ y¸Ô=ˆÜ×'Ñ'¬¯
©
´3·>±>À)Ó3L×3RÑ3RÐSUÑ3VÓ(WÐYb×YhÑYhÓiˆàŸ™×)Ñ)Ü�J‰J×Ñ˜qÓ! 9¨n¸lÐX]Ðjnð *ó 
ˆô ˜ wÑ/Ó0Ð0r4   úbatch_size, sequence_lengthr¡   Útrainrš   rý   rþ   Úpast_key_valuesc                 óp  — |�|n| j                   j                  }|�|n| j                   j                  }|�|n| j                   j                  }|€t	        j
                  |«      }|€St	        j                  t	        j                  t	        j                  |«      j                  d   «      |j                  «      }|€t	        j                  |«      }|€?t	        j                  | j                   j                  | j                   j                  f«      }i }|	�|	|d<   d|xs | j                  i}| j                   j                  râ|r	||d<   dg}nd}| j                   j#                  |t	        j$                  |d¬«      t	        j$                  |d¬«      t	        j$                  |d¬«      t	        j$                  |d¬«      t	        j$                  |d¬«      |||
 |||||¬«      }|�|r|\  }}t'        |d   «      |d	<   |S |�"|s |\  }}|d d
 t'        |d   «      fz   |d
d  z   }|S | j                   j#                  |t	        j$                  |d¬«      t	        j$                  |d¬«      t	        j$                  |d¬«      t	        j$                  |d¬«      t	        j$                  |d¬«      |
 ||||¬«      }|S )Nr¥   rR   r(  r~   FrX   rj   )r\   r]   r  rè   ré   rV   rš   rý   rþ   r_  Úmutableri  r$   )r\   r]   r  rV   rš   rý   rþ   r_  )r8   rš   rý   rþ   r0   rV  r‹   rŒ   rW  rx   rX  rr   rø   rk   r(  ræ   rN  r*  rƒ   r   )rT   r[   r^   r\   r]   r  rè   ré   r(  r¡   rh  rš   rý   rþ   ri  r_  Úinputsrk  r½   s                      r5   ra   z FlaxBertPreTrainedModel.__call__Y  s¡  € ð$ 2CÐ1NÑ-ÐTX×T_ÑT_×TqÑTqÐà$8Ð$DÑ È$Ï+É+×JjÑJjð 	ð &1Ð%<‘kÀ$Ç+Á+×BYÑBYˆð Ð!Ü Ÿ^™^¨IÓ6ˆNàÐÜ×+Ñ+¬C¯J©J´s·~±~ÀiÓ7P×7VÑ7VÐWYÑ7ZÓ,[Ð]f×]lÑ]lÓmˆLàÐ!Ü Ÿ]™]¨9Ó5ˆNàÐÜŸ™ $§+¡+×"?Ñ"?ÀÇÁ×A`ÑA`Ð!aÓbˆIð ˆØÐ"Ø)ˆD�‰Oà˜FÒ1 d§k¡kÐ2ˆà�;‰;×*Ò*ñ Ø"1��w‘Ø"˜)‘à�à—k‘k×'Ñ'ØÜ—	‘	˜)¨4Ô0Ü—	‘	˜.°Ô5Ü"Ÿy™y¨¸tÔDÜ ŸY™Y |¸4Ô@ÜŸ)™) I°TÔ:Ø&;Ø'=Ø"'˜iØ"3Ø%9Ø'ØØð (ó ˆGð$ Ð*©{Ø+2Ñ(�˜Ü-5°oÀgÑ6NÓ-O�Ð)Ñ*Ø�Ø Ð,±[Ø+2Ñ(�˜Ø! " 1˜+¬°/À'Ñ2JÓ)KÐ(MÑMÐPWÐXYÐXZÐP[Ñ[�ð" ˆð —k‘k×'Ñ'ØÜ—	‘	˜)¨4Ô0Ü—	‘	˜.°Ô5Ü"Ÿy™y¨¸tÔDÜ ŸY™Y |¸4Ô@ÜŸ)™) I°TÔ:Ø"'˜iØ"3Ø%9Ø'Øð (ó ˆGð ˆr4   rÙ   )NNNNNNNNFNNNN) r,   r-   r.   r/   r%   Úconfig_classÚbase_model_prefixrE  rA   ÚModuler2   r0   rc   r   Úintr9   rd   rL  rR  rE   rY  re  r   rc  r™   r"   ÚBERT_INPUTS_DOCSTRINGÚformatÚdictr   ra   Ú__classcell__)rO  s   @r5   rC  rC  ø  so  ø… ñð
 €LØÐØ"€L�"—)‘)Ó"ð
 $ØØŸ;™;ØØ',ñmàðmð ðmð ð	mð
 �y‰yðmð ðmð !%õmò$
ñ(! §
¡
× 2Ñ 2ð (!Àð (!ÐPZð (!Ðfpó (!òV1ñ& +Ð+@×+GÑ+GÐHeÓ+fÓgð ØØØØ"Ø#ØØ*.ØØ,0Ø/3Ø&*Ø $ñ^ð ð^ð —Z‘Z×'Ñ'ð^ð ð^ð $ D™>ð^ð ' t™nð^ð ˜d‘^ð^ð ò^ó hô^r4   rC  c                   ó8  — e Zd ZU eed<   ej                  Zej                  ed<   dZe	ed<   dZ
e	ed<   d„ Z	 	 	 	 	 	 	 	 	 	 dd	eej                     d
eej                     deej                     deej                     deej                     de	de	de	de	de	fd„Zy)ÚFlaxBertModuler8   r9   TÚadd_pooling_layerFrï   c                 óþ   — t        | j                  | j                  ¬«      | _        t	        | j                  | j                  | j
                  ¬«      | _        t        | j                  | j                  ¬«      | _        y )Nrj   r  )	r7   r8   r9   Ú
embeddingsr  rï   Úencoderr  ÚpoolerrS   s    r5   rU   zFlaxBertModule.setupÁ  sS   € Ü,¨T¯[©[ÀÇ
Á
ÔKˆŒÜ&Ø�K‰KØ—*‘*Ø#'×#>Ñ#>ô
ˆŒô
 % T§[¡[¸¿
¹
ÔCˆ�r4   Nr\   r]   r  rè   ré   r™   rV   rš   rý   rþ   c                 ó  — |€t        j                  |«      }|€St        j                  t        j                  t        j                  |«      j
                  d   «      |j
                  «      }| j                  |||||	¬«      }| j                  ||||	||||
||¬«
      }|d   }| j                  r| j                  |«      nd }|s|€	|f|dd  z   S ||f|dd  z   S t        |||j                  |j                  |j                  ¬«      S )Nr¥   rY   )r  rV   rè   ré   r™   rš   rý   rþ   r   r$   )r  Úpooler_outputr*   r+   r  )r0   rV  r‹   rŒ   rW  rx   ry  rz  rw  r{  r   r*   r+   r  )rT   r[   r^   r\   r]   r  rè   ré   r™   rV   rš   rý   rþ   r*   r½   Úpooleds                   r5   ra   zFlaxBertModule.__call__Ê  s,  € ð  Ð!Ü Ÿ^™^¨IÓ6ˆNð ÐÜ×+Ñ+¬C¯J©J´s·~±~ÀiÓ7P×7VÑ7VÐWYÑ7ZÓ,[Ð]f×]lÑ]lÓmˆLàŸ™Ø�~ |°^ÐS`ð (ó 
ˆð —,‘,ØØØØ'Ø"7Ø#9Ø!Ø/Ø!5Ø#ð ó 
ˆð   ™
ˆØ/3×/EÒ/E�—‘˜]Ô+È4ˆáàˆ~Ø%Ð'¨'°!°"¨+Ñ5Ð5Ø! 6Ð*¨W°Q°R¨[Ñ8Ð8ä?Ø+Ø Ø!×/Ñ/Ø×)Ñ)Ø$×5Ñ5ô
ð 	
r4   )
NNNNNFTFFT)r,   r-   r.   r%   r2   r0   rc   r9   rw  rd   rï   rU   r   r1   ra   r3   r4   r5   rv  rv  »  sì   … ØÓØ—{‘{€Eˆ3�9‰9Ó"Ø"Ð�tÓ"Ø#(Ð˜DÓ(òDð 15Ø.2Ø+/Ø7;Ø8<Ø Ø"Ø"'Ø%*Ø ñ5
ð ! §¡Ñ-ð	5
ð
 ˜sŸ{™{Ñ+ð5
ð ˜CŸK™KÑ(ð5
ð  (¨¯©Ñ4ð5
ð !)¨¯©Ñ 5ð5
ð ð5
ð ð5
ð  ð5
ð #ð5
ð ô5
r4   rv  z^The bare Bert Model transformer outputting raw hidden-states without any specific head on top.c                   ó   — e Zd ZeZy)ÚFlaxBertModelN)r,   r-   r.   rv  rE  r3   r4   r5   r€  r€    s	   „ ð
 "�Lr4   r€  c            	       ó„   — e Zd ZU eed<   ej                  Zej                  ed<   dZe	ed<   d„ Z
	 	 	 	 dde	de	de	d	e	fd
„Zy)ÚFlaxBertForPreTrainingModuler8   r9   Frï   c                 ó²   — t        | j                  | j                  | j                  ¬«      | _        t        | j                  | j                  ¬«      | _        y )NrJ  ©r8   r9   )rv  r8   r9   rï   rD  r=  ÚclsrS   s    r5   rU   z"FlaxBertForPreTrainingModule.setup  s=   € Ü"Ø—;‘;Ø—*‘*Ø#'×#>Ñ#>ô
ˆŒ	ô
 ,°4·;±;ÀdÇjÁjÔQˆ�r4   rV   rš   rý   rþ   c
                 óL  — | j                  |||||||||	¬«	      }
| j                  j                  r#| j                   j                  d   d   d   d   }nd }|
d   }|
d   }| j	                  |||¬«      \  }}|	s
||f|
d	d  z   S t        |||
j                  |
j                  ¬
«      S )N©rV   rš   rý   rþ   r(  ry  rI   Ú	embeddingr   r$   r5  rv   )r(   r)   r*   r+   )rD  r8   Útie_word_embeddingsr¦   r…  r'   r*   r+   )rT   r[   r^   r\   r]   r  rV   rš   rý   rþ   r½   r-  r*   r;  r@  rA  s                   r5   ra   z%FlaxBertForPreTrainingModule.__call__  sâ   € ð —)‘)ØØØØØØ'Ø/Ø!5Ø#ð ó 

ˆð �;‰;×*Ò*Ø#Ÿy™y×2Ñ2°8Ñ<¸\ÑJÐK\Ñ]Ð^iÑjÑà#Ðà ™
ˆØ ™
ˆà48·H±HØ˜=Ð;Kð 5=ó 5
Ñ1ÐÐ1ñ Ø%Ð'=Ð>ÀÈÈÀÑLÐLä+Ø/Ø$:Ø!×/Ñ/Ø×)Ñ)ô	
ð 	
r4   N©TFFT©r,   r-   r.   r%   r2   r0   rc   r9   rï   rd   rU   ra   r3   r4   r5   r‚  r‚    sf   … ØÓØ—{‘{€Eˆ3�9‰9Ó"Ø#(Ð˜DÓ(òRð #Ø"'Ø%*Ø ñ-
ð ð-
ð  ð-
ð #ð-
ð ô-
r4   r‚  z¨
    Bert Model with two heads on top as done during the pretraining: a `masked language modeling` head and a `next
    sentence prediction (classification)` head.
    c                   ó   — e Zd ZeZy)ÚFlaxBertForPreTrainingN)r,   r-   r.   r‚  rE  r3   r4   r5   r�  r�  J  s	   „ ð 0�Lr4   r�  a  
    Returns:

    Example:

    ```python
    >>> from transformers import AutoTokenizer, FlaxBertForPreTraining

    >>> tokenizer = AutoTokenizer.from_pretrained("google-bert/bert-base-uncased")
    >>> model = FlaxBertForPreTraining.from_pretrained("google-bert/bert-base-uncased")

    >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="np")
    >>> outputs = model(**inputs)

    >>> prediction_logits = outputs.prediction_logits
    >>> seq_relationship_logits = outputs.seq_relationship_logits
    ```
rg  )Úoutput_typerm  c            	       ó„   — e Zd ZU eed<   ej                  Zej                  ed<   dZe	ed<   d„ Z
	 	 	 	 dde	de	de	d	e	fd
„Zy)ÚFlaxBertForMaskedLMModuler8   r9   Frï   c                 ó´   — t        | j                  d| j                  | j                  ¬«      | _        t        | j                  | j                  ¬«      | _        y ©NF)r8   rw  r9   rï   r„  ©rv  r8   r9   rï   rD  r0  r…  rS   s    r5   rU   zFlaxBertForMaskedLMModule.setupv  ó@   € Ü"Ø—;‘;Ø#Ø—*‘*Ø#'×#>Ñ#>ô	
ˆŒ	ô '¨d¯k©kÀÇÁÔLˆ�r4   rV   rš   rý   rþ   c
                 ó6  — | j                  |||||||||	¬«	      }
|
d   }| j                  j                  r#| j                   j                  d   d   d   d   }nd }| j	                  ||¬«      }|	s	|f|
dd  z   S t        ||
j                  |
j                  ¬	«      S )
Nr‡  r   r(  ry  rI   rˆ  r5  r$   ©Úlogitsr*   r+   )rD  r8   r‰  r¦   r…  r   r*   r+   )rT   r[   r^   r\   r]   r  rV   rš   rý   rþ   r½   r*   r-  r—  s                 r5   ra   z"FlaxBertForMaskedLMModule.__call__  sÃ   € ð —)‘)ØØØØØØ'Ø/Ø!5Ø#ð ó 

ˆð   ™
ˆØ�;‰;×*Ò*Ø#Ÿy™y×2Ñ2°8Ñ<¸\ÑJÐK\Ñ]Ð^iÑjÑà#Ðð —‘˜-Ð:J�ÓKˆáØ�9˜w q r˜{Ñ*Ð*ä!ØØ!×/Ñ/Ø×)Ñ)ô
ð 	
r4   NrŠ  r‹  r3   r4   r5   r�  r�  q  sf   … ØÓØ—{‘{€Eˆ3�9‰9Ó"Ø#(Ð˜DÓ(òMð  #Ø"'Ø%*Ø ñ)
ð ð)
ð  ð)
ð #ð)
ð ô)
r4   r�  z2Bert Model with a `language modeling` head on top.c                   ó   — e Zd ZeZy)ÚFlaxBertForMaskedLMN)r,   r-   r.   r�  rE  r3   r4   r5   r™  r™  «  s   „ à,�Lr4   r™  c            	       ó„   — e Zd ZU eed<   ej                  Zej                  ed<   dZe	ed<   d„ Z
	 	 	 	 dde	de	de	d	e	fd
„Zy)Ú'FlaxBertForNextSentencePredictionModuler8   r9   Frï   c                 óœ   — t        | j                  | j                  | j                  ¬«      | _        t        | j                  ¬«      | _        y )NrJ  rj   )rv  r8   r9   rï   rD  r7  r…  rS   s    r5   rU   z-FlaxBertForNextSentencePredictionModule.setup¸  s7   € Ü"Ø—;‘;Ø—*‘*Ø#'×#>Ñ#>ô
ˆŒ	ô
 '¨T¯Z©ZÔ8ˆ�r4   rV   rš   rý   rþ   c
                 óð   — |	�|	n| j                   j                  }	| j                  |||||||||	¬«	      }
|
d   }| j                  |«      }|	s	|f|
dd  z   S t	        ||
j
                  |
j                  ¬«      S )Nr‡  r$   rv   r–  )r8   rþ   rD  r…  r   r*   r+   )rT   r[   r^   r\   r]   r  rV   rš   rý   rþ   r½   r;  Úseq_relationship_scoress                r5   ra   z0FlaxBertForNextSentencePredictionModule.__call__À  s�   € ð &1Ð%<‘kÀ$Ç+Á+×BYÑBYˆð —)‘)ØØØØØØ'Ø/Ø!5Ø#ð ó 

ˆð   ™
ˆØ"&§(¡(¨=Ó"9ÐáØ+Ð-°¸¸°Ñ;Ð;ä.Ø*Ø!×/Ñ/Ø×)Ñ)ô
ð 	
r4   NrŠ  r‹  r3   r4   r5   r›  r›  ³  se   … ØÓØ—{‘{€Eˆ3�9‰9Ó"Ø#(Ð˜DÓ(ò9ð #Ø"'Ø%*Ø ñ%
ð ð%
ð  ð%
ð #ð%
ð ô%
r4   r›  zJBert Model with a `next sentence prediction (classification)` head on top.c                   ó   — e Zd ZeZy)Ú!FlaxBertForNextSentencePredictionN)r,   r-   r.   r›  rE  r3   r4   r5   r   r   è  s	   „ ð
 ;�Lr4   r   aØ  
    Returns:

    Example:

    ```python
    >>> from transformers import AutoTokenizer, FlaxBertForNextSentencePrediction

    >>> tokenizer = AutoTokenizer.from_pretrained("google-bert/bert-base-uncased")
    >>> model = FlaxBertForNextSentencePrediction.from_pretrained("google-bert/bert-base-uncased")

    >>> prompt = "In Italy, pizza served in formal settings, such as at a restaurant, is presented unsliced."
    >>> next_sentence = "The sky is blue due to the shorter wavelength of blue light."
    >>> encoding = tokenizer(prompt, next_sentence, return_tensors="jax")

    >>> outputs = model(**encoding)
    >>> logits = outputs.logits
    >>> assert logits[0, 0] < logits[0, 1]  # next sentence was random
    ```
c            	       ó„   — e Zd ZU eed<   ej                  Zej                  ed<   dZe	ed<   d„ Z
	 	 	 	 dde	de	de	d	e	fd
„Zy)Ú'FlaxBertForSequenceClassificationModuler8   r9   Frï   c                 ó”  — t        | j                  | j                  | j                  ¬«      | _        | j                  j
                  �| j                  j
                  n| j                  j                  }t        j                  |¬«      | _	        t        j                  | j                  j                  | j                  ¬«      | _        y )NrJ  r?   rj   ©rv  r8   r9   rï   rD  Úclassifier_dropoutrQ   rA   rP   rR   rn   Ú
num_labelsÚ
classifier©rT   r¥  s     r5   rU   z-FlaxBertForSequenceClassificationModule.setup  s�   € Ü"Ø—;‘;Ø—*‘*Ø#'×#>Ñ#>ô
ˆŒ	ð �{‰{×-Ñ-Ð9ð �K‰K×*Ò*à—‘×0Ñ0ð 	ô
 —z‘zÐ'9Ô:ˆŒÜŸ(™(Ø�K‰K×"Ñ"Ø—*‘*ô
ˆ�r4   rV   rš   rý   rþ   c
                 óâ   — | j                  |||||||||	¬«	      }
|
d   }| j                  ||¬«      }| j                  |«      }|	s	|f|
dd  z   S t        ||
j                  |
j
                  ¬«      S )Nr‡  r$   rY   rv   r–  )rD  rR   r§  r   r*   r+   )rT   r[   r^   r\   r]   r  rV   rš   rý   rþ   r½   r;  r—  s                r5   ra   z0FlaxBertForSequenceClassificationModule.__call__%  s•   € ð —)‘)ØØØØØØ'Ø/Ø!5Ø#ð ó 

ˆð   ™
ˆØŸ™ ]À-˜ÓPˆØ—‘ Ó/ˆáØ�9˜w q r˜{Ñ*Ð*ä+ØØ!×/Ñ/Ø×)Ñ)ô
ð 	
r4   NrŠ  r‹  r3   r4   r5   r¢  r¢    se   … ØÓØ—{‘{€Eˆ3�9‰9Ó"Ø#(Ð˜DÓ(ò
ð0 #Ø"'Ø%*Ø ñ$
ð ð$
ð  ð$
ð #ð$
ð ô$
r4   r¢  zœ
    Bert Model transformer with a sequence classification/regression head on top (a linear layer on top of the pooled
    output) e.g. for GLUE tasks.
    c                   ó   — e Zd ZeZy)Ú!FlaxBertForSequenceClassificationN)r,   r-   r.   r¢  rE  r3   r4   r5   r«  r«  L  s	   „ ð ;�Lr4   r«  c            	       ó„   — e Zd ZU eed<   ej                  Zej                  ed<   dZe	ed<   d„ Z
	 	 	 	 dde	de	de	d	e	fd
„Zy)ÚFlaxBertForMultipleChoiceModuler8   r9   Frï   c                 ó  — t        | j                  | j                  | j                  ¬«      | _        t        j                  | j                  j                  ¬«      | _        t        j                  d| j                  ¬«      | _
        y )NrJ  r?   r$   rj   )rv  r8   r9   rï   rD  rA   rP   rQ   rR   rn   r§  rS   s    r5   rU   z%FlaxBertForMultipleChoiceModule.setupd  sW   € Ü"Ø—;‘;Ø—*‘*Ø#'×#>Ñ#>ô
ˆŒ	ô
 —z‘z t§{¡{×'FÑ'FÔGˆŒÜŸ(™( 1¨D¯J©JÔ7ˆ�r4   rV   rš   rý   rþ   c
                 ó<  — |j                   d   }
|�|j                  d|j                   d   «      nd }|�|j                  d|j                   d   «      nd }|�|j                  d|j                   d   «      nd }|�|j                  d|j                   d   «      nd }| j                  |||||||||	¬«	      }|d   }| j                  ||¬«      }| j	                  |«      }|j                  d|
«      }|	s	|f|dd  z   S t        ||j                  |j                  ¬«      S )Nr$   r¥   r‡  rY   rv   r–  )rx   rw   rD  rR   r§  r   r*   r+   )rT   r[   r^   r\   r]   r  rV   rš   rý   rþ   Únum_choicesr½   r;  r—  Úreshaped_logitss                  r5   ra   z(FlaxBertForMultipleChoiceModule.__call__m  sH  € ð  —o‘o aÑ(ˆØBKÐBW�I×%Ñ% b¨)¯/©/¸"Ñ*=Ô>Ð]aˆ	ØQ_ÐQk˜×/Ñ/°°N×4HÑ4HÈÑ4LÔMÐquˆØQ_ÐQk˜×/Ñ/°°N×4HÑ4HÈÑ4LÔMÐquˆØKWÐKc�|×+Ñ+¨B°×0BÑ0BÀ2Ñ0FÔGÐimˆð —)‘)ØØØØØØ'Ø/Ø!5Ø#ð ó 

ˆð   ™
ˆØŸ™ ]À-˜ÓPˆØ—‘ Ó/ˆà Ÿ.™.¨¨[Ó9ˆáØ#Ð%¨°°¨Ñ3Ð3ä,Ø"Ø!×/Ñ/Ø×)Ñ)ô
ð 	
r4   NrŠ  r‹  r3   r4   r5   r­  r­  _  se   … ØÓØ—{‘{€Eˆ3�9‰9Ó"Ø#(Ð˜DÓ(ò8ð  #Ø"'Ø%*Ø ñ,
ð ð,
ð  ð,
ð #ð,
ð ô,
r4   r­  z¥
    Bert Model with a multiple choice classification head on top (a linear layer on top of the pooled output and a
    softmax) e.g. for RocStories/SWAG tasks.
    c                   ó   — e Zd ZeZy)ÚFlaxBertForMultipleChoiceN)r,   r-   r.   r­  rE  r3   r4   r5   r³  r³  œ  s	   „ ð 3�Lr4   r³  z(batch_size, num_choices, sequence_lengthc            	       ó„   — e Zd ZU eed<   ej                  Zej                  ed<   dZe	ed<   d„ Z
	 	 	 	 dde	de	de	d	e	fd
„Zy)Ú$FlaxBertForTokenClassificationModuler8   r9   Frï   c                 ó–  — t        | j                  | j                  d| j                  ¬«      | _        | j                  j
                  �| j                  j
                  n| j                  j                  }t        j                  |¬«      | _	        t        j                  | j                  j                  | j                  ¬«      | _        y )NF©r8   r9   rw  rï   r?   rj   r¤  r¨  s     r5   rU   z*FlaxBertForTokenClassificationModule.setup´  sŽ   € Ü"Ø—;‘;Ø—*‘*Ø#Ø#'×#>Ñ#>ô	
ˆŒ	ð �{‰{×-Ñ-Ð9ð �K‰K×*Ò*à—‘×0Ñ0ð 	ô
 —z‘zÐ'9Ô:ˆŒÜŸ(™( 4§;¡;×#9Ñ#9ÀÇÁÔLˆ�r4   rV   rš   rý   rþ   c
                 óâ   — | j                  |||||||||	¬«	      }
|
d   }| j                  ||¬«      }| j                  |«      }|	s	|f|
dd  z   S t        ||
j                  |
j
                  ¬«      S )Nr‡  r   rY   r$   r–  )rD  rR   r§  r   r*   r+   )rT   r[   r^   r\   r]   r  rV   rš   rý   rþ   r½   r*   r—  s                r5   ra   z-FlaxBertForTokenClassificationModule.__call__Ã  s•   € ð —)‘)ØØØØØØ'Ø/Ø!5Ø#ð ó 

ˆð   ™
ˆØŸ™ ]À-˜ÓPˆØ—‘ Ó/ˆáØ�9˜w q r˜{Ñ*Ð*ä(ØØ!×/Ñ/Ø×)Ñ)ô
ð 	
r4   NrŠ  r‹  r3   r4   r5   rµ  rµ  ¯  sf   … ØÓØ—{‘{€Eˆ3�9‰9Ó"Ø#(Ð˜DÓ(òMð, #Ø"'Ø%*Ø ñ$
ð ð$
ð  ð$
ð #ð$
ð ô$
r4   rµ  z£
    Bert Model with a token classification head on top (a linear layer on top of the hidden-states output) e.g. for
    Named-Entity-Recognition (NER) tasks.
    c                   ó   — e Zd ZeZy)ÚFlaxBertForTokenClassificationN)r,   r-   r.   rµ  rE  r3   r4   r5   rº  rº  ê  s	   „ ð 8�Lr4   rº  c            	       ó„   — e Zd ZU eed<   ej                  Zej                  ed<   dZe	ed<   d„ Z
	 	 	 	 dde	de	de	d	e	fd
„Zy)Ú"FlaxBertForQuestionAnsweringModuler8   r9   Frï   c                 óÜ   — t        | j                  | j                  d| j                  ¬«      | _        t        j                  | j                  j                  | j                  ¬«      | _        y )NFr·  rj   )	rv  r8   r9   rï   rD  rA   rn   r¦  Ú
qa_outputsrS   s    r5   rU   z(FlaxBertForQuestionAnsweringModule.setupÿ  sJ   € Ü"Ø—;‘;Ø—*‘*Ø#Ø#'×#>Ñ#>ô	
ˆŒ	ô Ÿ(™( 4§;¡;×#9Ñ#9ÀÇÁÔLˆ�r4   rV   rš   rý   rþ   c
                 ób  — | j                  |||||||||	¬«	      }
|
d   }| j                  |«      }t        j                  || j                  j
                  d¬«      \  }}|j                  d«      }|j                  d«      }|	s
||f|
dd  z   S t        |||
j                  |
j                  ¬«      S )Nr‡  r   r¥   rž   r$   )Ústart_logitsÚ
end_logitsr*   r+   )
rD  r¾  r0   rZ  r8   r¦  Úsqueezer   r*   r+   )rT   r[   r^   r\   r]   r  rV   rš   rý   rþ   r½   r*   r—  rÀ  rÁ  s                  r5   ra   z+FlaxBertForQuestionAnsweringModule.__call__  sË   € ð —)‘)ØØØØØØ'Ø/Ø!5Ø#ð ó 

ˆð   ™
ˆà—‘ Ó/ˆÜ#&§9¡9¨V°T·[±[×5KÑ5KÐRTÔ#UÑ ˆ�jØ#×+Ñ+¨BÓ/ˆØ×'Ñ'¨Ó+ˆ
áØ  *Ð-°¸¸°Ñ;Ð;ä/Ø%Ø!Ø!×/Ñ/Ø×)Ñ)ô	
ð 	
r4   NrŠ  r‹  r3   r4   r5   r¼  r¼  ú  sf   … ØÓØ—{‘{€Eˆ3�9‰9Ó"Ø#(Ð˜DÓ(òMð  #Ø"'Ø%*Ø ñ(
ð ð(
ð  ð(
ð #ð(
ð ô(
r4   r¼  zÝ
    Bert Model with a span classification head on top for extractive question-answering tasks like SQuAD (a linear
    layers on top of the hidden-states output to compute `span start logits` and `span end logits`).
    c                   ó   — e Zd ZeZy)ÚFlaxBertForQuestionAnsweringN)r,   r-   r.   r¼  rE  r3   r4   r5   rÄ  rÄ  3  s	   „ ð 6�Lr4   rÄ  c                   ó
  — e Zd ZU eed<   ej                  Zej                  ed<   dZe	ed<   d„ Z
	 	 	 	 	 	 	 	 	 ddeej                     deej                     d	eej                     d
eej                     de	de	de	de	de	fd„Zy)ÚFlaxBertForCausalLMModuler8   r9   Frï   c                 ó´   — t        | j                  d| j                  | j                  ¬«      | _        t        | j                  | j                  ¬«      | _        y r’  r“  rS   s    r5   rU   zFlaxBertForCausalLMModule.setupK  r”  r4   Nr\   r  rè   ré   r™   rV   rš   rý   rþ   c                 óR  — | j                  |||||||||	|
||¬«      }|d   }| j                  j                  r#| j                   j                  d   d   d   d   }nd }| j	                  ||¬«      }|s	|f|dd  z   S t        ||j                  |j                  |j                  ¬	«      S )
N)rè   ré   r™   rV   rš   rý   rþ   r   r(  ry  rI   rˆ  r5  r$   )r—  r*   r+   r  )	rD  r8   r‰  r¦   r…  r   r*   r+   r  )rT   r[   r^   r]   r\   r  rè   ré   r™   rV   rš   rý   rþ   r½   r*   r-  r—  s                    r5   ra   z"FlaxBertForCausalLMModule.__call__T  sÕ   € ð  —)‘)ØØØØØØ"7Ø#9Ø!Ø'Ø/Ø!5Ø#ð ó 
ˆð   ™
ˆØ�;‰;×*Ò*Ø#Ÿy™y×2Ñ2°8Ñ<¸\ÑJÐK\Ñ]Ð^iÑjÑà#Ðð —‘˜-Ð:J�ÓKˆáØ�9˜w q r˜{Ñ*Ð*ä4ØØ!×/Ñ/Ø×)Ñ)Ø$×5Ñ5ô	
ð 	
r4   )	NNNNFTFFTr  r3   r4   r5   rÆ  rÆ  F  sË   … ØÓØ—{‘{€Eˆ3�9‰9Ó"Ø#(Ð˜DÓ(òMð 15Ø+/Ø7;Ø8<Ø Ø"Ø"'Ø%*Ø ñ0
ð
 ! §¡Ñ-ð0
ð ˜CŸK™KÑ(ð0
ð  (¨¯©Ñ4ð0
ð !)¨¯©Ñ 5ð0
ð ð0
ð ð0
ð  ð0
ð #ð0
ð ô0
r4   rÆ  z�
    Bert Model with a language modeling head on top (a linear layer on top of the hidden-states output) e.g for
    autoregressive tasks.
    c                   ó>   — e Zd ZeZddeej                     fd„Zd„ Z	y)ÚFlaxBertForCausalLMNr^   c                 óH  — |j                   \  }}| j                  ||«      }t        j                  ||fd¬«      }|�-|j	                  d¬«      dz
  }t        j                  ||d«      }n4t        j                  t        j                  |d¬«      d d d …f   ||f«      }|||dœS )NrX   rj   r¥   rž   r$   )r   r   )ri  r^   r]   )	rx   r™   r0   rr   Úcumsumr   rŠ   r‹   rŒ   )	rT   r[   r�   r^   r²   Ú
seq_lengthri  Úextended_attention_maskr]   s	            r5   Úprepare_inputs_for_generationz1FlaxBertForCausalLM.prepare_inputs_for_generation‘  s±   € à!*§¡Ñˆ
�JàŸ/™/¨*°jÓAˆô #&§(¡(¨J¸
Ð+CÈ4Ô"PÐØÐ%Ø)×0Ñ0°bÐ0Ó9¸AÑ=ˆLÜ&)×&>Ñ&>Ð?VÐXfÐhnÓ&oÑ#ä×+Ñ+¬C¯J©J°zÈÔ,NÈtÒUVÈwÑ,WÐZdÐfpÐYqÓrˆLð  /Ø5Ø(ñ
ð 	
r4   c                 óL   — |j                   |d<   |d   d d …dd …f   dz   |d<   |S )Nri  r]   r¥   r$   )ri  )rT   Úmodel_outputsÚmodel_kwargss      r5   Úupdate_inputs_for_generationz0FlaxBertForCausalLM.update_inputs_for_generation¦  s8   € Ø*7×*GÑ*GˆÐ&Ñ'Ø'3°NÑ'CÂAÀrÁsÀFÑ'KÈaÑ'Oˆ�^Ñ$ØÐr4   rÙ   )
r,   r-   r.   rÆ  rE  r   rE   ÚArrayrÏ  rÓ  r3   r4   r5   rÊ  rÊ  ‡  s'   „ ð -€Lñ
ÐS[Ð\_×\eÑ\eÑSfó 
ó*r4   rÊ  )
rÊ  r™  r³  r   r�  rÄ  r«  rº  r€  rC  )eÚtypingr   r   r   ÚflaxÚ
flax.linenÚlinenrA   rE   Ú	jax.numpyÚnumpyr0   r.  Úflax.core.frozen_dictr   r   r   r	   r
   r   Únn_partitioningÚflax.linen.attentionr   Úflax.traverse_utilr   r   r   Úmodeling_flax_outputsr   r   r   r   r   r   r   r   r   r   Úmodeling_flax_utilsr   r   r   r   r   Úutilsr    r!   r"   r#   Úconfiguration_bertr%   Ú
get_loggerr,   ÚloggerÚ_CHECKPOINT_FOR_DOCÚ_CONFIG_FOR_DOCrö   ÚstructÚ	dataclassr'   ÚBERT_START_DOCSTRINGrq  ro  r7   rf   rÁ   rË   rÒ   rÜ   rá   rî   r  r  r  r   r0  r7  r=  rC  rv  r€  r‚  r�  Ú#FLAX_BERT_FOR_PRETRAINING_DOCSTRINGrr  r�  r™  r›  r   Ú&FLAX_BERT_FOR_NEXT_SENT_PRED_DOCSTRINGr¢  r«  r­  r³  rµ  rº  r¼  rÄ  rÆ  rÊ  Ú__all__r3   r4   r5   ú<module>rí     s  ð÷  -Ñ ,ã Ý Û 
Ý Û ß >Ñ >ß 6Ý 6Ý >ß ;Ý ÷÷ ÷ ÷õ ÷ gÓ fÝ *ð 
ˆ×	Ñ	˜HÓ	%€à5Ð Ø€à×Ñ€ð ‡�×Ñô4 ;ó 4ó ð4ð:.Ð ð`$Ð ôN(˜Ÿ™ô (ôVh˜BŸI™Iô hôV˜Ÿ™ô ô('˜Ÿ	™	ô 'ôT˜2Ÿ9™9ô ô$�R—Y‘Yô ô(6�B—I‘Iô 6ôrN
˜bŸi™iô N
ôb$
�b—i‘iô $
ôN)�R—Y‘Yô )ô"- b§i¡iô -ô˜rŸy™yô ô.	˜"Ÿ)™)ô 	ô4˜"Ÿ)™)ô 4ô9˜rŸy™yô 9ô@Ð1ô @ôFD
�R—Y‘Yô D
ñN ØdØóô"Ð+ó "ó	ð"ñ ˜]Ð,?ÐA_ÐapÔ qô:
 2§9¡9ô :
ñz ðð óô0Ð4ó 0óð0ð'Ð #ñ& ØØ× Ñ Ð!>Ó?ÐBeÑeôñ !ØÐ(DÐSbõô
7
 §	¡	ô 7
ñt ÐNÐPdÓeô-Ð1ó -ó fð-ñ Ð0Ð2EÐGYÐ[jÔ kô2
¨b¯i©iô 2
ñj ØTØóô;Ð(?ó ;ó	ð;ð*Ð &ñ, Ø%Ø× Ñ Ð!>Ó?ÐBhÑhôñ !Ø%Ð3RÐapõô
:
¨b¯i©iô :
ñz ðð óô;Ð(?ó ;óð;ñ Ø%ØØ Øô	ô:
 b§i¡iô :
ñz ðð óô3Ð 7ó 3óð3ñ ØÐ4×;Ñ;Ð<fÓgôñ ØÐ2Ð4QÐSbôô
8
¨2¯9©9ô 8
ñv ðð óô8Ð%<ó 8óð8ñ Ø"Ð$7Ð9RÐTcôô
6
¨¯©ô 6
ñr ðð óô6Ð#:ó 6óð6ñ Ø ØØ$Øô	ô>
 §	¡	ô >
ñB ðð óôÐ1ó óðñ< ØØØ)Øô	ò�r4   