Ë
    T^(h·ß  ã                   óš  — d dl mZmZmZ d dlmZ d dlZd dlm	Z
 d dl	Zd dlmZmZmZ d dlmZmZ d dlmZ d dlmZ d dlmZmZ d dlmZ d	d
lmZmZmZmZmZm Z m!Z!m"Z"m#Z# d	dl$m%Z%m&Z&m'Z'm(Z( d	dl)m*Z*m+Z+m,Z, ddl-m.Z.  e,j^                  e0«      Z1dZ2dZ3ejh                  Z4d„ Z5dZ6dZ7 G d„ dejp                  «      Z9 G d„ dejp                  «      Z: G d„ dejp                  «      Z; G d„ dejp                  «      Z< G d„ dejp                  «      Z= G d„ dejp                  «      Z> G d „ d!ejp                  «      Z? G d"„ d#ejp                  «      Z@ G d$„ d%ejp                  «      ZA G d&„ d'ejp                  «      ZB G d(„ d)ejp                  «      ZC G d*„ d+ejp                  «      ZD G d,„ d-e&«      ZE G d.„ d/ejp                  «      ZF e*d0e6«       G d1„ d2eE«      «       ZG e'eGe2ee3«        G d3„ d4ejp                  «      ZH e*d5e6«       G d6„ d7eE«      «       ZI e'eIe2ee3d8¬9«        G d:„ d;ejp                  «      ZJ e*d<e6«       G d=„ d>eE«      «       ZK e'eKe2e"e3«        G d?„ d@ejp                  «      ZL e*dAe6«       G dB„ dCeE«      «       ZM e(eMe7j�                  dD«      «        e'eMe2e e3«        G dE„ dFejp                  «      ZO e*dGe6«       G dH„ dIeE«      «       ZP e'ePe2e#e3«        G dJ„ dKejp                  «      ZQ e*dLe6«       G dM„ dNeE«      «       ZR e'eRe2e!e3«        G dO„ dPejp                  «      ZS e*dQe6«       G dR„ dSeE«      «       ZT e'eTe2ee3«       g dT¢ZUy)Ué    )ÚCallableÚOptionalÚTupleN)Ú
FrozenDictÚfreezeÚunfreeze)Úcombine_masksÚmake_causal_mask)Úpartitioning)Údot_product_attention_weights)Úflatten_dictÚunflatten_dict)Úlaxé   )	Ú-FlaxBaseModelOutputWithPastAndCrossAttentionsÚFlaxBaseModelOutputWithPoolingÚ0FlaxBaseModelOutputWithPoolingAndCrossAttentionsÚ%FlaxCausalLMOutputWithCrossAttentionsÚFlaxMaskedLMOutputÚFlaxMultipleChoiceModelOutputÚ FlaxQuestionAnsweringModelOutputÚFlaxSequenceClassifierOutputÚFlaxTokenClassifierOutput)ÚACT2FNÚFlaxPreTrainedModelÚappend_call_sample_docstringÚoverwrite_call_docstring)Úadd_start_docstringsÚ%add_start_docstrings_to_model_forwardÚloggingé   )ÚRobertaConfigzFacebookAI/roberta-baser"   c                 óŒ  — | |k7  j                  d«      }|j                  dkD  re|j                  d|j                  d   f«      }t	        j
                  |d¬«      j                  d«      |z  }|j                  | j                  «      }n)t	        j
                  |d¬«      j                  d«      |z  }|j                  d«      |z   S )a!  
    Replace non-padding symbols with their position numbers. Position numbers begin at padding_idx+1. Padding symbols
    are ignored. This is modified from fairseq's `utils.make_positions`.

    Args:
        input_ids: jnp.ndarray
        padding_idx: int

    Returns: jnp.ndarray
    Úi4é   éÿÿÿÿr!   ©Úaxis)ÚastypeÚndimÚreshapeÚshapeÚjnpÚcumsum)Ú	input_idsÚpadding_idxÚmaskÚincremental_indicess       úo/var/www/skyplay_api_hub/venv/lib/python3.12/site-packages/transformers/models/roberta/modeling_flax_roberta.pyÚ"create_position_ids_from_input_idsr4   4   s¬   € ð ˜Ñ$×,Ñ,¨TÓ2€Dà‡y�y�1‚}Ø�|‰|˜R §¡¨B¡Ð0Ó1ˆÜ!Ÿj™j¨°AÔ6×=Ñ=¸dÓCÀdÑJÐØ1×9Ñ9¸)¿/¹/ÓJÑä!Ÿj™j¨°AÔ6×=Ñ=¸dÓCÀdÑJÐà×%Ñ% dÓ+¨kÑ9Ð9ó    a   

    This model inherits from [`FlaxPreTrainedModel`]. Check the superclass documentation for the generic methods the
    library implements for all its model (such as downloading, saving and converting weights from PyTorch models)

    This model is also a
    [flax.linen.Module](https://flax.readthedocs.io/en/latest/api_reference/flax.linen/module.html) subclass. Use it as
    a regular Flax linen Module and refer to the Flax documentation for all matter related to general usage and
    behavior.

    Finally, this model supports inherent JAX features such as:

    - [Just-In-Time (JIT) compilation](https://jax.readthedocs.io/en/latest/jax.html#just-in-time-compilation-jit)
    - [Automatic Differentiation](https://jax.readthedocs.io/en/latest/jax.html#automatic-differentiation)
    - [Vectorization](https://jax.readthedocs.io/en/latest/jax.html#vectorization-vmap)
    - [Parallelization](https://jax.readthedocs.io/en/latest/jax.html#parallelization-pmap)

    Parameters:
        config ([`RobertaConfig`]): Model configuration class with all the parameters of the
            model. Initializing with a config file does not load the weights associated with the model, only the
            configuration. Check out the [`~FlaxPreTrainedModel.from_pretrained`] method to load the model weights.
a²  
    Args:
        input_ids (`numpy.ndarray` of shape `({0})`):
            Indices of input sequence tokens in the vocabulary.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are input IDs?](../glossary#input-ids)
        attention_mask (`numpy.ndarray` of shape `({0})`, *optional*):
            Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:

            - 1 for tokens that are **not masked**,
            - 0 for tokens that are **masked**.

            [What are attention masks?](../glossary#attention-mask)
        token_type_ids (`numpy.ndarray` of shape `({0})`, *optional*):
            Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
            1]`:

            - 0 corresponds to a *sentence A* token,
            - 1 corresponds to a *sentence B* token.

            [What are token type IDs?](../glossary#token-type-ids)
        position_ids (`numpy.ndarray` of shape `({0})`, *optional*):
            Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
            config.max_position_embeddings - 1]`.
        head_mask (`numpy.ndarray` of shape `({0})`, `optional):
            Mask to nullify selected heads of the attention modules. Mask values selected in `[0, 1]`:

            - 1 indicates the head is **not masked**,
            - 0 indicates the head is **masked**.

        return_dict (`bool`, *optional*):
            Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
c                   óf   — e Zd ZU dZeed<   ej                  Zej                  ed<   d„ Z	dde
fd„Zy)	ÚFlaxRobertaEmbeddingszGConstruct the embeddings from word, position and token_type embeddings.ÚconfigÚdtypec                 ó  — t        j                  | j                  j                  | j                  j                  t
        j                   j                  j                  | j                  j                  ¬«      | j                  ¬«      | _
        t        j                  | j                  j                  | j                  j                  t
        j                   j                  j                  | j                  j                  ¬«      | j                  ¬«      | _        t        j                  | j                  j                  | j                  j                  t
        j                   j                  j                  | j                  j                  ¬«      | j                  ¬«      | _        t        j                  | j                  j                   | j                  ¬«      | _        t        j"                  | j                  j$                  ¬«      | _        y )N)Ústddev)Úembedding_initr9   ©Úepsilonr9   ©Úrate)ÚnnÚEmbedr8   Ú
vocab_sizeÚhidden_sizeÚjaxÚinitializersÚnormalÚinitializer_ranger9   Úword_embeddingsÚmax_position_embeddingsÚposition_embeddingsÚtype_vocab_sizeÚtoken_type_embeddingsÚ	LayerNormÚlayer_norm_epsÚDropoutÚhidden_dropout_probÚdropout©Úselfs    r3   ÚsetupzFlaxRobertaEmbeddings.setup�   sJ  € Ü!Ÿx™xØ�K‰K×"Ñ"Ø�K‰K×#Ñ#ÜŸ6™6×.Ñ.×5Ñ5¸T¿[¹[×=ZÑ=ZÐ5Ó[Ø—*‘*ô	 
ˆÔô $&§8¡8Ø�K‰K×/Ñ/Ø�K‰K×#Ñ#ÜŸ6™6×.Ñ.×5Ñ5¸T¿[¹[×=ZÑ=ZÐ5Ó[Ø—*‘*ô	$
ˆÔ ô &(§X¡XØ�K‰K×'Ñ'Ø�K‰K×#Ñ#ÜŸ6™6×.Ñ.×5Ñ5¸T¿[¹[×=ZÑ=ZÐ5Ó[Ø—*‘*ô	&
ˆÔ"ô Ÿ™¨d¯k©k×.HÑ.HÐPT×PZÑPZÔ[ˆŒÜ—z‘z t§{¡{×'FÑ'FÔGˆ�r5   Údeterministicc                 ó  — | j                  |j                  d«      «      }| j                  |j                  d«      «      }| j                  |j                  d«      «      }||z   |z   }	| j	                  |	«      }	| j                  |	|¬«      }	|	S )Nr$   ©rV   )rI   r)   rK   rM   rN   rR   )
rT   r/   Útoken_type_idsÚposition_idsÚattention_maskrV   Úinputs_embedsÚposition_embedsrM   Úhidden_statess
             r3   Ú__call__zFlaxRobertaEmbeddings.__call__¦   s�   € à×,Ñ,¨Y×-=Ñ-=¸dÓ-CÓDˆØ×2Ñ2°<×3FÑ3FÀtÓ3LÓMˆØ $× :Ñ :¸>×;PÑ;PÐQUÓ;VÓ WÐð &Ð(=Ñ=ÀÑOˆð Ÿ™ }Ó5ˆØŸ™ ]À-˜ÓPˆØÐr5   N©T)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r"   Ú__annotations__r-   Úfloat32r9   rU   Úboolr_   © r5   r3   r7   r7   Š   s0   … ÙQàÓØ—{‘{€Eˆ3�9‰9Ó"òHñ,Ð_cô r5   r7   c                   óÊ   — e Zd ZU eed<   dZeed<   ej                  Z	ej                  ed<   d„ Z
d„ Zd„ Zej                  d„ «       Z	 	 	 	 dd
eej"                     dedefd„Zy	)ÚFlaxRobertaSelfAttentionr8   FÚcausalr9   c                 ó6  — | j                   j                  | j                   j                  z  | _        | j                   j                  | j                   j                  z  dk7  rt	        d«      ‚t        j                  | j                   j                  | j                  t        j
                  j                  j                  | j                   j                  «      ¬«      | _        t        j                  | j                   j                  | j                  t        j
                  j                  j                  | j                   j                  «      ¬«      | _        t        j                  | j                   j                  | j                  t        j
                  j                  j                  | j                   j                  «      ¬«      | _        | j                  r>t!        t#        j$                  d| j                   j&                  fd¬«      d¬«      | _        y y )Nr   z›`config.hidden_size`: {self.config.hidden_size} has to be a multiple of `config.num_attention_heads`                    : {self.config.num_attention_heads}©r9   Úkernel_initr!   rg   ©r9   )r8   rD   Únum_attention_headsÚhead_dimÚ
ValueErrorrA   ÚDenser9   rE   rF   rG   rH   ÚqueryÚkeyÚvaluerk   r
   r-   ÚonesrJ   Úcausal_maskrS   s    r3   rU   zFlaxRobertaSelfAttention.setup»   si  € ØŸ™×/Ñ/°4·;±;×3RÑ3RÑRˆŒØ�;‰;×"Ñ" T§[¡[×%DÑ%DÑDÈÒIÜðIóð ô
 —X‘XØ�K‰K×#Ñ#Ø—*‘*ÜŸ™×+Ñ+×2Ñ2°4·;±;×3PÑ3PÓQô
ˆŒ
ô
 —8‘8Ø�K‰K×#Ñ#Ø—*‘*ÜŸ™×+Ñ+×2Ñ2°4·;±;×3PÑ3PÓQô
ˆŒô
 —X‘XØ�K‰K×#Ñ#Ø—*‘*ÜŸ™×+Ñ+×2Ñ2°4·;±;×3PÑ3PÓQô
ˆŒ
ð �;Š;Ü/Ü—‘˜!˜TŸ[™[×@Ñ@ÐAÈÔPÐX^ô ˆDÕð r5   c                 ó„   — |j                  |j                  d d | j                  j                  | j                  fz   «      S ©Nr%   )r+   r,   r8   rp   rq   ©rT   r^   s     r3   Ú_split_headsz%FlaxRobertaSelfAttention._split_headsØ   s;   € Ø×$Ñ$ ]×%8Ñ%8¸¸!Ð%<ÀÇÁ×@_Ñ@_Ðae×anÑanÐ?oÑ%oÓpÐpr5   c                 ón   — |j                  |j                  d d | j                  j                  fz   «      S rz   )r+   r,   r8   rD   r{   s     r3   Ú_merge_headsz%FlaxRobertaSelfAttention._merge_headsÛ   s2   € Ø×$Ñ$ ]×%8Ñ%8¸¸!Ð%<ÀÇÁ×@WÑ@WÐ?YÑ%YÓZÐZr5   c                 ó(  — | j                  dd«      }| j                  ddt        j                  |j                  |j
                  «      }| j                  ddt        j                  |j                  |j
                  «      }| j                  ddd„ «      }|rø|j                  j                  �^ }	}
}}|j                  }dt        |	«      z  |ddfz   }t        j                  |j                  ||«      }t        j                  |j                  ||«      }||_        ||_        |j                  d   }|j                  |z   |_        t        j                  t        j                  |
«      ||z   k  t        |	«      d||
fz   «      }t        ||«      }|||fS )	a\  
        This function takes projected key, value states from a single input token and concatenates the states to cached
        states from previous steps. This function is slightly adapted from the official Flax repository:
        https://github.com/google/flax/blob/491ce18759622506588784b4fca0e4bf05f8c8cd/flax/linen/attention.py#L252
        ÚcacheÚ
cached_keyÚcached_valueÚcache_indexc                  óL   — t        j                  dt         j                  ¬«      S )Nr   ro   )r-   ÚarrayÚint32rh   r5   r3   ú<lambda>z@FlaxRobertaSelfAttention._concatenate_to_cache.<locals>.<lambda>ê   s   € ÄCÇIÁIÈaÔWZ×W`ÑW`ÔDa€ r5   )r   r   r!   )Úhas_variableÚvariabler-   Úzerosr,   r9   rv   Úlenr   Údynamic_update_sliceÚbroadcast_toÚarangeÚtupler	   )rT   ru   rv   rt   r[   Úis_initializedr�   r‚   rƒ   Ú
batch_dimsÚ
max_lengthÚ	num_headsÚdepth_per_headÚ	cur_indexÚindicesÚnum_updated_cache_vectorsÚpad_masks                    r3   Ú_concatenate_to_cachez.FlaxRobertaSelfAttention._concatenate_to_cacheÞ   sr  € ð ×*Ñ*¨7°LÓAˆØ—]‘] 7¨L¼#¿)¹)ÀSÇYÁYÐPS×PYÑPYÓZˆ
Ø—}‘} W¨n¼c¿i¹iÈÏÉÐV[×VaÑVaÓbˆØ—m‘m G¨]Ñ<aÓbˆáØAK×AQÑAQ×AWÑAWÑ>ˆZ˜ Y°à#×)Ñ)ˆIØœS ›_Ñ,°	¸1¸aÐ/@Ñ@ˆGÜ×*Ñ*¨:×+;Ñ+;¸SÀ'ÓJˆCÜ×,Ñ,¨\×-?Ñ-?ÀÈÓPˆEØ"ˆJÔØ!&ˆLÔØ(-¯©°A©Ð%Ø +× 1Ñ 1Ð4MÑ MˆKÔä×'Ñ'Ü—
‘
˜:Ó&¨Ð5NÑ)NÑNÜ�jÓ! QÐ(AÀ:Ð$NÑNóˆHô +¨8°^ÓDˆNØ�E˜>Ð)Ð)r5   NÚkey_value_statesÚ
init_cacheÚoutput_attentionsc                 ó4  — |d u}|j                   d   }	| j                  |«      }
|r#| j                  |«      }| j                  |«      }n"| j                  |«      }| j                  |«      }| j	                  |
«      }
| j	                  |«      }| j	                  |«      }| j
                  rÍ|
j                   d   |j                   d   }}| j                  dd«      r[| j                  d   d   }| j                  d   d   j                   d   }t        j                  | j                  dd|dfdd||f«      }n| j                  d d …d d …d |…d |…f   }t        j                  ||	f|j                   dd  z   «      }|�N| j
                  rBt        j                  t        j                  |d¬«      j                   «      }t        ||«      }n(| j
                  r}n|�t        j                  |d¬«      }| j
                  r,| j                  dd«      s|r| j                  |||
|«      \  }}}|�°t        j                   |dkD  t        j"                  |j                   d«      j%                  | j&                  «      t        j"                  |j                   t        j(                  | j&                  «      j*                  «      j%                  | j&                  «      «      }nd }d }|s*| j,                  j.                  dkD  r| j1                  d	«      }t3        |
|||| j,                  j.                  d
|| j&                  d ¬«	      }|�t        j4                  d||«      }t        j4                  d||«      }|j7                  |j                   d d dz   «      }|r||f}|S |f}|S )Nr   r!   r€   r�   rƒ   )éýÿÿÿéþÿÿÿr'   g        rR   T)ÚbiasÚdropout_rngÚdropout_rateÚbroadcast_dropoutrV   r9   Ú	precisionz...hqk,h->...hqkz...hqk,...khd->...qhdr%   )r&   )r,   rt   ru   rv   r|   rk   rˆ   Ú	variablesr   Údynamic_slicerx   r-   r�   Úexpand_dimsr	   r™   ÚselectÚfullr)   r9   ÚfinfoÚminr8   Úattention_probs_dropout_probÚmake_rngr   Úeinsumr+   )rT   r^   r[   Úlayer_head_maskrš   r›   rV   rœ   Úis_cross_attentionÚ
batch_sizeÚquery_statesÚ
key_statesÚvalue_statesÚquery_lengthÚ
key_lengthÚ
mask_shiftÚmax_decoder_lengthrx   Úattention_biasr¡   Úattn_weightsÚattn_outputÚoutputss                          r3   r_   z!FlaxRobertaSelfAttention.__call__ÿ   sk  € ð .°TÐ9ÐØ"×(Ñ(¨Ñ+ˆ
ð —z‘z -Ó0ˆáàŸ™Ð"2Ó3ˆJØŸ:™:Ð&6Ó7‰Lð Ÿ™ -Ó0ˆJØŸ:™: mÓ4ˆLà×(Ñ(¨Ó6ˆØ×&Ñ& zÓ2ˆ
Ø×(Ñ(¨Ó6ˆð �;Š;Ø'3×'9Ñ'9¸!Ñ'<¸j×>NÑ>NÈqÑ>Q˜*ˆLØ× Ñ  ¨,Ô7Ø!Ÿ^™^¨GÑ4°]ÑC�
Ø%)§^¡^°GÑ%<¸\Ñ%J×%PÑ%PÐQRÑ%SÐ"Ü!×/Ñ/Ø×$Ñ$ q¨!¨Z¸Ð&;¸aÀÀLÐRdÐ=eó‘ð #×.Ñ.ªq²!°]°l°]ÀKÀZÀKÐ/OÑP�Ü×*Ñ*¨;¸¸È×HYÑHYÐZ[ÐZ\ÐH]Ñ8]Ó^ˆKð Ð%¨$¯+ª+Ü ×-Ñ-¬c¯o©o¸nÐS[Ô.\Ð^i×^oÑ^oÓpˆNÜ*¨>¸;ÓG‰NØ�[Š[Ø(‰NØÐ'Ü Ÿ_™_¨^À(ÔKˆNð �;Š;˜D×-Ñ-¨g°|ÔDÉ
Ø7;×7QÑ7QØ˜L¨,¸ó8Ñ4ˆJ˜ nð
 Ð%ä ŸZ™ZØ Ñ"Ü—‘˜×-Ñ-¨sÓ3×:Ñ:¸4¿:¹:ÓFÜ—‘˜×-Ñ-¬s¯y©y¸¿¹Ó/D×/HÑ/HÓI×PÑPÐQU×Q[ÑQ[Ó\ó‰Nð "ˆNàˆÙ §¡×!IÑ!IÈCÒ!OØŸ-™-¨	Ó2ˆKä4ØØØØ#ØŸ™×AÑAØ"Ø'Ø—*‘*Øô

ˆð Ð&ÜŸ:™:Ð&8¸,ÈÓXˆLä—j‘jÐ!8¸,ÈÓUˆØ!×)Ñ)¨+×*;Ñ*;¸B¸QÐ*?À%Ñ*GÓHˆá1B�; Ð-ˆØˆð JUÈˆØˆr5   ©NFTF)ra   rb   rc   r"   re   rk   rg   r-   rf   r9   rU   r|   r~   rA   Úcompactr™   r   Úndarrayr_   rh   r5   r3   rj   rj   ¶   sŒ   … ØÓØ€FˆDÓØ—{‘{€Eˆ3�9‰9Ó"òò:qò[ð ‡Z�Zñ*ó ð*ðH 37Ø ØØ"'ñ_ð
 # 3§;¡;Ñ/ð_ð ð_ð  ô_r5   rj   c                   ób   — e Zd ZU eed<   ej                  Zej                  ed<   d„ Zdde	fd„Z
y)ÚFlaxRobertaSelfOutputr8   r9   c                 óÂ  — t        j                  | j                  j                  t        j                   j
                  j                  | j                  j                  «      | j                  ¬«      | _	        t        j                  | j                  j                  | j                  ¬«      | _
        t        j                  | j                  j                  ¬«      | _        y )N©rn   r9   r=   r?   )rA   rs   r8   rD   rE   rF   rG   rH   r9   ÚdenserN   rO   rP   rQ   rR   rS   s    r3   rU   zFlaxRobertaSelfOutput.setupf  s‡   € Ü—X‘XØ�K‰K×#Ñ#ÜŸ™×+Ñ+×2Ñ2°4·;±;×3PÑ3PÓQØ—*‘*ô
ˆŒ
ô
 Ÿ™¨d¯k©k×.HÑ.HÐPT×PZÑPZÔ[ˆŒÜ—z‘z t§{¡{×'FÑ'FÔGˆ�r5   rV   c                 óv   — | j                  |«      }| j                  ||¬«      }| j                  ||z   «      }|S ©NrX   ©rÄ   rR   rN   )rT   r^   Úinput_tensorrV   s       r3   r_   zFlaxRobertaSelfOutput.__call__o  s;   € ØŸ
™
 =Ó1ˆØŸ™ ]À-˜ÓPˆØŸ™ }°|Ñ'CÓDˆØÐr5   Nr`   ©ra   rb   rc   r"   re   r-   rf   r9   rU   rg   r_   rh   r5   r3   rÁ   rÁ   b  s,   … ØÓØ—{‘{€Eˆ3�9‰9Ó"òHñÀ4ô r5   rÁ   c                   óx   — e Zd ZU eed<   dZeed<   ej                  Z	ej                  ed<   d„ Z
	 	 	 	 d	defd„Zy)
ÚFlaxRobertaAttentionr8   Frk   r9   c                 ó²   — t        | j                  | j                  | j                  ¬«      | _        t        | j                  | j                  ¬«      | _        y )N©rk   r9   ro   )rj   r8   rk   r9   rT   rÁ   ÚoutputrS   s    r3   rU   zFlaxRobertaAttention.setup|  s7   € Ü,¨T¯[©[ÀÇÁÐTX×T^ÑT^Ô_ˆŒ	Ü+¨D¯K©K¸t¿z¹zÔJˆ�r5   Nrœ   c           	      ó„   — | j                  |||||||¬«      }|d   }	| j                  |	||¬«      }|f}
|r	|
|d   fz  }
|
S )N)r¯   rš   r›   rV   rœ   r   rX   r!   )rT   rÎ   )rT   r^   r[   r¯   rš   r›   rV   rœ   Úattn_outputsr»   r¼   s              r3   r_   zFlaxRobertaAttention.__call__€  sl   € ð —y‘yØØØ+Ø-Ø!Ø'Ø/ð !ó 
ˆð # 1‘oˆØŸ™ K°Èm˜Ó\ˆà Ð"ˆáØ˜ Q™Ð)Ñ)ˆGàˆr5   r½   )ra   rb   rc   r"   re   rk   rg   r-   rf   r9   rU   r_   rh   r5   r3   rË   rË   w  sG   … ØÓØ€FˆDÓØ—{‘{€Eˆ3�9‰9Ó"òKð ØØØ"'ñð  ôr5   rË   c                   óZ   — e Zd ZU eed<   ej                  Zej                  ed<   d„ Zd„ Z	y)ÚFlaxRobertaIntermediater8   r9   c                 ó4  — t        j                  | j                  j                  t        j                   j
                  j                  | j                  j                  «      | j                  ¬«      | _	        t        | j                  j                     | _        y ©NrÃ   )rA   rs   r8   Úintermediate_sizerE   rF   rG   rH   r9   rÄ   r   Ú
hidden_actÚ
activationrS   s    r3   rU   zFlaxRobertaIntermediate.setup¦  s`   € Ü—X‘XØ�K‰K×)Ñ)ÜŸ™×+Ñ+×2Ñ2°4·;±;×3PÑ3PÓQØ—*‘*ô
ˆŒ
ô
 ! §¡×!7Ñ!7Ñ8ˆ�r5   c                 óJ   — | j                  |«      }| j                  |«      }|S ©N)rÄ   r×   r{   s     r3   r_   z FlaxRobertaIntermediate.__call__®  s$   € ØŸ
™
 =Ó1ˆØŸ™¨Ó6ˆØÐr5   N©
ra   rb   rc   r"   re   r-   rf   r9   rU   r_   rh   r5   r3   rÒ   rÒ   ¢  s$   … ØÓØ—{‘{€Eˆ3�9‰9Ó"ò9ór5   rÒ   c                   ób   — e Zd ZU eed<   ej                  Zej                  ed<   d„ Zdde	fd„Z
y)ÚFlaxRobertaOutputr8   r9   c                 óÂ  — t        j                  | j                  j                  t        j                   j
                  j                  | j                  j                  «      | j                  ¬«      | _	        t        j                  | j                  j                  ¬«      | _        t        j                  | j                  j                  | j                  ¬«      | _        y )NrÃ   r?   r=   )rA   rs   r8   rD   rE   rF   rG   rH   r9   rÄ   rP   rQ   rR   rN   rO   rS   s    r3   rU   zFlaxRobertaOutput.setup¹  s‡   € Ü—X‘XØ�K‰K×#Ñ#ÜŸ™×+Ñ+×2Ñ2°4·;±;×3PÑ3PÓQØ—*‘*ô
ˆŒ
ô
 —z‘z t§{¡{×'FÑ'FÔGˆŒÜŸ™¨d¯k©k×.HÑ.HÐPT×PZÑPZÔ[ˆ�r5   rV   c                 óv   — | j                  |«      }| j                  ||¬«      }| j                  ||z   «      }|S rÆ   rÇ   )rT   r^   Úattention_outputrV   s       r3   r_   zFlaxRobertaOutput.__call__Â  s<   € ØŸ
™
 =Ó1ˆØŸ™ ]À-˜ÓPˆØŸ™ }Ð7GÑ'GÓHˆØÐr5   Nr`   rÉ   rh   r5   r3   rÜ   rÜ   µ  s,   … ØÓØ—{‘{€Eˆ3�9‰9Ó"ò\ñÀtô r5   rÜ   c                   ó°   — e Zd ZU eed<   ej                  Zej                  ed<   d„ Z	 	 	 	 	 dde	ej                     de	ej                     deded	ef
d
„Zy)ÚFlaxRobertaLayerr8   r9   c                 óŽ  — t        | j                  | j                  j                  | j                  ¬«      | _        t        | j                  | j                  ¬«      | _        t        | j                  | j                  ¬«      | _        | j                  j                  r(t        | j                  d| j                  ¬«      | _
        y y )NrÍ   ro   F)rË   r8   Ú
is_decoderr9   Ú	attentionrÒ   ÚintermediaterÜ   rÎ   Úadd_cross_attentionÚcrossattentionrS   s    r3   rU   zFlaxRobertaLayer.setupÎ  s‚   € Ü-¨d¯k©kÀ$Ç+Á+×BXÑBXÐ`d×`jÑ`jÔkˆŒÜ3°D·K±KÀtÇzÁzÔRˆÔÜ'¨¯©¸4¿:¹:ÔFˆŒØ�;‰;×*Ò*Ü"6°t·{±{È5ÐX\×XbÑXbÔ"cˆDÕð +r5   NÚencoder_hidden_statesÚencoder_attention_maskr›   rV   rœ   c	                 óö   — | j                  ||||||¬«      }	|	d   }
|�| j                  |
|||||¬«      }|d   }
| j                  |
«      }| j                  ||
|¬«      }|f}|r||	d   fz  }|�	|d   fz  }|S )N)r¯   r›   rV   rœ   r   )r[   r¯   rš   rV   rœ   rX   r!   )rä   rç   rå   rÎ   )rT   r^   r[   r¯   rè   ré   r›   rV   rœ   Úattention_outputsrß   Úcross_attention_outputsr¼   s                r3   r_   zFlaxRobertaLayer.__call__Õ  s×   € ð !ŸN™NØØØ+Ø!Ø'Ø/ð +ó 
Ðð -¨QÑ/Ðð !Ð,Ø&*×&9Ñ&9Ø Ø5Ø /Ø!6Ø+Ø"3ð ':ó 'Ð#ð  7°qÑ9Ðà×)Ñ)Ð*:Ó;ˆØŸ™ MÐ3CÐS`˜Óaˆà Ð"ˆáØÐ)¨!Ñ,Ð.Ñ.ˆGØ$Ð0ØÐ3°AÑ6Ð8Ñ8�Øˆr5   )NNFTF)ra   rb   rc   r"   re   r-   rf   r9   rU   r   r¿   rg   r_   rh   r5   r3   rá   rá   Ê  sz   … ØÓØ—{‘{€Eˆ3�9‰9Ó"òdð 8<Ø8<Ø Ø"Ø"'ñ+ð
  (¨¯©Ñ4ð+ð !)¨¯©Ñ 5ð+ð ð+ð ð+ð  ô+r5   rá   c                   óÊ   — e Zd ZU eed<   ej                  Zej                  ed<   dZe	ed<   d„ Z
	 	 	 	 	 	 	 ddeej                     deej                     d	e	d
e	de	de	de	fd„Zy)ÚFlaxRobertaLayerCollectionr8   r9   FÚgradient_checkpointingc           	      óº  — | j                   rjt        t        d¬«      }t        | j                  j
                  «      D �cg c]*  } || j                  t        |«      | j                  ¬«      ‘Œ, c}| _        y t        | j                  j
                  «      D �cg c]-  }t        | j                  t        |«      | j                  ¬«      ‘Œ/ c}| _        y c c}w c c}w )N)é   é   é   )Ústatic_argnums)Únamer9   )	rï   Úrematrá   Úranger8   Únum_hidden_layersÚstrr9   Úlayers)rT   ÚFlaxRobertaCheckpointLayerÚis      r3   rU   z FlaxRobertaLayerCollection.setup	  s¤   € Ø×&Ò&Ü).Ô/?ÐPYÔ)ZÐ&ô ˜tŸ{™{×<Ñ<Ó=öàñ +¨4¯;©;¼SÀ»VÈ4Ï:É:ÖVòˆD�Kô ˜tŸ{™{×<Ñ<Ó=öàô ! §¡´3°q³6ÀÇÁÖLòˆD�Kùòùò
s   ¿/CÂ2CNrè   ré   r›   rV   rœ   Úoutput_hidden_statesÚreturn_dictc                 óî  — |rdnd }|	rdnd }|r|�dnd }|�W|j                   d   t        | j                  «      k7  r2t        dt        | j                  «      › d|j                   d   › d�«      ‚t	        | j                  «      D ]@  \  }}|	r||fz  } ||||�||   nd |||||«      }|d   }|sŒ,||d   fz  }|€Œ8||d   fz  }ŒB |	r||fz  }||||f}|
st        d„ |D «       «      S t        ||||¬	«      S )
Nrh   r   z&The head_mask should be specified for z/ layers, but it is for                         ú.r!   r%   c              3   ó&   K  — | ]	  }|€Œ|–— Œ y ­wrÙ   rh   )Ú.0Úvs     r3   ú	<genexpr>z6FlaxRobertaLayerCollection.__call__.<locals>.<genexpr>L  s   è ø€ Ò=˜q¨q©}œÑ=ùs   ‚Š)Úlast_hidden_stater^   Ú
attentionsÚcross_attentions)r,   r‹   rú   rr   Ú	enumerater�   r   )rT   r^   r[   Ú	head_maskrè   ré   r›   rV   rœ   rý   rþ   Úall_attentionsÚall_hidden_statesÚall_cross_attentionsrü   ÚlayerÚlayer_outputsr¼   s                     r3   r_   z#FlaxRobertaLayerCollection.__call__  sl  € ñ  1™°dˆÙ"6™B¸DÐÙ&7Ð<QÐ<]™rÐdhÐð Ð Ø�‰˜qÑ!¤c¨$¯+©+Ó&6Ò7Ü Ø<¼SÀÇÁÓ=MÐ<Nð OØ'Ÿo™o¨aÑ0Ð1°ð4óð ô
 " $§+¡+Ó.ò 	@‰HˆAˆuÙ#Ø! mÐ%5Ñ5Ð!á!ØØØ )Ð 5�	˜!’¸4Ø%Ø&ØØØ!ó	ˆMð *¨!Ñ,ˆMâ Ø =°Ñ#3Ð"5Ñ5�à(Ñ4Ø(¨]¸1Ñ-=Ð,?Ñ?Ñ(ð+	@ñ.  Ø -Ð!1Ñ1Ðà Ð"3°^ÐEYÐZˆáÜÑ= GÔ=Ó=Ð=ä<Ø+Ø+Ø%Ø1ô	
ð 	
r5   ©NNFTFFT©ra   rb   rc   r"   re   r-   rf   r9   rï   rg   rU   r   r¿   r_   rh   r5   r3   rî   rî     sž   … ØÓØ—{‘{€Eˆ3�9‰9Ó"Ø#(Ð˜DÓ(òð$ 8<Ø8<Ø Ø"Ø"'Ø%*Ø ñ=
ð
  (¨¯©Ñ4ð=
ð !)¨¯©Ñ 5ð=
ð ð=
ð ð=
ð  ð=
ð #ð=
ð ô=
r5   rî   c                   óÊ   — e Zd ZU eed<   ej                  Zej                  ed<   dZe	ed<   d„ Z
	 	 	 	 	 	 	 ddeej                     deej                     d	e	d
e	de	de	de	fd„Zy)ÚFlaxRobertaEncoderr8   r9   Frï   c                 óf   — t        | j                  | j                  | j                  ¬«      | _        y )N©r9   rï   )rî   r8   r9   rï   r  rS   s    r3   rU   zFlaxRobertaEncoder.setup\  s%   € Ü/Ø�K‰KØ—*‘*Ø#'×#>Ñ#>ô
ˆ�
r5   Nrè   ré   r›   rV   rœ   rý   rþ   c                 ó8   — | j                  |||||||||	|
¬«
      S )N)r	  rè   ré   r›   rV   rœ   rý   rþ   )r  )rT   r^   r[   r	  rè   ré   r›   rV   rœ   rý   rþ   s              r3   r_   zFlaxRobertaEncoder.__call__c  s8   € ð �z‰zØØØØ"7Ø#9Ø!Ø'Ø/Ø!5Ø#ð ó 
ð 	
r5   r  r  rh   r5   r3   r  r  W  sž   … ØÓØ—{‘{€Eˆ3�9‰9Ó"Ø#(Ð˜DÓ(ò
ð 8<Ø8<Ø Ø"Ø"'Ø%*Ø ñ
ð
  (¨¯©Ñ4ð
ð !)¨¯©Ñ 5ð
ð ð
ð ð
ð  ð
ð #ð
ð ô
r5   r  c                   óZ   — e Zd ZU eed<   ej                  Zej                  ed<   d„ Zd„ Z	y)ÚFlaxRobertaPoolerr8   r9   c                 óð   — t        j                  | j                  j                  t        j                   j
                  j                  | j                  j                  «      | j                  ¬«      | _	        y rÔ   )
rA   rs   r8   rD   rE   rF   rG   rH   r9   rÄ   rS   s    r3   rU   zFlaxRobertaPooler.setupƒ  sH   € Ü—X‘XØ�K‰K×#Ñ#ÜŸ™×+Ñ+×2Ñ2°4·;±;×3PÑ3PÓQØ—*‘*ô
ˆ�
r5   c                 ó`   — |d d …df   }| j                  |«      }t        j                  |«      S )Nr   )rÄ   rA   Útanh)rT   r^   Úcls_hidden_states      r3   r_   zFlaxRobertaPooler.__call__Š  s1   € Ø(ª¨A¨Ñ.ÐØŸ:™:Ð&6Ó7ÐÜ�w‰wÐ'Ó(Ð(r5   NrÚ   rh   r5   r3   r  r    s$   … ØÓØ—{‘{€Eˆ3�9‰9Ó"ò
ó)r5   r  c                   óÄ   — e Zd ZU eed<   ej                  Zej                  ed<   ej                  j                  j                  Zedej                  f   ed<   d„ Zdd„Zy)	ÚFlaxRobertaLMHeadr8   r9   .Ú	bias_initc                 óÀ  — t        j                  | j                  j                  | j                  t
        j                   j                  j                  | j                  j                  «      ¬«      | _	        t        j                  | j                  j                  | j                  ¬«      | _        t        j                  | j                  j                  | j                  dt
        j                   j                  j                  | j                  j                  «      ¬«      | _        | j                  d| j                   | j                  j                  f«      | _        y )Nrm   r=   F)r9   Úuse_biasrn   r    )rA   rs   r8   rD   r9   rE   rF   rG   rH   rÄ   rN   rO   Ú
layer_normrC   ÚdecoderÚparamr  r    rS   s    r3   rU   zFlaxRobertaLMHead.setup•  sÝ   € Ü—X‘XØ�K‰K×#Ñ#Ø—*‘*ÜŸ™×+Ñ+×2Ñ2°4·;±;×3PÑ3PÓQô
ˆŒ
ô
 Ÿ,™,¨t¯{©{×/IÑ/IÐQU×Q[ÑQ[Ô\ˆŒÜ—x‘xØ�K‰K×"Ñ"Ø—*‘*ØÜŸ™×+Ñ+×2Ñ2°4·;±;×3PÑ3PÓQô	
ˆŒð —J‘J˜v t§~¡~¸¿¹×8NÑ8NÐ7PÓQˆ�	r5   Nc                 ó@  — | j                  |«      }t        d   |«      }| j                  |«      }|�+| j                  j	                  dd|j
                  ii|«      }n| j                  |«      }t        j                  | j                  | j                  «      }||z  }|S )NÚgeluÚparamsÚkernel)
rÄ   r   r!  r"  ÚapplyÚTr-   Úasarrayr    r9   )rT   r^   Úshared_embeddingr    s       r3   r_   zFlaxRobertaLMHead.__call__¤  s�   € ØŸ
™
 =Ó1ˆÜ˜v™ }Ó5ˆØŸ™¨Ó6ˆàÐ'Ø ŸL™L×.Ñ.°¸8ÐEU×EWÑEWÐ:XÐ/YÐ[hÓi‰Mà ŸL™L¨Ó7ˆMä�{‰{˜4Ÿ9™9 d§j¡jÓ1ˆØ˜ÑˆØÐr5   rÙ   )ra   rb   rc   r"   re   r-   rf   r9   rE   rA   rF   rŠ   r  r   Únpr¿   rU   r_   rh   r5   r3   r  r  �  sL   … ØÓØ—{‘{€Eˆ3�9‰9Ó"Ø+.¯6©6×+>Ñ+>×+DÑ+D€Iˆx˜˜RŸZ™Z˜Ñ(ÓDòRôr5   r  c                   ó\   — e Zd ZU eed<   ej                  Zej                  ed<   d„ Zdd„Z	y)ÚFlaxRobertaClassificationHeadr8   r9   c                 ó–  — t        j                  | j                  j                  | j                  t
        j                   j                  j                  | j                  j                  «      ¬«      | _	        | j                  j                  �| j                  j                  n| j                  j                  }t        j                  |¬«      | _        t        j                  | j                  j                  | j                  t
        j                   j                  j                  | j                  j                  «      ¬«      | _        y )Nrm   r?   )rA   rs   r8   rD   r9   rE   rF   rG   rH   rÄ   Úclassifier_dropoutrQ   rP   rR   Ú
num_labelsÚout_proj©rT   r0  s     r3   rU   z#FlaxRobertaClassificationHead.setup·  sÔ   € Ü—X‘XØ�K‰K×#Ñ#Ø—*‘*ÜŸ™×+Ñ+×2Ñ2°4·;±;×3PÑ3PÓQô
ˆŒ
ð �{‰{×-Ñ-Ð9ð �K‰K×*Ò*à—‘×0Ñ0ð 	ô
 —z‘zÐ'9Ô:ˆŒÜŸ™Ø�K‰K×"Ñ"Ø—*‘*ÜŸ™×+Ñ+×2Ñ2°4·;±;×3PÑ3PÓQô
ˆ�r5   c                 óØ   — |d d …dd d …f   }| j                  ||¬«      }| j                  |«      }t        j                  |«      }| j                  ||¬«      }| j	                  |«      }|S )Nr   rX   )rR   rÄ   rA   r  r2  )rT   r^   rV   s      r3   r_   z&FlaxRobertaClassificationHead.__call__É  sf   € Ø%¢a¨ªA gÑ.ˆØŸ™ ]À-˜ÓPˆØŸ
™
 =Ó1ˆÜŸ™ Ó.ˆØŸ™ ]À-˜ÓPˆØŸ™ mÓ4ˆØÐr5   Nr`   rÚ   rh   r5   r3   r.  r.  ³  s$   … ØÓØ—{‘{€Eˆ3�9‰9Ó"ò
ô$r5   r.  c                   ó˜  ‡ — e Zd ZU dZeZdZdZej                  e
d<   ddej                  ddfd	ed
ededej                  dedefˆ fd„Zd„ Zddej(                  j*                  d
ededefd„Zd„ Z eej7                  d«      «      	 	 	 	 	 	 	 	 	 	 	 	 	 ddedej(                  j*                  dedee   dee   dee   defd„«       Zˆ xZS ) ÚFlaxRobertaPreTrainedModelz†
    An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained
    models.
    ÚrobertaNÚmodule_class)r!   r!   r   TFr8   Úinput_shapeÚseedr9   Ú_do_initrï   c                 ó\   •—  | j                   d|||dœ|¤Ž}t        ‰	| �	  ||||||¬«       y )N©r8   r9   rï   )r9  r:  r9   r;  rh   )r8  ÚsuperÚ__init__)
rT   r8   r9  r:  r9   r;  rï   ÚkwargsÚmoduleÚ	__class__s
            €r3   r?  z#FlaxRobertaPreTrainedModel.__init__Þ  sA   ø€ ð #�×"Ñ"Ðw¨&¸ÐVlÑwÐpvÑwˆÜ‰Ñ˜ °[ÀtÐSXÐckÐÕlr5   c                 ó^   — | j                  | j                  | j                  d¬«      | _        y )NTr=  )r8  r8   r9   Ú_modulerS   s    r3   Úenable_gradient_checkpointingz8FlaxRobertaPreTrainedModel.enable_gradient_checkpointingì  s*   € Ø×(Ñ(Ø—;‘;Ø—*‘*Ø#'ð )ó 
ˆ�r5   Úrngr&  Úreturnc                 ó‚  — t        j                  |d¬«      }t        j                  |«      }t        || j                  j
                  «      }t        j                  |«      }t        j                  | j                  j                  | j                  j                  f«      }t        j                  j                  |«      \  }	}
|	|
dœ}| j                  j                  rTt        j                  || j                  j                  fz   «      }|}| j                  j                  ||||||||d¬«	      }n"| j                  j                  ||||||d¬«      }|d   }|�dt!        t#        |«      «      }t!        t#        |«      «      }| j$                  D ]
  }||   ||<   Œ t'        «       | _        t)        t+        |«      «      S |S )Nr$   ro   )r&  rR   F)rþ   r&  )r-   rŠ   Ú	ones_liker4   r8   Úpad_token_idrw   rø   rp   rE   ÚrandomÚsplitræ   rD   rA  Úinitr   r   Ú_missing_keysÚsetr   r   )rT   rF  r9  r&  r/   rY   rZ   r[   r	  Ú
params_rngr¡   Úrngsrè   ré   Úmodule_init_outputsÚrandom_paramsÚmissing_keys                    r3   Úinit_weightsz'FlaxRobertaPreTrainedModel.init_weightsó  s™  € ä—I‘I˜k°Ô6ˆ	ÜŸ™ yÓ1ˆÜ9¸)ÀTÇ[Á[×E]ÑE]Ó^ˆÜŸ™ yÓ1ˆÜ—H‘H˜dŸk™k×;Ñ;¸T¿[¹[×=\Ñ=\Ð]Ó^ˆ	ä"%§*¡*×"2Ñ"2°3Ó"7Ñˆ
�KØ$°Ñ=ˆà�;‰;×*Ò*Ü$'§I¡I¨k¸T¿[¹[×=TÑ=TÐ<VÑ.VÓ$WÐ!Ø%3Ð"Ø"&§+¡+×"2Ñ"2ØØØØØØØ%Ø&Ø!ð #3ó 
#Ñð #'§+¡+×"2Ñ"2Ø�i °ÀÈyÐfkð #3ó #Ðð ,¨HÑ5ˆàÐÜ(¬°-Ó)@ÓAˆMÜ!¤(¨6Ó"2Ó3ˆFØ#×1Ñ1ò A�Ø&3°KÑ&@��{Ò#ðAä!$£ˆDÔÜœ.¨Ó0Ó1Ð1à Ð r5   c                 ó   — t        j                  ||fd¬«      }t        j                  |d¬«      }t        j                  t        j                  t        j
                  |«      j                  d   «      |j                  «      }| j                  j                  t        j                  j                  d«      |||dd¬«      }t        |d   «      S )	aW  
        Args:
            batch_size (`int`):
                batch_size used for fast auto-regressive decoding. Defines the batch size of the initialized cache.
            max_length (`int`):
                maximum possible length for auto-regressive decoding. Defines the sequence length of the initialized
                cache.
        r$   ro   r&   r   FT)rþ   r›   r€   )r-   rw   rI  r�   rŽ   Ú
atleast_2dr,   rA  rM  rE   rK  ÚPRNGKeyr   )rT   r±   r’   r/   r[   rZ   Úinit_variabless          r3   r›   z%FlaxRobertaPreTrainedModel.init_cache  s©   € ô —H‘H˜j¨*Ð5¸TÔBˆ	ÜŸ™ y¸Ô=ˆÜ×'Ñ'¬¯
©
´3·>±>À)Ó3L×3RÑ3RÐSUÑ3VÓ(WÐYb×YhÑYhÓiˆàŸ™×)Ñ)Ü�J‰J×Ñ˜qÓ! 9¨n¸lÐX]Ðjnð *ó 
ˆô ˜ wÑ/Ó0Ð0r5   zbatch_size, sequence_lengthr¡   Útrainrœ   rý   rþ   Úpast_key_valuesc                 ó
  — |�|n| j                   j                  }|�|n| j                   j                  }|�|n| j                   j                  }|€t	        j
                  |«      }|€ t        || j                   j                  «      }|€t	        j                  |«      }|€?t	        j                  | j                   j                  | j                   j                  f«      }i }|	�|	|d<   d|xs | j                  i}| j                   j                  râ|r	||d<   dg}nd}| j                  j                  |t	        j                   |d¬«      t	        j                   |d¬«      t	        j                   |d¬«      t	        j                   |d¬«      t	        j                   |d¬«      |||
 |||||¬«      }|�|r|\  }}t#        |d   «      |d<   |S |�"|s |\  }}|d d	 t#        |d   «      fz   |d	d  z   }|S | j                  j                  |t	        j                   |d¬«      t	        j                   |d¬«      t	        j                   |d¬«      t	        j                   |d¬«      t	        j                   |d¬«      |
 ||||¬
«      }|S )NrR   r&  r€   Fr$   ro   )rY   rZ   r	  rè   ré   rV   rœ   rý   rþ   rQ  Úmutabler[  r!   )rY   rZ   r	  rV   rœ   rý   rþ   rQ  )r8   rœ   rý   rþ   r-   Ú
zeros_liker4   rJ  rI  rw   rø   rp   r&  ræ   rA  r(  r…   r   )rT   r/   r[   rY   rZ   r	  rè   ré   r&  r¡   rZ  rœ   rý   rþ   r[  rQ  Úinputsr]  r¼   s                      r3   r_   z#FlaxRobertaPreTrainedModel.__call__1  s€  € ð$ 2CÐ1NÑ-ÐTX×T_ÑT_×TqÑTqÐà$8Ð$DÑ È$Ï+É+×JjÑJjð 	ð &1Ð%<‘kÀ$Ç+Á+×BYÑBYˆð Ð!Ü Ÿ^™^¨IÓ6ˆNàÐÜ=¸iÈÏÉ×IaÑIaÓbˆLàÐ!Ü Ÿ]™]¨9Ó5ˆNàÐÜŸ™ $§+¡+×"?Ñ"?ÀÇÁ×A`ÑA`Ð!aÓbˆIð ˆØÐ"Ø)ˆD�‰Oà˜FÒ1 d§k¡kÐ2ˆà�;‰;×*Ò*ñ Ø"1��w‘Ø"˜)‘à�à—k‘k×'Ñ'ØÜ—	‘	˜)¨4Ô0Ü—	‘	˜.°Ô5Ü"Ÿy™y¨¸tÔDÜ ŸY™Y |¸4Ô@ÜŸ)™) I°TÔ:Ø&;Ø'=Ø"'˜iØ"3Ø%9Ø'ØØð (ó ˆGð$ Ð*©{Ø+2Ñ(�˜Ü-5°oÀgÑ6NÓ-O�Ð)Ñ*Ø�Ø Ð,±[Ø+2Ñ(�˜Ø! " 1˜+¬°/À'Ñ2JÓ)KÐ(MÑMÐPWÐXYÐXZÐP[Ñ[�ð" ˆð —k‘k×'Ñ'ØÜ—	‘	˜)¨4Ô0Ü—	‘	˜.°Ô5Ü"Ÿy™y¨¸tÔDÜ ŸY™Y |¸4Ô@ÜŸ)™) I°TÔ:Ø"'˜iØ"3Ø%9Ø'Øð (ó ˆGð ˆr5   rÙ   )NNNNNNNNFNNNN) ra   rb   rc   rd   r"   Úconfig_classÚbase_model_prefixr8  rA   ÚModulere   r-   rf   r   Úintr9   rg   r?  rE  rE   rK  rX  r   rU  r›   r   ÚROBERTA_INPUTS_DOCSTRINGÚformatÚdictr   r_   Ú__classcell__)rB  s   @r3   r6  r6  Ó  so  ø… ñð
 !€LØ!Ðà"€L�"—)‘)Ó"ð
 $ØØŸ;™;ØØ',ñmàðmð ðmð ð	mð
 �y‰yðmð ðmð !%õmò
ñ(! §
¡
× 2Ñ 2ð (!Àð (!ÐPZð (!Ðfpó (!òV1ñ& +Ð+C×+JÑ+JÐKhÓ+iÓjð ØØØØ"Ø#ØØ*.ØØ,0Ø/3Ø&*Ø $ñ^ð ð^ð —Z‘Z×'Ñ'ð^ð ð^ð $ D™>ð^ð ' t™nð^ð ˜d‘^ð^ð ò^ó kô^r5   r6  c                   ó8  — e Zd ZU eed<   ej                  Zej                  ed<   dZe	ed<   dZ
e	ed<   d„ Z	 	 	 	 	 	 	 	 	 	 dd	eej                     d
eej                     deej                     deej                     deej                     de	de	de	de	de	fd„Zy)ÚFlaxRobertaModuler8   r9   TÚadd_pooling_layerFrï   c                 óþ   — t        | j                  | j                  ¬«      | _        t	        | j                  | j                  | j
                  ¬«      | _        t        | j                  | j                  ¬«      | _        y )Nro   r  )	r7   r8   r9   Ú
embeddingsr  rï   Úencoderr  ÚpoolerrS   s    r3   rU   zFlaxRobertaModule.setupš  sS   € Ü/°·±À4Ç:Á:ÔNˆŒÜ)Ø�K‰KØ—*‘*Ø#'×#>Ñ#>ô
ˆŒô
 (¨¯©¸4¿:¹:ÔFˆ�r5   NrY   rZ   r	  rè   ré   r›   rV   rœ   rý   rþ   c                 ó  — |€t        j                  |«      }|€St        j                  t        j                  t        j                  |«      j
                  d   «      |j
                  «      }| j                  |||||	¬«      }| j                  ||||	||||
||¬«
      }|d   }| j                  r| j                  |«      nd }|s|€	|f|dd  z   S ||f|dd  z   S t        |||j                  |j                  |j                  ¬«      S )Nr&   rX   )r	  rV   rè   ré   r›   rœ   rý   rþ   r   r!   )r  Úpooler_outputr^   r  r  )r-   r^  r�   rŽ   rW  r,   rl  rm  rj  rn  r   r^   r  r  )rT   r/   r[   rY   rZ   r	  rè   ré   r›   rV   rœ   rý   rþ   r^   r¼   Úpooleds                   r3   r_   zFlaxRobertaModule.__call__£  s,  € ð  Ð!Ü Ÿ^™^¨IÓ6ˆNð ÐÜ×+Ñ+¬C¯J©J´s·~±~ÀiÓ7P×7VÑ7VÐWYÑ7ZÓ,[Ð]f×]lÑ]lÓmˆLàŸ™Ø�~ |°^ÐS`ð (ó 
ˆð —,‘,ØØØØ'Ø"7Ø#9Ø!Ø/Ø!5Ø#ð ó 
ˆð   ™
ˆØ/3×/EÒ/E�—‘˜]Ô+È4ˆáàˆ~Ø%Ð'¨'°!°"¨+Ñ5Ð5Ø! 6Ð*¨W°Q°R¨[Ñ8Ð8ä?Ø+Ø Ø!×/Ñ/Ø×)Ñ)Ø$×5Ñ5ô
ð 	
r5   )
NNNNNFTFFT)ra   rb   rc   r"   re   r-   rf   r9   rj  rg   rï   rU   r   r¿   r_   rh   r5   r3   ri  ri  ”  sì   … ØÓØ—{‘{€Eˆ3�9‰9Ó"Ø"Ð�tÓ"Ø#(Ð˜DÓ(òGð 15Ø.2Ø+/Ø7;Ø8<Ø Ø"Ø"'Ø%*Ø ñ5
ð ! §¡Ñ-ð	5
ð
 ˜sŸ{™{Ñ+ð5
ð ˜CŸK™KÑ(ð5
ð  (¨¯©Ñ4ð5
ð !)¨¯©Ñ 5ð5
ð ð5
ð ð5
ð  ð5
ð #ð5
ð ô5
r5   ri  zaThe bare RoBERTa Model transformer outputting raw hidden-states without any specific head on top.c                   ó   — e Zd ZeZy)ÚFlaxRobertaModelN)ra   rb   rc   ri  r8  rh   r5   r3   rs  rs  Û  s	   „ ð
 %�Lr5   rs  c            	       ó„   — e Zd ZU eed<   ej                  Zej                  ed<   dZe	ed<   d„ Z
	 	 	 	 dde	de	de	d	e	fd
„Zy)ÚFlaxRobertaForMaskedLMModuler8   r9   Frï   c                 ó´   — t        | j                  d| j                  | j                  ¬«      | _        t        | j                  | j                  ¬«      | _        y ©NF)r8   rj  r9   rï   ©r8   r9   ©ri  r8   r9   rï   r7  r  Úlm_headrS   s    r3   rU   z"FlaxRobertaForMaskedLMModule.setupë  ó@   € Ü(Ø—;‘;Ø#Ø—*‘*Ø#'×#>Ñ#>ô	
ˆŒô )°·±À4Ç:Á:ÔNˆ�r5   rV   rœ   rý   rþ   c
                 ó6  — | j                  |||||||||	¬«	      }
|
d   }| j                  j                  r#| j                   j                  d   d   d   d   }nd }| j	                  ||¬«      }|	s	|f|
dd  z   S t        ||
j                  |
j                  ¬	«      S )
N©rV   rœ   rý   rþ   r   r&  rl  rI   Ú	embedding©r+  r!   ©Úlogitsr^   r  )r7  r8   Útie_word_embeddingsr¥   rz  r   r^   r  )rT   r/   r[   rY   rZ   r	  rV   rœ   rý   rþ   r¼   r^   r+  r�  s                 r3   r_   z%FlaxRobertaForMaskedLMModule.__call__ô  sÃ   € ð —,‘,ØØØØØØ'Ø/Ø!5Ø#ð ó 

ˆð   ™
ˆØ�;‰;×*Ò*Ø#Ÿ|™|×5Ñ5°hÑ?ÀÑMÐN_Ñ`ÐalÑmÑà#Ðð —‘˜mÐ>N�ÓOˆáØ�9˜w q r˜{Ñ*Ð*ä!ØØ!×/Ñ/Ø×)Ñ)ô
ð 	
r5   N©TFFT©ra   rb   rc   r"   re   r-   rf   r9   rï   rg   rU   r_   rh   r5   r3   ru  ru  æ  sf   … ØÓØ—{‘{€Eˆ3�9‰9Ó"Ø#(Ð˜DÓ(òOð  #Ø"'Ø%*Ø ñ)
ð ð)
ð  ð)
ð #ð)
ð ô)
r5   ru  z5RoBERTa Model with a `language modeling` head on top.c                   ó   — e Zd ZeZy)ÚFlaxRobertaForMaskedLMN)ra   rb   rc   ru  r8  rh   r5   r3   r†  r†     s   „ à/�Lr5   r†  z<mask>)r1   c            	       ó„   — e Zd ZU eed<   ej                  Zej                  ed<   dZe	ed<   d„ Z
	 	 	 	 dde	de	de	d	e	fd
„Zy)Ú*FlaxRobertaForSequenceClassificationModuler8   r9   Frï   c                 ó´   — t        | j                  | j                  d| j                  ¬«      | _        t        | j                  | j                  ¬«      | _        y )NF©r8   r9   rj  rï   rx  )ri  r8   r9   rï   r7  r.  Ú
classifierrS   s    r3   rU   z0FlaxRobertaForSequenceClassificationModule.setup3  sC   € Ü(Ø—;‘;Ø—*‘*Ø#Ø#'×#>Ñ#>ô	
ˆŒô 8¸t¿{¹{ÐRV×R\ÑR\Ô]ˆ�r5   rV   rœ   rý   rþ   c
                 óÀ   — | j                  |||||||||	¬«	      }
|
d   }| j                  ||¬«      }|	s	|f|
dd  z   S t        ||
j                  |
j                  ¬«      S ©Nr}  r   rX   r!   r€  )r7  r‹  r   r^   r  )rT   r/   r[   rY   rZ   r	  rV   rœ   rý   rþ   r¼   Úsequence_outputr�  s                r3   r_   z3FlaxRobertaForSequenceClassificationModule.__call__<  s‡   € ð —,‘,ØØØØØØ'Ø/Ø!5Ø#ð ó 

ˆð " !™*ˆØ—‘ À�ÓNˆáØ�9˜w q r˜{Ñ*Ð*ä+ØØ!×/Ñ/Ø×)Ñ)ô
ð 	
r5   Nrƒ  r„  rh   r5   r3   rˆ  rˆ  .  sf   … ØÓØ—{‘{€Eˆ3�9‰9Ó"Ø#(Ð˜DÓ(ò^ð  #Ø"'Ø%*Ø ñ#
ð ð#
ð  ð#
ð #ð#
ð ô#
r5   rˆ  zŸ
    Roberta Model transformer with a sequence classification/regression head on top (a linear layer on top of the
    pooled output) e.g. for GLUE tasks.
    c                   ó   — e Zd ZeZy)Ú$FlaxRobertaForSequenceClassificationN)ra   rb   rc   rˆ  r8  rh   r5   r3   r�  r�  b  s	   „ ð >�Lr5   r�  c            	       ó„   — e Zd ZU eed<   ej                  Zej                  ed<   dZe	ed<   d„ Z
	 	 	 	 dde	de	de	d	e	fd
„Zy)Ú"FlaxRobertaForMultipleChoiceModuler8   r9   Frï   c                 ó  — t        | j                  | j                  | j                  ¬«      | _        t        j                  | j                  j                  ¬«      | _        t        j                  d| j                  ¬«      | _
        y )Nr=  r?   r!   ro   )ri  r8   r9   rï   r7  rA   rP   rQ   rR   rs   r‹  rS   s    r3   rU   z(FlaxRobertaForMultipleChoiceModule.setup{  sW   € Ü(Ø—;‘;Ø—*‘*Ø#'×#>Ñ#>ô
ˆŒô
 —z‘z t§{¡{×'FÑ'FÔGˆŒÜŸ(™( 1¨D¯J©JÔ7ˆ�r5   rV   rœ   rý   rþ   c
                 ó<  — |j                   d   }
|�|j                  d|j                   d   «      nd }|�|j                  d|j                   d   «      nd }|�|j                  d|j                   d   «      nd }|�|j                  d|j                   d   «      nd }| j                  |||||||||	¬«	      }|d   }| j                  ||¬«      }| j	                  |«      }|j                  d|
«      }|	s	|f|dd  z   S t        ||j                  |j                  ¬«      S )Nr!   r&   r}  rX   r%   r€  )r,   r+   r7  rR   r‹  r   r^   r  )rT   r/   r[   rY   rZ   r	  rV   rœ   rý   rþ   Únum_choicesr¼   Úpooled_outputr�  Úreshaped_logitss                  r3   r_   z+FlaxRobertaForMultipleChoiceModule.__call__„  sH  € ð  —o‘o aÑ(ˆØBKÐBW�I×%Ñ% b¨)¯/©/¸"Ñ*=Ô>Ð]aˆ	ØQ_ÐQk˜×/Ñ/°°N×4HÑ4HÈÑ4LÔMÐquˆØQ_ÐQk˜×/Ñ/°°N×4HÑ4HÈÑ4LÔMÐquˆØKWÐKc�|×+Ñ+¨B°×0BÑ0BÀ2Ñ0FÔGÐimˆð —,‘,ØØØØØØ'Ø/Ø!5Ø#ð ó 

ˆð   ™
ˆØŸ™ ]À-˜ÓPˆØ—‘ Ó/ˆà Ÿ.™.¨¨[Ó9ˆáØ#Ð%¨°°¨Ñ3Ð3ä,Ø"Ø!×/Ñ/Ø×)Ñ)ô
ð 	
r5   Nrƒ  r„  rh   r5   r3   r’  r’  v  se   … ØÓØ—{‘{€Eˆ3�9‰9Ó"Ø#(Ð˜DÓ(ò8ð  #Ø"'Ø%*Ø ñ,
ð ð,
ð  ð,
ð #ð,
ð ô,
r5   r’  z¨
    Roberta Model with a multiple choice classification head on top (a linear layer on top of the pooled output and a
    softmax) e.g. for RocStories/SWAG tasks.
    c                   ó   — e Zd ZeZy)ÚFlaxRobertaForMultipleChoiceN)ra   rb   rc   r’  r8  rh   r5   r3   r™  r™  ³  s	   „ ð 6�Lr5   r™  z(batch_size, num_choices, sequence_lengthc            	       ó„   — e Zd ZU eed<   ej                  Zej                  ed<   dZe	ed<   d„ Z
	 	 	 	 dde	de	de	d	e	fd
„Zy)Ú'FlaxRobertaForTokenClassificationModuler8   r9   Frï   c                 ó–  — t        | j                  | j                  d| j                  ¬«      | _        | j                  j
                  �| j                  j
                  n| j                  j                  }t        j                  |¬«      | _	        t        j                  | j                  j                  | j                  ¬«      | _        y )NFrŠ  r?   ro   )ri  r8   r9   rï   r7  r0  rQ   rA   rP   rR   rs   r1  r‹  r3  s     r3   rU   z-FlaxRobertaForTokenClassificationModule.setupÏ  sŽ   € Ü(Ø—;‘;Ø—*‘*Ø#Ø#'×#>Ñ#>ô	
ˆŒð �{‰{×-Ñ-Ð9ð �K‰K×*Ò*à—‘×0Ñ0ð 	ô
 —z‘zÐ'9Ô:ˆŒÜŸ(™( 4§;¡;×#9Ñ#9ÀÇÁÔLˆ�r5   rV   rœ   rý   rþ   c
                 óâ   — | j                  |||||||||	¬«	      }
|
d   }| j                  ||¬«      }| j                  |«      }|	s	|f|
dd  z   S t        ||
j                  |
j
                  ¬«      S r�  )r7  rR   r‹  r   r^   r  )rT   r/   r[   rY   rZ   r	  rV   rœ   rý   rþ   r¼   r^   r�  s                r3   r_   z0FlaxRobertaForTokenClassificationModule.__call__Þ  s•   € ð —,‘,ØØØØØØ'Ø/Ø!5Ø#ð ó 

ˆð   ™
ˆØŸ™ ]À-˜ÓPˆØ—‘ Ó/ˆáØ�9˜w q r˜{Ñ*Ð*ä(ØØ!×/Ñ/Ø×)Ñ)ô
ð 	
r5   Nrƒ  r„  rh   r5   r3   r›  r›  Ê  sf   … ØÓØ—{‘{€Eˆ3�9‰9Ó"Ø#(Ð˜DÓ(òMð, #Ø"'Ø%*Ø ñ$
ð ð$
ð  ð$
ð #ð$
ð ô$
r5   r›  z¦
    Roberta Model with a token classification head on top (a linear layer on top of the hidden-states output) e.g. for
    Named-Entity-Recognition (NER) tasks.
    c                   ó   — e Zd ZeZy)Ú!FlaxRobertaForTokenClassificationN)ra   rb   rc   r›  r8  rh   r5   r3   rŸ  rŸ    s	   „ ð ;�Lr5   rŸ  c            	       ó„   — e Zd ZU eed<   ej                  Zej                  ed<   dZe	ed<   d„ Z
	 	 	 	 dde	de	de	d	e	fd
„Zy)Ú%FlaxRobertaForQuestionAnsweringModuler8   r9   Frï   c                 óÜ   — t        | j                  | j                  d| j                  ¬«      | _        t        j                  | j                  j                  | j                  ¬«      | _        y )NFrŠ  ro   )	ri  r8   r9   rï   r7  rA   rs   r1  Ú
qa_outputsrS   s    r3   rU   z+FlaxRobertaForQuestionAnsweringModule.setup  sJ   € Ü(Ø—;‘;Ø—*‘*Ø#Ø#'×#>Ñ#>ô	
ˆŒô Ÿ(™( 4§;¡;×#9Ñ#9ÀÇÁÔLˆ�r5   rV   rœ   rý   rþ   c
                 ób  — | j                  |||||||||	¬«	      }
|
d   }| j                  |«      }t        j                  || j                  j
                  d¬«      \  }}|j                  d«      }|j                  d«      }|	s
||f|
dd  z   S t        |||
j                  |
j                  ¬«      S )Nr}  r   r&   r'   r!   )Ústart_logitsÚ
end_logitsr^   r  )
r7  r£  r-   rL  r8   r1  Úsqueezer   r^   r  )rT   r/   r[   rY   rZ   r	  rV   rœ   rý   rþ   r¼   r^   r�  r¥  r¦  s                  r3   r_   z.FlaxRobertaForQuestionAnsweringModule.__call__'  sË   € ð —,‘,ØØØØØØ'Ø/Ø!5Ø#ð ó 

ˆð   ™
ˆà—‘ Ó/ˆÜ#&§9¡9¨V°T·[±[×5KÑ5KÐRTÔ#UÑ ˆ�jØ#×+Ñ+¨BÓ/ˆØ×'Ñ'¨Ó+ˆ
áØ  *Ð-°¸¸°Ñ;Ð;ä/Ø%Ø!Ø!×/Ñ/Ø×)Ñ)ô	
ð 	
r5   Nrƒ  r„  rh   r5   r3   r¡  r¡    sf   … ØÓØ—{‘{€Eˆ3�9‰9Ó"Ø#(Ð˜DÓ(òMð  #Ø"'Ø%*Ø ñ(
ð ð(
ð  ð(
ð #ð(
ð ô(
r5   r¡  zà
    Roberta Model with a span classification head on top for extractive question-answering tasks like SQuAD (a linear
    layers on top of the hidden-states output to compute `span start logits` and `span end logits`).
    c                   ó   — e Zd ZeZy)ÚFlaxRobertaForQuestionAnsweringN)ra   rb   rc   r¡  r8  rh   r5   r3   r©  r©  R  s	   „ ð 9�Lr5   r©  c                   ó
  — e Zd ZU eed<   ej                  Zej                  ed<   dZe	ed<   d„ Z
	 	 	 	 	 	 	 	 	 ddeej                     deej                     d	eej                     d
eej                     de	de	de	de	de	fd„Zy)ÚFlaxRobertaForCausalLMModuler8   r9   Frï   c                 ó´   — t        | j                  d| j                  | j                  ¬«      | _        t        | j                  | j                  ¬«      | _        y rw  ry  rS   s    r3   rU   z"FlaxRobertaForCausalLMModule.setupj  r{  r5   NrY   r	  rè   ré   r›   rV   rœ   rý   rþ   c                 óR  — | j                  |||||||||	|
||¬«      }|d   }| j                  j                  r#| j                   j                  d   d   d   d   }nd }| j	                  ||¬«      }|s	|f|dd  z   S t        ||j                  |j                  |j                  ¬	«      S )
N)rè   ré   r›   rV   rœ   rý   rþ   r   r&  rl  rI   r~  r  r!   )r�  r^   r  r  )	r7  r8   r‚  r¥   rz  r   r^   r  r  )rT   r/   r[   rZ   rY   r	  rè   ré   r›   rV   rœ   rý   rþ   r¼   r^   r+  r�  s                    r3   r_   z%FlaxRobertaForCausalLMModule.__call__s  sÕ   € ð  —,‘,ØØØØØØ"7Ø#9Ø!Ø'Ø/Ø!5Ø#ð ó 
ˆð   ™
ˆØ�;‰;×*Ò*Ø#Ÿ|™|×5Ñ5°hÑ?ÀÑMÐN_Ñ`ÐalÑmÑà#Ðð —‘˜mÐ>N�ÓOˆáØ�9˜w q r˜{Ñ*Ð*ä4ØØ!×/Ñ/Ø×)Ñ)Ø$×5Ñ5ô	
ð 	
r5   )	NNNNFTFFTr  rh   r5   r3   r«  r«  e  sË   … ØÓØ—{‘{€Eˆ3�9‰9Ó"Ø#(Ð˜DÓ(òOð 15Ø+/Ø7;Ø8<Ø Ø"Ø"'Ø%*Ø ñ0
ð
 ! §¡Ñ-ð0
ð ˜CŸK™KÑ(ð0
ð  (¨¯©Ñ4ð0
ð !)¨¯©Ñ 5ð0
ð ð0
ð ð0
ð  ð0
ð #ð0
ð ô0
r5   r«  z’
    Roberta Model with a language modeling head on top (a linear layer on top of the hidden-states output) e.g for
    autoregressive tasks.
    c                   ó>   — e Zd ZeZddeej                     fd„Zd„ Z	y)ÚFlaxRobertaForCausalLMNr[   c                 óH  — |j                   \  }}| j                  ||«      }t        j                  ||fd¬«      }|�-|j	                  d¬«      dz
  }t        j                  ||d«      }n4t        j                  t        j                  |d¬«      d d d …f   ||f«      }|||dœS )Nr$   ro   r&   r'   r!   )r   r   )r[  r[   rZ   )	r,   r›   r-   rw   r.   r   rŒ   r�   rŽ   )	rT   r/   r’   r[   r±   Ú
seq_lengthr[  Úextended_attention_maskrZ   s	            r3   Úprepare_inputs_for_generationz4FlaxRobertaForCausalLM.prepare_inputs_for_generation°  s±   € à!*§¡Ñˆ
�JàŸ/™/¨*°jÓAˆô #&§(¡(¨J¸
Ð+CÈ4Ô"PÐØÐ%Ø)×0Ñ0°bÐ0Ó9¸AÑ=ˆLÜ&)×&>Ñ&>Ð?VÐXfÐhnÓ&oÑ#ä×+Ñ+¬C¯J©J°zÈÔ,NÈtÒUVÈwÑ,WÐZdÐfpÐYqÓrˆLð  /Ø5Ø(ñ
ð 	
r5   c                 óL   — |j                   |d<   |d   d d …dd …f   dz   |d<   |S )Nr[  rZ   r&   r!   )r[  )rT   Úmodel_outputsÚmodel_kwargss      r3   Úupdate_inputs_for_generationz3FlaxRobertaForCausalLM.update_inputs_for_generationÅ  s8   € Ø*7×*GÑ*GˆÐ&Ñ'Ø'3°NÑ'CÂAÀrÁsÀFÑ'KÈaÑ'Oˆ�^Ñ$ØÐr5   rÙ   )
ra   rb   rc   r«  r8  r   rE   ÚArrayr³  r·  rh   r5   r3   r¯  r¯  ¦  s'   „ ð 0€Lñ
ÐS[Ð\_×\eÑ\eÑSfó 
ó*r5   r¯  )r¯  r†  r™  r©  r�  rŸ  rs  r6  )VÚtypingr   r   r   Ú
flax.linenÚlinenrA   rE   Ú	jax.numpyÚnumpyr-   r,  Úflax.core.frozen_dictr   r   r   r	   r
   r   Únn_partitioningÚflax.linen.attentionr   Úflax.traverse_utilr   r   r   Úmodeling_flax_outputsr   r   r   r   r   r   r   r   r   Úmodeling_flax_utilsr   r   r   r   Úutilsr   r   r    Úconfiguration_robertar"   Ú
get_loggerra   ÚloggerÚ_CHECKPOINT_FOR_DOCÚ_CONFIG_FOR_DOCrö   r4   ÚROBERTA_START_DOCSTRINGrd  rb  r7   rj   rÁ   rË   rÒ   rÜ   rá   rî   r  r  r  r.  r6  ri  rs  ru  r†  rˆ  r�  r’  r™  re  r›  rŸ  r¡  r©  r«  r¯  Ú__all__rh   r5   r3   ú<module>rÌ     sÆ  ð÷ -Ñ ,å Û 
Ý Û ß >Ñ >ß 6Ý 6Ý >ß ;Ý ÷
÷ 
õ 
÷ wÓ vß YÑ YÝ 0ð 
ˆ×	Ñ	˜HÓ	%€à/Ð Ø!€à×Ñ€ò:ð0Ð ð.#Ð ôN(˜BŸI™Iô (ôXh˜rŸy™yô hôX˜BŸI™Iô ô*'˜2Ÿ9™9ô 'ôV˜bŸi™iô ô&˜Ÿ	™	ô ô*6�r—y‘yô 6ôtO
 §¡ô O
ôf$
˜Ÿ™ô $
ôP)˜Ÿ	™	ô )ô" ˜Ÿ	™	ô  ôF B§I¡Iô ô@}Ð!4ô }ôBD
˜Ÿ	™	ô D
ñN ØgØóô%Ð1ó %ó	ð%ñ Ð-Ð/BÐDbÐdsÔ tô7
 2§9¡9ô 7
ñt ÐQÐSjÓkô0Ð7ó 0ó lð0ñ ØØØ"ØØ	õô1
°·±ô 1
ñh ðð óô>Ð+Eó >óð>ñ Ø(ØØ Øô	ô:
¨¯©ô :
ñz ðð óô6Ð#=ó 6óð6ñ Ø Ð":×"AÑ"AÐBlÓ"môñ Ø ØØ!Øô	ô8
¨b¯i©iô 8
ñv ðð óô;Ð(Bó ;óð;ñ Ø%ØØØô	ô6
¨B¯I©Iô 6
ñr ðð óô9Ð&@ó 9óð9ñ Ø#ØØ$Øô	ô>
 2§9¡9ô >
ñB ðð óôÐ7ó óðñ< ØØØ)Øô	ò	�r5   