Ë
    S^(hH   ã                   ól  — d dl mZmZmZ d dlZd dlmZ d dlZd dl	m
Z d dl
Zd dlmZmZmZ d dlmZ d dlmZmZ d dlmZ ddlmZmZmZmZmZmZmZ dd	lm Z m!Z!m"Z"m#Z#m$Z$ dd
l%m&Z&m'Z'm(Z(m)Z) ddl*m+Z+  e)jX                  e-«      Z.dZ/dZ0ejb                  jd                   G d„ de&«      «       Z3dZ4dZ5 G d„ dejl                  «      Z7 G d„ dejl                  «      Z8 G d„ dejl                  «      Z9 G d„ dejl                  «      Z: G d„ dejl                  «      Z; G d„ dejl                  «      Z< G d„ d ejl                  «      Z= G d!„ d"ejl                  «      Z> G d#„ d$ejl                  «      Z? G d%„ d&e!«      Z@ G d'„ d(ejl                  «      ZA e'd)e4«       G d*„ d+e@«      «       ZB e"eBe/ee0«        G d,„ d-ejl                  «      ZC e'd.e4«       G d/„ d0e@«      «       ZDd1ZE e$eDe5j�                  d2«      eEz   «        e#eDe3e0¬3«        G d4„ d5ejl                  «      ZG e'd6e4«       G d7„ d8e@«      «       ZH e"eHe/ee0d9¬:«        G d;„ d<ejl                  «      ZI e'd=e4«       G d>„ d?e@«      «       ZJ e"eJe/ee0«        G d@„ dAejl                  «      ZK e'dBe4«       G dC„ dDe@«      «       ZL e$eLe5j�                  dE«      «        e"eLe/ee0«        G dF„ dGejl                  «      ZM e'dHe4«       G dI„ dJe@«      «       ZN e"eNe/ee0«        G dK„ dLejl                  «      ZO e'dMe4«       G dN„ dOe@«      «       ZP e"ePe/ee0«       g dP¢ZQy)Qé    )ÚCallableÚOptionalÚTupleN)Ú
FrozenDictÚfreezeÚunfreeze)Údot_product_attention_weights)Úflatten_dictÚunflatten_dict)Úlaxé   )ÚFlaxBaseModelOutputÚFlaxBaseModelOutputWithPoolingÚFlaxMaskedLMOutputÚFlaxMultipleChoiceModelOutputÚ FlaxQuestionAnsweringModelOutputÚFlaxSequenceClassifierOutputÚFlaxTokenClassifierOutput)ÚACT2FNÚFlaxPreTrainedModelÚappend_call_sample_docstringÚ append_replace_return_docstringsÚoverwrite_call_docstring)ÚModelOutputÚadd_start_docstringsÚ%add_start_docstrings_to_model_forwardÚloggingé   )ÚAlbertConfigzalbert/albert-base-v2r   c                   ó²   — e Zd ZU dZdZej                  ed<   dZej                  ed<   dZ	e
eej                        ed<   dZe
eej                        ed<   y)ÚFlaxAlbertForPreTrainingOutputaB  
    Output type of [`FlaxAlbertForPreTraining`].

    Args:
        prediction_logits (`jnp.ndarray` of shape `(batch_size, sequence_length, config.vocab_size)`):
            Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
        sop_logits (`jnp.ndarray` of shape `(batch_size, 2)`):
            Prediction scores of the next sequence prediction (classification) head (scores of True/False continuation
            before SoftMax).
        hidden_states (`tuple(jnp.ndarray)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
            Tuple of `jnp.ndarray` (one for the output of the embeddings + one for the output of each layer) of shape
            `(batch_size, sequence_length, hidden_size)`.

            Hidden-states of the model at the output of each layer plus the initial embedding outputs.
        attentions (`tuple(jnp.ndarray)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
            Tuple of `jnp.ndarray` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
            sequence_length)`.

            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
            heads.
    NÚprediction_logitsÚ
sop_logitsÚhidden_statesÚ
attentions)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r"   ÚjnpÚndarrayÚ__annotations__r#   r$   r   r   r%   © ó    úm/var/www/skyplay_api_hub/venv/lib/python3.12/site-packages/transformers/models/albert/modeling_flax_albert.pyr!   r!   6   sV   … ñð, &*Ð�s—{‘{Ó)Ø"€J�—‘Ó"Ø26€M�8˜E #§+¡+Ñ.Ñ/Ó6Ø/3€J�˜˜sŸ{™{Ñ+Ñ,Ô3r.   r!   aþ  

    This model inherits from [`FlaxPreTrainedModel`]. Check the superclass documentation for the generic methods the
    library implements for all its model (such as downloading, saving and converting weights from PyTorch models)

    This model is also a
    [flax.linen.Module](https://flax.readthedocs.io/en/latest/api_reference/flax.linen/module.html) subclass. Use it as
    a regular Flax linen Module and refer to the Flax documentation for all matter related to general usage and
    behavior.

    Finally, this model supports inherent JAX features such as:

    - [Just-In-Time (JIT) compilation](https://jax.readthedocs.io/en/latest/jax.html#just-in-time-compilation-jit)
    - [Automatic Differentiation](https://jax.readthedocs.io/en/latest/jax.html#automatic-differentiation)
    - [Vectorization](https://jax.readthedocs.io/en/latest/jax.html#vectorization-vmap)
    - [Parallelization](https://jax.readthedocs.io/en/latest/jax.html#parallelization-pmap)

    Parameters:
        config ([`AlbertConfig`]): Model configuration class with all the parameters of the model.
            Initializing with a config file does not load the weights associated with the model, only the
            configuration. Check out the [`~FlaxPreTrainedModel.from_pretrained`] method to load the model weights.
        dtype (`jax.numpy.dtype`, *optional*, defaults to `jax.numpy.float32`):
            The data type of the computation. Can be one of `jax.numpy.float32`, `jax.numpy.float16` (on GPUs) and
            `jax.numpy.bfloat16` (on TPUs).

            This can be used to enable mixed-precision training or half-precision inference on GPUs or TPUs. If
            specified all the computation will be performed with the given `dtype`.

            **Note that this only specifies the dtype of the computation and does not influence the dtype of model
            parameters.**

            If you wish to change the dtype of the model parameters, see [`~FlaxPreTrainedModel.to_fp16`] and
            [`~FlaxPreTrainedModel.to_bf16`].
a¡  
    Args:
        input_ids (`numpy.ndarray` of shape `({0})`):
            Indices of input sequence tokens in the vocabulary.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are input IDs?](../glossary#input-ids)
        attention_mask (`numpy.ndarray` of shape `({0})`, *optional*):
            Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:

            - 1 for tokens that are **not masked**,
            - 0 for tokens that are **masked**.

            [What are attention masks?](../glossary#attention-mask)
        token_type_ids (`numpy.ndarray` of shape `({0})`, *optional*):
            Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
            1]`:

            - 0 corresponds to a *sentence A* token,
            - 1 corresponds to a *sentence B* token.

            [What are token type IDs?](../glossary#token-type-ids)
        position_ids (`numpy.ndarray` of shape `({0})`, *optional*):
            Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
            config.max_position_embeddings - 1]`.
        return_dict (`bool`, *optional*):
            Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.

c                   óf   — e Zd ZU dZeed<   ej                  Zej                  ed<   d„ Z	dde
fd„Zy)	ÚFlaxAlbertEmbeddingszGConstruct the embeddings from word, position and token_type embeddings.ÚconfigÚdtypec                 óÜ  — t        j                  | j                  j                  | j                  j                  t
        j                   j                  j                  | j                  j                  ¬«      ¬«      | _	        t        j                  | j                  j                  | j                  j                  t
        j                   j                  j                  | j                  j                  ¬«      ¬«      | _        t        j                  | j                  j                  | j                  j                  t
        j                   j                  j                  | j                  j                  ¬«      ¬«      | _        t        j                  | j                  j                  | j                   ¬«      | _        t        j"                  | j                  j$                  ¬«      | _        y )N)Ústddev)Úembedding_init©Úepsilonr3   ©Úrate)ÚnnÚEmbedr2   Ú
vocab_sizeÚembedding_sizeÚjaxÚinitializersÚnormalÚinitializer_rangeÚword_embeddingsÚmax_position_embeddingsÚposition_embeddingsÚtype_vocab_sizeÚtoken_type_embeddingsÚ	LayerNormÚlayer_norm_epsr3   ÚDropoutÚhidden_dropout_probÚdropout©Úselfs    r/   ÚsetupzFlaxAlbertEmbeddings.setupž   s5  € Ü!Ÿx™xØ�K‰K×"Ñ"Ø�K‰K×&Ñ&ÜŸ6™6×.Ñ.×5Ñ5¸T¿[¹[×=ZÑ=ZÐ5Ó[ô 
ˆÔô
 $&§8¡8Ø�K‰K×/Ñ/Ø�K‰K×&Ñ&ÜŸ6™6×.Ñ.×5Ñ5¸T¿[¹[×=ZÑ=ZÐ5Ó[ô$
ˆÔ ô
 &(§X¡XØ�K‰K×'Ñ'Ø�K‰K×&Ñ&ÜŸ6™6×.Ñ.×5Ñ5¸T¿[¹[×=ZÑ=ZÐ5Ó[ô&
ˆÔ"ô
 Ÿ™¨d¯k©k×.HÑ.HÐPT×PZÑPZÔ[ˆŒÜ—z‘z t§{¡{×'FÑ'FÔGˆ�r.   Údeterministicc                 ó  — | j                  |j                  d«      «      }| j                  |j                  d«      «      }| j                  |j                  d«      «      }||z   |z   }| j	                  |«      }| j                  ||¬«      }|S )NÚi4©rP   )rC   ÚastyperE   rG   rH   rL   )	rN   Ú	input_idsÚtoken_type_idsÚposition_idsrP   Úinputs_embedsÚposition_embedsrG   r$   s	            r/   Ú__call__zFlaxAlbertEmbeddings.__call__±   s�   € à×,Ñ,¨Y×-=Ñ-=¸dÓ-CÓDˆØ×2Ñ2°<×3FÑ3FÀtÓ3LÓMˆØ $× :Ñ :¸>×;PÑ;PÐQUÓ;VÓ WÐð &Ð(=Ñ=ÀÑOˆð Ÿ™ }Ó5ˆØŸ™ ]À-˜ÓPˆØÐr.   N©T)r&   r'   r(   r)   r   r,   r*   Úfloat32r3   rO   ÚboolrZ   r-   r.   r/   r1   r1   ˜   s/   … ÙQàÓØ—{‘{€Eˆ3�9‰9Ó"òHñ&Ètô r.   r1   c                   ób   — e Zd ZU eed<   ej                  Zej                  ed<   d„ Zdde	fd„Z
y)ÚFlaxAlbertSelfAttentionr2   r3   c                 óü  — | j                   j                  | j                   j                  z  dk7  rt        d«      ‚t	        j
                  | j                   j                  | j                  t        j                  j                  j                  | j                   j                  «      ¬«      | _        t	        j
                  | j                   j                  | j                  t        j                  j                  j                  | j                   j                  «      ¬«      | _        t	        j
                  | j                   j                  | j                  t        j                  j                  j                  | j                   j                  «      ¬«      | _        t	        j
                  | j                   j                  t        j                  j                  j                  | j                   j                  «      | j                  ¬«      | _        t	        j                  | j                   j                   | j                  ¬«      | _        t	        j"                  | j                   j$                  ¬«      | _        y )Nr   z›`config.hidden_size`: {self.config.hidden_size} has to be a multiple of `config.num_attention_heads`                    : {self.config.num_attention_heads})r3   Úkernel_init©ra   r3   r7   r9   )r2   Úhidden_sizeÚnum_attention_headsÚ
ValueErrorr;   ÚDenser3   r?   r@   rA   rB   ÚqueryÚkeyÚvalueÚdenserH   rI   rJ   rK   rL   rM   s    r/   rO   zFlaxAlbertSelfAttention.setupÄ   s•  € Ø�;‰;×"Ñ" T§[¡[×%DÑ%DÑDÈÒIÜðIóð ô
 —X‘XØ�K‰K×#Ñ#Ø—*‘*ÜŸ™×+Ñ+×2Ñ2°4·;±;×3PÑ3PÓQô
ˆŒ
ô
 —8‘8Ø�K‰K×#Ñ#Ø—*‘*ÜŸ™×+Ñ+×2Ñ2°4·;±;×3PÑ3PÓQô
ˆŒô
 —X‘XØ�K‰K×#Ñ#Ø—*‘*ÜŸ™×+Ñ+×2Ñ2°4·;±;×3PÑ3PÓQô
ˆŒ
ô
 —X‘XØ�K‰K×#Ñ#ÜŸ™×+Ñ+×2Ñ2°4·;±;×3PÑ3PÓQØ—*‘*ô
ˆŒ
ô
 Ÿ™¨d¯k©k×.HÑ.HÐPT×PZÑPZÔ[ˆŒÜ—z‘z t§{¡{×'FÑ'FÔGˆ�r.   Úoutput_attentionsc                 óP  — | j                   j                  | j                   j                  z  }| j                  |«      j	                  |j
                  d d | j                   j                  |fz   «      }| j                  |«      j	                  |j
                  d d | j                   j                  |fz   «      }| j                  |«      j	                  |j
                  d d | j                   j                  |fz   «      }|�Çt        j                  |d¬«      }t        j                  |dkD  t        j                  |j
                  d«      j                  | j                  «      t        j                  |j
                  t        j                  | j                  «      j                   «      j                  | j                  «      «      }	nd }	d }
|s*| j                   j"                  dkD  r| j%                  d«      }
t'        |||	|
| j                   j"                  d|| j                  d ¬«	      }t        j(                  d	||«      }|j	                  |j
                  d d d
z   «      }| j+                  |«      }| j-                  ||¬«      }| j/                  ||z   «      }|r||f}|S |f}|S )Né   )éýÿÿÿéþÿÿÿ©Úaxisr   g        rL   T)ÚbiasÚdropout_rngÚdropout_rateÚbroadcast_dropoutrP   r3   Ú	precisionz...hqk,...khd->...qhd)éÿÿÿÿrS   )r2   rc   rd   rg   ÚreshapeÚshaperi   rh   r*   Úexpand_dimsr   ÚselectÚfullrT   r3   ÚfinfoÚminÚattention_probs_dropout_probÚmake_rngr	   Úeinsumrj   rL   rH   )rN   r$   Úattention_maskrP   rk   Úhead_dimÚquery_statesÚvalue_statesÚ
key_statesÚattention_biasrs   Úattn_weightsÚattn_outputÚprojected_attn_outputÚlayernormed_attn_outputÚoutputss                   r/   rZ   z FlaxAlbertSelfAttention.__call__â   sY  € Ø—;‘;×*Ñ*¨d¯k©k×.MÑ.MÑMˆà—z‘z -Ó0×8Ñ8Ø×Ñ  Ð# t§{¡{×'FÑ'FÈÐ&QÑQó
ˆð —z‘z -Ó0×8Ñ8Ø×Ñ  Ð# t§{¡{×'FÑ'FÈÐ&QÑQó
ˆð —X‘X˜mÓ,×4Ñ4Ø×Ñ  Ð# t§{¡{×'FÑ'FÈÐ&QÑQó
ˆ
ð
 Ð%ä Ÿ_™_¨^À(ÔKˆNÜ ŸZ™ZØ Ñ"Ü—‘˜×-Ñ-¨sÓ3×:Ñ:¸4¿:¹:ÓFÜ—‘˜×-Ñ-¬s¯y©y¸¿¹Ó/D×/HÑ/HÓI×PÑPÐQU×Q[ÑQ[Ó\ó‰Nð "ˆNàˆÙ §¡×!IÑ!IÈCÒ!OØŸ-™-¨	Ó2ˆKä4ØØØØ#ØŸ™×AÑAØ"Ø'Ø—*‘*Øô

ˆô —j‘jÐ!8¸,ÈÓUˆØ!×)Ñ)¨+×*;Ñ*;¸B¸QÐ*?À%Ñ*GÓHˆà $§
¡
¨;Ó 7ÐØ $§¡Ð-BÐR_ Ó `ÐØ"&§.¡.Ð1FÈÑ1VÓ"WÐÙ=NÐ*¨LÐ9ˆØˆð VmÐTnˆØˆr.   N©TF©r&   r'   r(   r   r,   r*   r\   r3   rO   r]   rZ   r-   r.   r/   r_   r_   À   s-   … ØÓØ—{‘{€Eˆ3�9‰9Ó"òHñ<0Ð]aô 0r.   r_   c                   ój   — e Zd ZU eed<   ej                  Zej                  ed<   d„ Z	 	 dde	de	fd„Z
y)	ÚFlaxAlbertLayerr2   r3   c                 ó>  — t        | j                  | j                  ¬«      | _        t	        j
                  | j                  j                  t        j                  j                  j                  | j                  j                  «      | j                  ¬«      | _        t        | j                  j                     | _        t	        j
                  | j                  j                  t        j                  j                  j                  | j                  j                  «      | j                  ¬«      | _        t	        j"                  | j                  j$                  | j                  ¬«      | _        t	        j(                  | j                  j*                  ¬«      | _        y )N©r3   rb   r7   r9   )r_   r2   r3   Ú	attentionr;   rf   Úintermediate_sizer?   r@   rA   rB   Úffnr   Ú
hidden_actÚ
activationrc   Ú
ffn_outputrH   rI   Úfull_layer_layer_normrJ   rK   rL   rM   s    r/   rO   zFlaxAlbertLayer.setup  sü   € Ü0°·±ÀDÇJÁJÔOˆŒÜ—8‘8Ø�K‰K×)Ñ)ÜŸ™×+Ñ+×2Ñ2°4·;±;×3PÑ3PÓQØ—*‘*ô
ˆŒô
 ! §¡×!7Ñ!7Ñ8ˆŒÜŸ(™(Ø�K‰K×#Ñ#ÜŸ™×+Ñ+×2Ñ2°4·;±;×3PÑ3PÓQØ—*‘*ô
ˆŒô
 &(§\¡\¸$¿+¹+×:TÑ:TÐ\`×\fÑ\fÔ%gˆÔ"Ü—z‘z t§{¡{×'FÑ'FÔGˆ�r.   rP   rk   c                 ó
  — | j                  ||||¬«      }|d   }| j                  |«      }| j                  |«      }| j                  |«      }| j	                  ||¬«      }| j                  ||z   «      }|f}|r	||d   fz  }|S )N©rP   rk   r   rS   r   )r“   r•   r—   r˜   rL   r™   )	rN   r$   r‚   rP   rk   Úattention_outputsÚattention_outputr˜   rŒ   s	            r/   rZ   zFlaxAlbertLayer.__call__)  s¥   € ð !ŸN™NØ˜>¸ÐZkð +ó 
Ðð -¨QÑ/ÐØ—X‘XÐ.Ó/ˆ
Ø—_‘_ ZÓ0ˆ
Ø—_‘_ ZÓ0ˆ
Ø—\‘\ *¸M�\ÓJˆ
Ø×2Ñ2°:Ð@PÑ3PÓQˆà Ð"ˆáØÐ)¨!Ñ,Ð.Ñ.ˆGØˆr.   Nr�   rŽ   r-   r.   r/   r�   r�     sA   … ØÓØ—{‘{€Eˆ3�9‰9Ó"òHð( #Ø"'ñð ð	ð
  ôr.   r�   c                   óp   — e Zd ZU eed<   ej                  Zej                  ed<   d„ Z	 	 	 d	de	de	de	fd„Z
y)
ÚFlaxAlbertLayerCollectionr2   r3   c           	      óÄ   — t        | j                  j                  «      D �cg c]-  }t        | j                  t	        |«      | j
                  ¬«      ‘Œ/ c}| _        y c c}w )N)Únamer3   )Úranger2   Úinner_group_numr�   Ústrr3   Úlayers©rN   Úis     r/   rO   zFlaxAlbertLayerCollection.setupE  sE   € äQVÐW[×WbÑWb×WrÑWrÓQsö
ØLMŒO˜DŸK™K¬c°!«f¸D¿J¹JÖGò
ˆ�ùò 
s   ¢2ArP   rk   Úoutput_hidden_statesc                 ó¸   — d}d}t        | j                  «      D ]*  \  }}	 |	||||¬«      }
|
d   }|r	||
d   fz   }|sŒ%||fz   }Œ, |f}|r||fz   }|r||fz   }|S )Nr-   r›   r   r   )Ú	enumerater¥   )rN   r$   r‚   rP   rk   r¨   Úlayer_hidden_statesÚlayer_attentionsÚlayer_indexÚalbert_layerÚlayer_outputrŒ   s               r/   rZ   z"FlaxAlbertLayerCollection.__call__J  s§   € ð !ÐØÐä)2°4·;±;Ó)?ò 	MÑ%ˆK˜Ù'ØØØ+Ø"3ô	ˆLð )¨™OˆMá Ø#3°|ÀA±Ð6HÑ#HÐ â#Ø&9¸]Ð<LÑ&LÑ#ð	Mð !Ð"ˆÙØÐ!4Ð 6Ñ6ˆGÙØÐ!1Ð 3Ñ3ˆGØˆr.   N©TFFrŽ   r-   r.   r/   rŸ   rŸ   A  sM   … ØÓØ—{‘{€Eˆ3�9‰9Ó"ò
ð #Ø"'Ø%*ñð ð	ð
  ðð #ôr.   rŸ   c                   ó„   — e Zd ZU eed<   ej                  Zej                  ed<   dZe	e
   ed<   d„ Z	 	 	 d
dededefd	„Zy)ÚFlaxAlbertLayerCollectionsr2   r3   Nr­   c                 óP   — t        | j                  | j                  ¬«      | _        y )Nr’   )rŸ   r2   r3   Úalbert_layersrM   s    r/   rO   z FlaxAlbertLayerCollections.setupq  s   € Ü6°t·{±{È$Ï*É*ÔUˆÕr.   rP   rk   r¨   c                 ó2   — | j                  |||||¬«      }|S ©N©rP   rk   r¨   )r´   )rN   r$   r‚   rP   rk   r¨   rŒ   s          r/   rZ   z#FlaxAlbertLayerCollections.__call__t  s/   € ð ×$Ñ$ØØØ'Ø/Ø!5ð %ó 
ˆð ˆr.   r°   )r&   r'   r(   r   r,   r*   r\   r3   r­   r   r¤   rO   r]   rZ   r-   r.   r/   r²   r²   l  s\   … ØÓØ—{‘{€Eˆ3�9‰9Ó"Ø!%€K�˜#‘Ó%òVð #Ø"'Ø%*ñð ð	ð
  ðð #ôr.   r²   c            	       óv   — e Zd ZU eed<   ej                  Zej                  ed<   d„ Z	 	 	 	 d
de	de	de	de	fd„Z
y	)ÚFlaxAlbertLayerGroupsr2   r3   c           
      óØ   — t        | j                  j                  «      D �cg c]7  }t        | j                  t	        |«      t	        |«      | j
                  ¬«      ‘Œ9 c}| _        y c c}w )N)r¡   r­   r3   )r¢   r2   Únum_hidden_groupsr²   r¤   r3   r¥   r¦   s     r/   rO   zFlaxAlbertLayerGroups.setupŠ  sQ   € ô ˜4Ÿ;™;×8Ñ8Ó9ö
àô ' t§{¡{¼¸Q»ÌSÐQRËVÐ[_×[eÑ[eÖfò
ˆ�ùò 
s   ¢<A'rP   rk   r¨   Úreturn_dictc                 ó„  — |rdnd }|r|fnd }t        | j                  j                  «      D ]m  }	t        |	| j                  j                  | j                  j                  z  z  «      }
 | j
                  |
   |||||¬«      }|d   }|r||d   z   }|sŒh||fz   }Œo |st        d„ |||fD «       «      S t        |||¬«      S )Nr-   r·   r   rw   c              3   ó&   K  — | ]	  }|€Œ|–— Œ y ­w©Nr-   )Ú.0Úvs     r/   ú	<genexpr>z1FlaxAlbertLayerGroups.__call__.<locals>.<genexpr>¯  s   è ø€ Òh˜qÐZ[ÑZgœÑhùs   ‚Š)Úlast_hidden_stater$   r%   )r¢   r2   Únum_hidden_layersÚintr»   r¥   Útupler   )rN   r$   r‚   rP   rk   r¨   r¼   Úall_attentionsÚall_hidden_statesr§   Ú	group_idxÚlayer_group_outputs               r/   rZ   zFlaxAlbertLayerGroups.__call__�  së   € ñ  1™°dˆÙ0D˜]Ñ,È$Ðä�t—{‘{×4Ñ4Ó5ò 	IˆAä˜A §¡×!>Ñ!>ÀÇÁ×A^ÑA^Ñ!^Ñ_Ó`ˆIØ!7 §¡¨YÑ!7ØØØ+Ø"3Ø%9ô"Ðð /¨qÑ1ˆMá Ø!/Ð2DÀRÑ2HÑ!H�â#Ø$5¸Ð8HÑ$HÑ!ð!	Iñ$ ÜÑh ]Ð4EÀ~Ð$VÔhÓhÐhÜ"Ø+Ð;LÐYgô
ð 	
r.   N©TFFTrŽ   r-   r.   r/   r¹   r¹   †  sZ   … ØÓØ—{‘{€Eˆ3�9‰9Ó"ò
ð #Ø"'Ø%*Ø ñ"
ð ð	"
ð
  ð"
ð #ð"
ð ô"
r.   r¹   c            	       óv   — e Zd ZU eed<   ej                  Zej                  ed<   d„ Z	 	 	 	 d
de	de	de	de	fd„Z
y	)ÚFlaxAlbertEncoderr2   r3   c                 ó<  — t        j                  | j                  j                  t        j                   j
                  j                  | j                  j                  «      | j                  ¬«      | _	        t        | j                  | j                  ¬«      | _        y )Nrb   r’   )r;   rf   r2   rc   r?   r@   rA   rB   r3   Úembedding_hidden_mapping_inr¹   Úalbert_layer_groupsrM   s    r/   rO   zFlaxAlbertEncoder.setup¹  sb   € Ü+-¯8©8Ø�K‰K×#Ñ#ÜŸ™×+Ñ+×2Ñ2°4·;±;×3PÑ3PÓQØ—*‘*ô,
ˆÔ(ô
 $9¸¿¹ÈDÏJÉJÔ#WˆÕ r.   rP   rk   r¨   r¼   c                 óP   — | j                  |«      }| j                  |||||¬«      S r¶   )rÏ   rÐ   )rN   r$   r‚   rP   rk   r¨   r¼   s          r/   rZ   zFlaxAlbertEncoder.__call__Á  s;   € ð ×8Ñ8¸ÓGˆØ×'Ñ'ØØØ'Ø/Ø!5ð (ó 
ð 	
r.   NrË   rŽ   r-   r.   r/   rÍ   rÍ   µ  s[   … ØÓØ—{‘{€Eˆ3�9‰9Ó"òXð #Ø"'Ø%*Ø ñ
ð ð	
ð
  ð
ð #ð
ð ô
r.   rÍ   c                   óÄ   — e Zd ZU eed<   ej                  Zej                  ed<   ej                  j                  j                  Zedej                  f   ed<   d„ Zdd„Zy)	ÚFlaxAlbertOnlyMLMHeadr2   r3   .Ú	bias_initc                 ó  — t        j                  | j                  j                  | j                  ¬«      | _        t        | j                  j                     | _        t        j                  | j                  j                  | j                  ¬«      | _	        t        j                  | j                  j                  | j                  d¬«      | _        | j                  d| j                  | j                  j                  f«      | _        y )Nr’   r7   F)r3   Úuse_biasrr   )r;   rf   r2   r>   r3   rj   r   r–   r—   rH   rI   r=   ÚdecoderÚparamrÔ   rr   rM   s    r/   rO   zFlaxAlbertOnlyMLMHead.setupÙ  s£   € Ü—X‘X˜dŸk™k×8Ñ8ÀÇ
Á
ÔKˆŒ
Ü  §¡×!7Ñ!7Ñ8ˆŒÜŸ™¨d¯k©k×.HÑ.HÐPT×PZÑPZÔ[ˆŒÜ—x‘x §¡× 6Ñ 6¸d¿j¹jÐSXÔYˆŒØ—J‘J˜v t§~¡~¸¿¹×8NÑ8NÐ7PÓQˆ�	r.   Nc                 ó  — | j                  |«      }| j                  |«      }| j                  |«      }|�+| j                  j	                  dd|j
                  ii|«      }n| j                  |«      }|| j                  z  }|S )NÚparamsÚkernel)rj   r—   rH   r×   ÚapplyÚTrr   )rN   r$   Úshared_embeddings      r/   rZ   zFlaxAlbertOnlyMLMHead.__call__à  s|   € ØŸ
™
 =Ó1ˆØŸ™¨Ó6ˆØŸ™ }Ó5ˆàÐ'Ø ŸL™L×.Ñ.°¸8ÐEU×EWÑEWÐ:XÐ/YÐ[hÓi‰Mà ŸL™L¨Ó7ˆMà˜Ÿ™Ñ"ˆØÐr.   r¿   )r&   r'   r(   r   r,   r*   r\   r3   r?   r;   r@   ÚzerosrÔ   r   Únpr+   rO   rZ   r-   r.   r/   rÓ   rÓ   Ô  sL   … ØÓØ—{‘{€Eˆ3�9‰9Ó"Ø+.¯6©6×+>Ñ+>×+DÑ+D€Iˆx˜˜RŸZ™Z˜Ñ(ÓDòRôr.   rÓ   c                   ó\   — e Zd ZU eed<   ej                  Zej                  ed<   d„ Zdd„Z	y)ÚFlaxAlbertSOPHeadr2   r3   c                 ó¬   — t        j                  | j                  j                  «      | _        t        j
                  d| j                  ¬«      | _        y )Nrm   r’   )r;   rJ   r2   Úclassifier_dropout_probrL   rf   r3   Ú
classifierrM   s    r/   rO   zFlaxAlbertSOPHead.setupò  s2   € Ü—z‘z $§+¡+×"EÑ"EÓFˆŒÜŸ(™( 1¨D¯J©JÔ7ˆ�r.   c                 óN   — | j                  ||¬«      }| j                  |«      }|S )NrS   )rL   rå   )rN   Úpooled_outputrP   Úlogitss       r/   rZ   zFlaxAlbertSOPHead.__call__ö  s'   € ØŸ™ ]À-˜ÓPˆØ—‘ Ó/ˆØˆr.   Nr[   )
r&   r'   r(   r   r,   r*   r\   r3   rO   rZ   r-   r.   r/   râ   râ   î  s$   … ØÓØ—{‘{€Eˆ3�9‰9Ó"ò8ôr.   râ   c                   óz  ‡ — e Zd ZU dZeZdZdZej                  e
d<   ddej                  dfded	ed
edej                  def
ˆ fd„Zddej&                  j(                  d	ededefd„Z eej3                  d«      «      	 	 	 	 	 	 	 	 	 ddedej&                  j(                  dedee   dee   dee   fd„«       Zˆ xZS )ÚFlaxAlbertPreTrainedModelz†
    An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained
    models.
    ÚalbertNÚmodule_class)r   r   r   Tr2   Úinput_shapeÚseedr3   Ú_do_initc                 óZ   •—  | j                   d||dœ|¤Ž}t        ‰| �	  ||||||¬«       y )N©r2   r3   )rí   rî   r3   rï   r-   )rì   ÚsuperÚ__init__)	rN   r2   rí   rî   r3   rï   ÚkwargsÚmoduleÚ	__class__s	           €r/   ró   z"FlaxAlbertPreTrainedModel.__init__  s=   ø€ ð #�×"Ñ"ÐH¨&¸ÑHÀÑHˆÜ‰Ñ˜ °[ÀtÐSXÐckÐÕlr.   ÚrngrÚ   Úreturnc                 ó|  — t        j                  |d¬«      }t        j                  |«      }t        j                  t        j                  t        j
                  |«      j                  d   «      |«      }t        j                  |«      }t        j                  j                  |«      \  }}	||	dœ}
| j                  j                  |
||||d¬«      d   }|�dt        t        |«      «      }t        t        |«      «      }| j                  D ]
  }||   ||<   Œ t!        «       | _        t#        t%        |«      «      S |S )NrR   r’   rw   )rÚ   rL   F)r¼   rÚ   )r*   rß   Ú
zeros_likeÚbroadcast_toÚarangeÚ
atleast_2dry   Ú	ones_liker?   ÚrandomÚsplitrõ   Úinitr
   r   Ú_missing_keysÚsetr   r   )rN   r÷   rí   rÚ   rU   rV   rW   r‚   Ú
params_rngrs   ÚrngsÚrandom_paramsÚmissing_keys                r/   Úinit_weightsz&FlaxAlbertPreTrainedModel.init_weights  s  € ä—I‘I˜k°Ô6ˆ	ÜŸ™¨	Ó2ˆÜ×'Ñ'¬¯
©
´3·>±>À)Ó3L×3RÑ3RÐSUÑ3VÓ(WÐYdÓeˆÜŸ™ yÓ1ˆä"%§*¡*×"2Ñ"2°3Ó"7Ñˆ
�KØ$°Ñ=ˆàŸ™×(Ñ(Ø�)˜^¨^¸\ÐW\ð )ó 
à
ñˆð ÐÜ(¬°-Ó)@ÓAˆMÜ!¤(¨6Ó"2Ó3ˆFØ#×1Ñ1ò A�Ø&3°KÑ&@��{Ò#ðAä!$£ˆDÔÜœ.¨Ó0Ó1Ð1à Ð r.   úbatch_size, sequence_lengthrs   Útrainrk   r¨   r¼   c                 óÊ  — |�|n| j                   j                  }|	�|	n| j                   j                  }	|
�|
n| j                   j                  }
|€t	        j
                  |«      }|€St	        j                  t	        j                  t	        j                  |«      j                  d   «      |j                  «      }|€t	        j                  |«      }i }|�||d<   | j                  j                  d|xs | j                  it	        j                  |d¬«      t	        j                  |d¬«      t	        j                  |d¬«      t	        j                  |d¬«      | ||	|
|¬«
      S )Nrw   rL   rÚ   rR   r’   )r  )r2   rk   r¨   r¼   r*   rú   rû   rü   rý   ry   rþ   rõ   rÜ   rÚ   Úarray)rN   rU   r‚   rV   rW   rÚ   rs   r
  rk   r¨   r¼   r  s               r/   rZ   z"FlaxAlbertPreTrainedModel.__call__*  sE  € ð 2CÐ1NÑ-ÐTX×T_ÑT_×TqÑTqÐà$8Ð$DÑ È$Ï+É+×JjÑJjð 	ð &1Ð%<‘kÀ$Ç+Á+×BYÑBYˆð Ð!Ü Ÿ^™^¨IÓ6ˆNàÐÜ×+Ñ+¬C¯J©J´s·~±~ÀiÓ7P×7VÑ7VÐWYÑ7ZÓ,[Ð]f×]lÑ]lÓmˆLàÐ!Ü Ÿ]™]¨9Ó5ˆNð ˆØÐ"Ø)ˆD�‰Oà�{‰{× Ñ Ø�vÒ, §¡Ð-Ü�I‰I�i tÔ,Ü�I‰I�n¨DÔ1Ü�I‰I�n¨DÔ1Ü�I‰I�l¨$Ô/ØˆIØØ ØØð !ó 
ð 	
r.   r¿   )	NNNNNFNNN)r&   r'   r(   r)   r   Úconfig_classÚbase_model_prefixrì   r;   ÚModuler,   r*   r\   r   rÅ   r3   r]   ró   r?   rÿ   ÚPRNGKeyr   r  r   ÚALBERT_INPUTS_DOCSTRINGÚformatÚdictr   rZ   Ú__classcell__)rö   s   @r/   rê   rê   ü  s7  ø… ñð
  €LØ ÐØ"€L�"—)‘)Ó"ð
 $ØØŸ;™;Øñ
màð
mð ð
mð ð	
mð
 �y‰yð
mð õ
mñ! §
¡
× 2Ñ 2ð !Àð !ÐPZð !Ðfpó !ñ0 +Ð+B×+IÑ+IÐJgÓ+hÓið ØØØØ*.ØØ,0Ø/3Ø&*ñ-
ð ð-
ð —Z‘Z×'Ñ'ð-
ð ð-
ð $ D™>ð-
ð ' t™nð-
ð ˜d‘^ò-
ó jô-
r.   rê   c                   óÄ   — e Zd ZU eed<   ej                  Zej                  ed<   dZe	ed<   d„ Z
	 	 	 	 	 	 ddeej                     deej                     d	e	d
e	de	de	fd„Zy)ÚFlaxAlbertModuler2   r3   TÚadd_pooling_layerc                 óê  — t        | j                  | j                  ¬«      | _        t	        | j                  | j                  ¬«      | _        | j                  r�t        j                  | j                  j                  t        j                  j                  j                  | j                  j                  «      | j                  d¬«      | _        t        j                  | _        y d | _        d | _        y )Nr’   Úpooler)ra   r3   r¡   )r1   r2   r3   Ú
embeddingsrÍ   Úencoderr  r;   rf   rc   r?   r@   rA   rB   r  ÚtanhÚpooler_activationrM   s    r/   rO   zFlaxAlbertModule.setup`  s�   € Ü.¨t¯{©{À$Ç*Á*ÔMˆŒÜ(¨¯©¸D¿J¹JÔGˆŒØ×!Ò!ÜŸ(™(Ø—‘×'Ñ'ÜŸF™F×/Ñ/×6Ñ6°t·{±{×7TÑ7TÓUØ—j‘jØô	ˆDŒKô &(§W¡WˆDÕ"àˆDŒKØ%)ˆDÕ"r.   NrV   rW   rP   rk   r¨   r¼   c	                 ó   — |€t        j                  |«      }|€St        j                  t        j                  t        j                  |«      j
                  d   «      |j
                  «      }| j                  ||||¬«      }	| j                  |	|||||¬«      }
|
d   }	| j                  r*| j                  |	d d …df   «      }| j                  |«      }nd }|s|€	|	f|
dd  z   S |	|f|
dd  z   S t        |	||
j                  |
j                  ¬«      S )Nrw   rS   ©rP   rk   r¨   r¼   r   r   )rÃ   Úpooler_outputr$   r%   )r*   rú   rû   rü   rý   ry   r  r  r  r  r  r   r$   r%   )rN   rU   r‚   rV   rW   rP   rk   r¨   r¼   r$   rŒ   Úpooleds               r/   rZ   zFlaxAlbertModule.__call__o  s*  € ð Ð!Ü Ÿ^™^¨IÓ6ˆNð ÐÜ×+Ñ+¬C¯J©J´s·~±~ÀiÓ7P×7VÑ7VÐWYÑ7ZÓ,[Ð]f×]lÑ]lÓmˆLàŸ™¨	°>À<Ð_l˜Ómˆà—,‘,ØØØ'Ø/Ø!5Ø#ð ó 
ˆð   ™
ˆØ×!Ò!Ø—[‘[ ªq°!¨tÑ!4Ó5ˆFØ×+Ñ+¨FÓ3‰FàˆFáàˆ~Ø%Ð'¨'°!°"¨+Ñ5Ð5Ø! 6Ð*¨W°Q°R¨[Ñ8Ð8ä-Ø+Ø Ø!×/Ñ/Ø×)Ñ)ô	
ð 	
r.   )NNTFFT)r&   r'   r(   r   r,   r*   r\   r3   r  r]   rO   r   rà   r+   rZ   r-   r.   r/   r  r  [  s‘   … ØÓØ—{‘{€Eˆ3�9‰9Ó"Ø"Ð�tÓ"ò*ð& 04Ø-1Ø"Ø"'Ø%*Ø ñ/
ð ! §¡Ñ,ð	/
ð
 ˜rŸz™zÑ*ð/
ð ð/
ð  ð/
ð #ð/
ð ô/
r.   r  z`The bare Albert Model transformer outputting raw hidden-states without any specific head on top.c                   ó   — e Zd ZeZy)ÚFlaxAlbertModelN)r&   r'   r(   r  rì   r-   r.   r/   r#  r#  ¡  s	   „ ð
 $�Lr.   r#  c            	       óv   — e Zd ZU eed<   ej                  Zej                  ed<   d„ Z	 	 	 	 d
de	de	de	de	fd„Z
y	)ÚFlaxAlbertForPreTrainingModuler2   r3   c                 óè   — t        | j                  | j                  ¬«      | _        t	        | j                  | j                  ¬«      | _        t        | j                  | j                  ¬«      | _        y )Nrñ   )r  r2   r3   rë   rÓ   Úpredictionsrâ   Úsop_classifierrM   s    r/   rO   z$FlaxAlbertForPreTrainingModule.setup°  sF   € Ü&¨d¯k©kÀÇÁÔLˆŒÜ0¸¿¹È4Ï:É:ÔVˆÔÜ/°t·{±{È$Ï*É*ÔUˆÕr.   rP   rk   r¨   r¼   c	           
      óh  — | j                  ||||||||¬«      }	| j                  j                  r#| j                   j                  d   d   d   d   }
nd }
|	d   }|	d   }| j	                  ||
¬«      }| j                  ||¬	«      }|s
||f|	d
d  z   S t        |||	j                  |	j                  ¬«      S )Nr  rÚ   r  rC   Ú	embeddingr   r   ©rÞ   rS   rm   )r"   r#   r$   r%   )	rë   r2   Útie_word_embeddingsÚ	variablesr'  r(  r!   r$   r%   )rN   rU   r‚   rV   rW   rP   rk   r¨   r¼   rŒ   rÞ   r$   rç   Úprediction_scoresÚ
sop_scoress                  r/   rZ   z'FlaxAlbertForPreTrainingModule.__call__µ  sæ   € ð —+‘+ØØØØØ'Ø/Ø!5Ø#ð ó 	
ˆð �;‰;×*Ò*Ø#Ÿ{™{×4Ñ4°XÑ>¸|ÑLÐM^Ñ_Ð`kÑlÑà#Ðà ™
ˆØ ™
ˆà ×,Ñ,¨]ÐM]Ð,Ó^ÐØ×(Ñ(¨ÀmÐ(ÓTˆ
áØ% zÐ2°W¸Q¸R°[Ñ@Ð@ä-Ø/Ø!Ø!×/Ñ/Ø×)Ñ)ô	
ð 	
r.   NrË   rŽ   r-   r.   r/   r%  r%  ¬  s[   … ØÓØ—{‘{€Eˆ3�9‰9Ó"òVð #Ø"'Ø%*Ø ñ*
ð ð*
ð  ð*
ð #ð*
ð ô*
r.   r%  z«
    Albert Model with two heads on top as done during the pretraining: a `masked language modeling` head and a
    `sentence order prediction (classification)` head.
    c                   ó   — e Zd ZeZy)ÚFlaxAlbertForPreTrainingN)r&   r'   r(   r%  rì   r-   r.   r/   r1  r1  â  s	   „ ð 2�Lr.   r1  aò  
    Returns:

    Example:

    ```python
    >>> from transformers import AutoTokenizer, FlaxAlbertForPreTraining

    >>> tokenizer = AutoTokenizer.from_pretrained("albert/albert-base-v2")
    >>> model = FlaxAlbertForPreTraining.from_pretrained("albert/albert-base-v2")

    >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="np")
    >>> outputs = model(**inputs)

    >>> prediction_logits = outputs.prediction_logits
    >>> seq_relationship_logits = outputs.sop_logits
    ```
r	  )Úoutput_typer  c            	       óv   — e Zd ZU eed<   ej                  Zej                  ed<   d„ Z	 	 	 	 d
de	de	de	de	fd„Z
y	)ÚFlaxAlbertForMaskedLMModuler2   r3   c                 óž   — t        | j                  d| j                  ¬«      | _        t	        | j                  | j                  ¬«      | _        y )NF)r2   r  r3   rñ   )r  r2   r3   rë   rÓ   r'  rM   s    r/   rO   z!FlaxAlbertForMaskedLMModule.setup  s4   € Ü&¨d¯k©kÈUÐZ^×ZdÑZdÔeˆŒÜ0¸¿¹È4Ï:É:ÔVˆÕr.   rP   rk   r¨   r¼   c	           
      ó4  — | j                  ||||||||¬«      }	|	d   }
| j                  j                  r#| j                   j                  d   d   d   d   }nd }| j	                  |
|¬«      }|s	|f|	dd  z   S t        ||	j                  |	j                  ¬	«      S )
Nr  r   rÚ   r  rC   r*  r+  r   ©rè   r$   r%   )rë   r2   r,  r-  r'  r   r$   r%   )rN   rU   r‚   rV   rW   rP   rk   r¨   r¼   rŒ   r$   rÞ   rè   s                r/   rZ   z$FlaxAlbertForMaskedLMModule.__call__  sÃ   € ð —+‘+ØØØØØ'Ø/Ø!5Ø#ð ó 	
ˆð   ™
ˆØ�;‰;×*Ò*Ø#Ÿ{™{×4Ñ4°XÑ>¸|ÑLÐM^Ñ_Ð`kÑlÑà#Ðð ×!Ñ! -ÐBRÐ!ÓSˆáØ�9˜w q r˜{Ñ*Ð*ä!ØØ!×/Ñ/Ø×)Ñ)ô
ð 	
r.   NrË   rŽ   r-   r.   r/   r4  r4  	  s[   … ØÓØ—{‘{€Eˆ3�9‰9Ó"òWð #Ø"'Ø%*Ø ñ'
ð ð'
ð  ð'
ð #ð'
ð ô'
r.   r4  z4Albert Model with a `language modeling` head on top.c                   ó   — e Zd ZeZy)ÚFlaxAlbertForMaskedLMN)r&   r'   r(   r4  rì   r-   r.   r/   r9  r9  ;  s   „ à.�Lr.   r9  z
refs/pr/11)Úrevisionc            	       óv   — e Zd ZU eed<   ej                  Zej                  ed<   d„ Z	 	 	 	 d
de	de	de	de	fd„Z
y	)Ú)FlaxAlbertForSequenceClassificationModuler2   r3   c                 ó~  — t        | j                  | j                  ¬«      | _        | j                  j                  �| j                  j                  n| j                  j
                  }t        j                  |¬«      | _        t        j                  | j                  j                  | j                  ¬«      | _        y )Nrñ   r9   r’   ©r  r2   r3   rë   rä   rK   r;   rJ   rL   rf   Ú
num_labelsrå   ©rN   Úclassifier_dropouts     r/   rO   z/FlaxAlbertForSequenceClassificationModule.setupI  s‚   € Ü&¨d¯k©kÀÇÁÔLˆŒð �{‰{×2Ñ2Ð>ð �K‰K×/Ò/à—‘×0Ñ0ð 	ô
 —z‘zÐ'9Ô:ˆŒÜŸ(™(Ø�K‰K×"Ñ"Ø—*‘*ô
ˆ�r.   rP   rk   r¨   r¼   c	           
      óà   — | j                  ||||||||¬«      }	|	d   }
| j                  |
|¬«      }
| j                  |
«      }|s	|f|	dd  z   S t        ||	j                  |	j
                  ¬«      S )Nr  r   rS   rm   r7  )rë   rL   rå   r   r$   r%   )rN   rU   r‚   rV   rW   rP   rk   r¨   r¼   rŒ   rç   rè   s               r/   rZ   z2FlaxAlbertForSequenceClassificationModule.__call__V  s’   € ð —+‘+ØØØØØ'Ø/Ø!5Ø#ð ó 	
ˆð   ™
ˆØŸ™ ]À-˜ÓPˆØ—‘ Ó/ˆáØ�9˜w q r˜{Ñ*Ð*ä+ØØ!×/Ñ/Ø×)Ñ)ô
ð 	
r.   NrË   rŽ   r-   r.   r/   r<  r<  E  sZ   … ØÓØ—{‘{€Eˆ3�9‰9Ó"ò
ð& #Ø"'Ø%*Ø ñ"
ð ð"
ð  ð"
ð #ð"
ð ô"
r.   r<  zž
    Albert Model transformer with a sequence classification/regression head on top (a linear layer on top of the pooled
    output) e.g. for GLUE tasks.
    c                   ó   — e Zd ZeZy)Ú#FlaxAlbertForSequenceClassificationN)r&   r'   r(   r<  rì   r-   r.   r/   rD  rD  {  s	   „ ð =�Lr.   rD  c            	       óv   — e Zd ZU eed<   ej                  Zej                  ed<   d„ Z	 	 	 	 d
de	de	de	de	fd„Z
y	)Ú!FlaxAlbertForMultipleChoiceModuler2   r3   c                 óú   — t        | j                  | j                  ¬«      | _        t	        j
                  | j                  j                  ¬«      | _        t	        j                  d| j                  ¬«      | _	        y )Nrñ   r9   r   r’   )
r  r2   r3   rë   r;   rJ   rK   rL   rf   rå   rM   s    r/   rO   z'FlaxAlbertForMultipleChoiceModule.setup’  sH   € Ü&¨d¯k©kÀÇÁÔLˆŒÜ—z‘z t§{¡{×'FÑ'FÔGˆŒÜŸ(™( 1¨D¯J©JÔ7ˆ�r.   rP   rk   r¨   r¼   c	           
      ó:  — |j                   d   }	|�|j                  d|j                   d   «      nd }|�|j                  d|j                   d   «      nd }|�|j                  d|j                   d   «      nd }|�|j                  d|j                   d   «      nd }| j                  ||||||||¬«      }
|
d   }| j                  ||¬«      }| j	                  |«      }|j                  d|	«      }|s	|f|
dd  z   S t        ||
j                  |
j                  ¬«      S )Nr   rw   r  rS   rm   r7  )ry   rx   rë   rL   rå   r   r$   r%   )rN   rU   r‚   rV   rW   rP   rk   r¨   r¼   Únum_choicesrŒ   rç   rè   Úreshaped_logitss                 r/   rZ   z*FlaxAlbertForMultipleChoiceModule.__call__—  sE  € ð  —o‘o aÑ(ˆØBKÐBW�I×%Ñ% b¨)¯/©/¸"Ñ*=Ô>Ð]aˆ	ØQ_ÐQk˜×/Ñ/°°N×4HÑ4HÈÑ4LÔMÐquˆØQ_ÐQk˜×/Ñ/°°N×4HÑ4HÈÑ4LÔMÐquˆØKWÐKc�|×+Ñ+¨B°×0BÑ0BÀ2Ñ0FÔGÐimˆð —+‘+ØØØØØ'Ø/Ø!5Ø#ð ó 	
ˆð   ™
ˆØŸ™ ]À-˜ÓPˆØ—‘ Ó/ˆà Ÿ.™.¨¨[Ó9ˆáØ#Ð%¨°°¨Ñ3Ð3ä,Ø"Ø!×/Ñ/Ø×)Ñ)ô
ð 	
r.   NrË   rŽ   r-   r.   r/   rF  rF  Ž  sZ   … ØÓØ—{‘{€Eˆ3�9‰9Ó"ò8ð #Ø"'Ø%*Ø ñ*
ð ð*
ð  ð*
ð #ð*
ð ô*
r.   rF  z§
    Albert Model with a multiple choice classification head on top (a linear layer on top of the pooled output and a
    softmax) e.g. for RocStories/SWAG tasks.
    c                   ó   — e Zd ZeZy)ÚFlaxAlbertForMultipleChoiceN)r&   r'   r(   rF  rì   r-   r.   r/   rL  rL  Ä  s	   „ ð 5�Lr.   rL  z(batch_size, num_choices, sequence_lengthc            	       óv   — e Zd ZU eed<   ej                  Zej                  ed<   d„ Z	 	 	 	 d
de	de	de	de	fd„Z
y	)Ú&FlaxAlbertForTokenClassificationModuler2   r3   c                 ó€  — t        | j                  | j                  d¬«      | _        | j                  j                  �| j                  j                  n| j                  j
                  }t        j                  |¬«      | _        t        j                  | j                  j                  | j                  ¬«      | _        y )NF©r2   r3   r  r9   r’   r>  r@  s     r/   rO   z,FlaxAlbertForTokenClassificationModule.setupÞ  s�   € Ü&¨d¯k©kÀÇÁÐ_dÔeˆŒð �{‰{×2Ñ2Ð>ð �K‰K×/Ò/à—‘×0Ñ0ð 	ô
 —z‘zÐ'9Ô:ˆŒÜŸ(™( 4§;¡;×#9Ñ#9ÀÇÁÔLˆ�r.   rP   rk   r¨   r¼   c	           
      óà   — | j                  ||||||||¬«      }	|	d   }
| j                  |
|¬«      }
| j                  |
«      }|s	|f|	dd  z   S t        ||	j                  |	j
                  ¬«      S )Nr  r   rS   r   r7  )rë   rL   rå   r   r$   r%   )rN   rU   r‚   rV   rW   rP   rk   r¨   r¼   rŒ   r$   rè   s               r/   rZ   z/FlaxAlbertForTokenClassificationModule.__call__è  s’   € ð —+‘+ØØØØØ'Ø/Ø!5Ø#ð ó 	
ˆð   ™
ˆØŸ™ ]À-˜ÓPˆØ—‘ Ó/ˆáØ�9˜w q r˜{Ñ*Ð*ä(ØØ!×/Ñ/Ø×)Ñ)ô
ð 	
r.   NrË   rŽ   r-   r.   r/   rN  rN  Ú  s[   … ØÓØ—{‘{€Eˆ3�9‰9Ó"òMð  #Ø"'Ø%*Ø ñ"
ð ð"
ð  ð"
ð #ð"
ð ô"
r.   rN  z¥
    Albert Model with a token classification head on top (a linear layer on top of the hidden-states output) e.g. for
    Named-Entity-Recognition (NER) tasks.
    c                   ó   — e Zd ZeZy)Ú FlaxAlbertForTokenClassificationN)r&   r'   r(   rN  rì   r-   r.   r/   rS  rS    s	   „ ð :�Lr.   rS  c            	       óv   — e Zd ZU eed<   ej                  Zej                  ed<   d„ Z	 	 	 	 d
de	de	de	de	fd„Z
y	)Ú$FlaxAlbertForQuestionAnsweringModuler2   r3   c                 óÆ   — t        | j                  | j                  d¬«      | _        t	        j
                  | j                  j                  | j                  ¬«      | _        y )NFrP  r’   )r  r2   r3   rë   r;   rf   r?  Ú
qa_outputsrM   s    r/   rO   z*FlaxAlbertForQuestionAnsweringModule.setup$  s;   € Ü&¨d¯k©kÀÇÁÐ_dÔeˆŒÜŸ(™( 4§;¡;×#9Ñ#9ÀÇÁÔLˆ�r.   rP   rk   r¨   r¼   c	           
      ó`  — | j                  ||||||||¬«      }	|	d   }
| j                  |
«      }t        j                  || j                  j
                  d¬«      \  }}|j                  d«      }|j                  d«      }|s
||f|	dd  z   S t        |||	j                  |	j                  ¬«      S )Nr  r   rw   rp   r   )Ústart_logitsÚ
end_logitsr$   r%   )
rë   rW  r*   r   r2   r?  Úsqueezer   r$   r%   )rN   rU   r‚   rV   rW   rP   rk   r¨   r¼   rŒ   r$   rè   rY  rZ  s                 r/   rZ   z-FlaxAlbertForQuestionAnsweringModule.__call__(  sÈ   € ð —+‘+ØØØØØ'Ø/Ø!5Ø#ð ó 	
ˆð   ™
ˆà—‘ Ó/ˆÜ#&§9¡9¨V°T·[±[×5KÑ5KÐRTÔ#UÑ ˆ�jØ#×+Ñ+¨BÓ/ˆØ×'Ñ'¨Ó+ˆ
áØ  *Ð-°¸¸°Ñ;Ð;ä/Ø%Ø!Ø!×/Ñ/Ø×)Ñ)ô	
ð 	
r.   NrË   rŽ   r-   r.   r/   rU  rU     s[   … ØÓØ—{‘{€Eˆ3�9‰9Ó"òMð #Ø"'Ø%*Ø ñ&
ð ð&
ð  ð&
ð #ð&
ð ô&
r.   rU  zß
    Albert Model with a span classification head on top for extractive question-answering tasks like SQuAD (a linear
    layers on top of the hidden-states output to compute `span start logits` and `span end logits`).
    c                   ó   — e Zd ZeZy)ÚFlaxAlbertForQuestionAnsweringN)r&   r'   r(   rU  rì   r-   r.   r/   r]  r]  Q  s	   „ ð 8�Lr.   r]  )rê   r#  r1  r9  rD  rL  rS  r]  )RÚtypingr   r   r   ÚflaxÚ
flax.linenÚlinenr;   r?   Ú	jax.numpyÚnumpyr*   rà   Úflax.core.frozen_dictr   r   r   Úflax.linen.attentionr	   Úflax.traverse_utilr
   r   r   Úmodeling_flax_outputsr   r   r   r   r   r   r   Úmodeling_flax_utilsr   r   r   r   r   Úutilsr   r   r   r   Úconfiguration_albertr   Ú
get_loggerr&   ÚloggerÚ_CHECKPOINT_FOR_DOCÚ_CONFIG_FOR_DOCÚstructÚ	dataclassr!   ÚALBERT_START_DOCSTRINGr  r  r1   r_   r�   rŸ   r²   r¹   rÍ   rÓ   râ   rê   r  r#  r%  r1  Ú%FLAX_ALBERT_FOR_PRETRAINING_DOCSTRINGr  r4  r9  r<  rD  rF  rL  rN  rS  rU  r]  Ú__all__r-   r.   r/   ú<module>rt     sÍ  ð÷  -Ñ ,ã Ý Û 
Ý Û ß >Ñ >Ý >ß ;Ý ÷÷ ñ ÷õ ÷ gÓ fÝ .ð 
ˆ×	Ñ	˜HÓ	%€à-Ð Ø €ð ‡�×Ñô4 [ó 4ó ð4ð:!Ð ðFÐ ôB%˜2Ÿ9™9ô %ôPR˜bŸi™iô Rôj)�b—i‘iô )ôX( §	¡	ô (ôV §¡ô ô4,
˜BŸI™Iô ,
ô^
˜Ÿ	™	ô 
ô>˜BŸI™Iô ô4˜Ÿ	™	ô ô\
Ð 3ô \
ô~C
�r—y‘yô C
ñL ØfØóô$Ð/ó $ó	ð$ñ ˜_Ð.AÐCaÐcrÔ sô3
 R§Y¡Yô 3
ñl ðð óô2Ð8ó 2óð2ð)Ð %ñ& ØØ×"Ñ"Ð#@ÓAÐDiÑiôñ !ØÐ*HÐWfõô
/
 "§)¡)ô /
ñd ÐPÐRhÓiô/Ð5ó /ó jð/ñ ØÐ.Ð0BÀOÐ^jõô
3
°·	±	ô 3
ñl ðð óô=Ð*Có =óð=ñ Ø'ØØ Øô	ô3
¨¯	©	ô 3
ñl ðð óô5Ð";ó 5óð5ñ ØÐ!8×!?Ñ!?Ð@jÓ!kôñ ØØØ!Øô	ô0
¨R¯Y©Yô 0
ñf ðð óô:Ð'@ó :óð:ñ Ø$ØØØô	ô.
¨2¯9©9ô .
ñb ðð óô8Ð%>ó 8óð8ñ Ø"ØØ$Øô	ò	�r.   