Ë
    S^(h‘  ã                   ó0  — d dl mZmZmZmZ d dlZd dlmZ d dl	Z	d dl
mZ d dlZd dlmZmZmZ d dlmZ d dlmZmZ ddlmZmZmZmZ ddlmZmZmZm Z  dd	l!m"Z"m#Z# d
dl$m%Z% ejL                  jN                   G d„ de«      «       Z(dZ)dZ*dee+e+f   dejX                  fd„Z-ej\                  fd„Z/ G d„ dej`                  «      Z1 G d„ dej`                  «      Z2 G d„ dej`                  «      Z3 G d„ dej`                  «      Z4 G d„ dej`                  «      Z5 G d„ dej`                  «      Z6 G d „ d!ej`                  «      Z7 G d"„ d#ej`                  «      Z8 G d$„ d%ej`                  «      Z9 G d&„ d'ej`                  «      Z: G d(„ d)ej`                  «      Z; G d*„ d+ej`                  «      Z< G d,„ d-e«      Z= G d.„ d/ej`                  «      Z> G d0„ d1ej`                  «      Z? e"d2e)«       G d3„ d4e=«      «       Z@d5ZA e e@eA«        ee@e(e%¬6«        G d7„ d8ej`                  «      ZB e"d9e)«       G d:„ d;e=«      «       ZCd<ZD e eCeD«        eeCee%¬6«        G d=„ d>ej`                  «      ZE e"d?e)«       G d@„ dAe=«      «       ZFdBZG e eFeG«        eeFee%¬6«       g dC¢ZHy)Dé    )ÚCallableÚListÚOptionalÚTupleN)Ú
FrozenDictÚfreezeÚunfreeze)Údot_product_attention_weights)Úflatten_dictÚunflatten_dicté   )ÚFlaxBaseModelOutputÚFlaxBaseModelOutputWithPoolingÚFlaxMaskedLMOutputÚFlaxSequenceClassifierOutput)ÚACT2FNÚFlaxPreTrainedModelÚ append_replace_return_docstringsÚoverwrite_call_docstring)Úadd_start_docstringsÚ%add_start_docstrings_to_model_forwardé   )Ú
BeitConfigc                   ó   — e Zd ZdZy)ÚFlaxBeitModelOutputWithPoolinga†  
    Class for outputs of [`FlaxBeitModel`].

    Args:
        last_hidden_state (`jnp.ndarray` of shape `(batch_size, sequence_length, hidden_size)`):
            Sequence of hidden-states at the output of the last layer of the model.
        pooler_output (`jnp.ndarray` of shape `(batch_size, hidden_size)`):
            Average of the last layer hidden states of the patch tokens (excluding the *[CLS]* token) if
            *config.use_mean_pooling* is set to True. If set to False, then the final hidden state of the *[CLS]* token
            will be returned.
        hidden_states (`tuple(jnp.ndarray)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
            Tuple of `jnp.ndarray` (one for the output of the embeddings + one for the output of each layer) of shape
            `(batch_size, sequence_length, hidden_size)`. Hidden-states of the model at the output of each layer plus
            the initial embedding outputs.
        attentions (`tuple(jnp.ndarray)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
            Tuple of `jnp.ndarray` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
            sequence_length)`. Attentions weights after the attention softmax, used to compute the weighted average in
            the self-attention heads.
    N)Ú__name__Ú
__module__Ú__qualname__Ú__doc__© ó    úi/var/www/skyplay_api_hub/venv/lib/python3.12/site-packages/transformers/models/beit/modeling_flax_beit.pyr   r   ,   s   „ òr!   r   aü  

    This model inherits from [`FlaxPreTrainedModel`]. Check the superclass documentation for the generic methods the
    library implements for all its model (such as downloading, saving and converting weights from PyTorch models)

    This model is also a
    [flax.linen.Module](https://flax.readthedocs.io/en/latest/api_reference/flax.linen/module.html) subclass. Use it as
    a regular Flax linen Module and refer to the Flax documentation for all matter related to general usage and
    behavior.

    Finally, this model supports inherent JAX features such as:

    - [Just-In-Time (JIT) compilation](https://jax.readthedocs.io/en/latest/jax.html#just-in-time-compilation-jit)
    - [Automatic Differentiation](https://jax.readthedocs.io/en/latest/jax.html#automatic-differentiation)
    - [Vectorization](https://jax.readthedocs.io/en/latest/jax.html#vectorization-vmap)
    - [Parallelization](https://jax.readthedocs.io/en/latest/jax.html#parallelization-pmap)

    Parameters:
        config ([`BeitConfig`]): Model configuration class with all the parameters of the model.
            Initializing with a config file does not load the weights associated with the model, only the
            configuration. Check out the [`~FlaxPreTrainedModel.from_pretrained`] method to load the model weights.
        dtype (`jax.numpy.dtype`, *optional*, defaults to `jax.numpy.float32`):
            The data type of the computation. Can be one of `jax.numpy.float32`, `jax.numpy.float16` (on GPUs) and
            `jax.numpy.bfloat16` (on TPUs).

            This can be used to enable mixed-precision training or half-precision inference on GPUs or TPUs. If
            specified all the computation will be performed with the given `dtype`.

            **Note that this only specifies the dtype of the computation and does not influence the dtype of model
            parameters.**

            If you wish to change the dtype of the model parameters, see [`~FlaxPreTrainedModel.to_fp16`] and
            [`~FlaxPreTrainedModel.to_bf16`].
a  
    Args:
        pixel_values (`numpy.ndarray` of shape `(batch_size, num_channels, height, width)`):
            Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See
            [`AutoImageProcessor.__call__`] for details.

        output_attentions (`bool`, *optional*):
            Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
            tensors for more detail.
        output_hidden_states (`bool`, *optional*):
            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
            more detail.
        return_dict (`bool`, *optional*):
            Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
Úwindow_sizeÚreturnc                 ó  — d| d   z  dz
  d| d   z  dz
  z  dz   }t        j                  | d   «      }t        j                  | d   «      }t        j                  t        j                  ||d¬«      «      }t        j                  |d«      }|dd…dd…df   |dd…ddd…f   z
  }t        j
                  |d	«      }|dd…dd…dfxx   | d   dz
  z  cc<   |dd…dd…dfxx   | d   dz
  z  cc<   |dd…dd…dfxx   d| d   z  dz
  z  cc<   t        j                  | d   | d   z  dz   fdz  |j                  ¬
«      }|j                  d«      |dd…dd…f<   |dz
  |ddd…f<   |dz
  |dd…df<   |dz
  |d<   t        j                  |«      S )zP
    get pair-wise relative position index for each token inside the window
    é   r   r   r   Úij)Úindexing)r&   éÿÿÿÿN)r   r&   r   ©ÚshapeÚdtyper)   )r   r   )ÚnpÚarangeÚstackÚmeshgridÚreshapeÚ	transposeÚzerosr,   ÚsumÚjnpÚarray)r#   Únum_relative_distanceÚcoords_hÚcoords_wÚcoordsÚcoords_flattenÚrelative_coordsÚrelative_position_indexs           r"   Úrelative_position_index_initr>   w   s°  € ð  ¨Q¡Ñ/°!Ñ3¸¸KÈ¹NÑ8JÈQÑ8NÑOÐRSÑSÐä�y‰y˜ Q™Ó(€HÜ�y‰y˜ Q™Ó(€HÜ�X‰X”b—k‘k (¨H¸tÔDÓE€FÜ—Z‘Z ¨Ó0€NØ$¢Qª¨4 ZÑ0°>Â!ÀTÊ1À*Ñ3MÑM€OÜ—l‘l ?°IÓ>€OØ’A’q˜!�GÓ ¨A¡°Ñ 2Ñ2ÓØ’A’q˜!�GÓ ¨A¡°Ñ 2Ñ2ÓØ’A’q˜!�GÓ  K°¡NÑ 2°QÑ 6Ñ6Óä Ÿh™h¨k¸!©n¸{È1¹~Ñ.MÐPQÑ.QÐ-SÐVWÑ-WÐ_n×_tÑ_tÔuÐØ&5×&9Ñ&9¸"Ó&=Ð˜A™B ¡˜FÑ#Ø%:¸QÑ%>Ð˜A˜q™r˜EÑ"Ø%:¸QÑ%>Ð˜A™B ˜EÑ"Ø$9¸AÑ$=Ð˜DÑ!Ü�9‰9Ð,Ó-Ð-r!   c                 ó4   — t        j                  ||«      |z  S ©N)r5   Úones)Úkeyr+   Úscaler,   s       r"   Úones_with_scalerD   �   s   € Ü�8‰8�E˜5Ó! EÑ)Ð)r!   c                   ób   — e Zd ZU dZeed<   ej                  j                  dde	e
   fd„«       Zy)ÚFlaxBeitDropPathzXDrop paths (Stochastic Depth) per sample (when applied in main path of residual blocks).ÚrateÚdeterministicc                 óN  — | j                   dk(  r|S d| j                   z
  }|r|S |j                  d   fd|j                  dz
  z  z   }| j                  d«      }|t        j
                  j                  |||j                  ¬«      z   }t        j                  |«      }||z  |z  }|S )Nç        g      ð?r   )r   r   Údroppathr*   )
rG   r+   ÚndimÚmake_rngÚjaxÚrandomÚuniformr,   r5   Úfloor)	ÚselfÚinputsrH   Ú	keep_probr+   ÚrngÚrandom_tensorÚbinary_tensorÚoutputs	            r"   Ú__call__zFlaxBeitDropPath.__call__˜   s¡   € à�9‰9˜ÒØˆMØ˜$Ÿ)™)‘Oˆ	ÙØˆMà—\‘\ !‘_Ð&¨°·±¸q±Ñ)AÑAˆEØ—-‘- 
Ó+ˆCØ%¬¯
©
×(:Ñ(:¸3ÀeÐSY×S_ÑS_Ð(:Ó(`Ñ`ˆMÜŸI™I mÓ4ˆMØ˜iÑ'¨-Ñ7ˆFØˆMr!   N©T)r   r   r   r   ÚfloatÚ__annotations__ÚnnÚmoduleÚcompactr   ÚboolrY   r    r!   r"   rF   rF   “   s1   … Ùbà
ƒKà‡Y�Y×Ññ¨h°t©nò ó ñr!   rF   c                   óZ   — e Zd ZU eed<   ej                  Zej                  ed<   d„ Zd„ Z	y)ÚFlaxBeitPatchEmbeddingsÚconfigr,   c           
      óÒ  — | j                   j                  | _        | j                   j                  }| j                   j                  }||z  ||z  z  }||z  ||z  f}|| _        || _        t        j                  | j                   j                  ||f||fd| j                  t        j                  j                  j                  | j                   j                  «      ¬«      | _        y )NÚVALID)Úkernel_sizeÚstridesÚpaddingr,   Úkernel_init)rc   Únum_channelsÚ
image_sizeÚ
patch_sizeÚnum_patchesÚpatch_shaper]   ÚConvÚhidden_sizer,   rN   ÚinitializersÚnormalÚinitializer_rangeÚ
projection)rR   rk   rl   rm   rn   s        r"   ÚsetupzFlaxBeitPatchEmbeddings.setup¬   sÃ   € Ø ŸK™K×4Ñ4ˆÔØ—[‘[×+Ñ+ˆ
Ø—[‘[×+Ñ+ˆ
Ø! ZÑ/°JÀ*Ñ4LÑMˆØ! ZÑ/°¸zÑ1IÐJˆØ&ˆÔØ&ˆÔÜŸ'™'Ø�K‰K×#Ñ#Ø# ZÐ0Ø Ð,ØØ—*‘*ÜŸ™×+Ñ+×2Ñ2°4·;±;×3PÑ3PÓQô
ˆ�r!   c                 óÊ   — |j                   d   }|| j                  k7  rt        d«      ‚| j                  |«      }|j                   \  }}}}t	        j
                  ||d|f«      S )Nr)   zeMake sure that the channel dimension of the pixel values match with the one set in the configuration.)r+   rj   Ú
ValueErrorrt   r5   r1   )rR   Úpixel_valuesrj   Ú
embeddingsÚ
batch_sizeÚ_Úchannelss          r"   rY   z FlaxBeitPatchEmbeddings.__call__½   sl   € Ø#×)Ñ)¨"Ñ-ˆØ˜4×,Ñ,Ò,ÜØwóð ð —_‘_ \Ó2ˆ
Ø%/×%5Ñ%5Ñ"ˆ
�A�q˜(Ü�{‰{˜:¨
°B¸Ð'AÓBÐBr!   N©
r   r   r   r   r\   r5   Úfloat32r,   ru   rY   r    r!   r"   rb   rb   ¨   s%   … ØÓØ—{‘{€Eˆ3�9‰9Ó"ò
ó"Cr!   rb   c                   ó`   — e Zd ZU dZeed<   ej                  Zej                  ed<   d„ Z	dd„Z
y)ÚFlaxBeitEmbeddingsz7Construct the CLS token, position and patch embeddings.rc   r,   c                 óâ  — | j                  dt        j                  j                  dd| j                  j
                  f«      | _        | j                  j                  rG| j                  dt        j                  j                  dd| j                  j
                  f«      | _        t        | j                  | j                  ¬«      | _        | j                  j                  }| j                  j                  rJ| j                  dt        j                  j                  d|dz   | j                  j
                  f«      | _        t        j                  | j                  j                   ¬«      | _        y )NÚ	cls_tokenr   Ú
mask_token©r,   Úposition_embeddings©rG   )Úparamr]   rq   r3   rc   rp   r‚   Úuse_mask_tokenrƒ   rb   r,   Úpatch_embeddingsrm   Ú use_absolute_position_embeddingsr…   ÚDropoutÚhidden_dropout_probÚdropout)rR   rm   s     r"   ru   zFlaxBeitEmbeddings.setupÎ   sý   € ØŸ™ K´·±×1FÑ1FÈÈAÈtÏ{É{×OfÑOfÐHgÓhˆŒØ�;‰;×%Ò%Ø"Ÿj™j¨´r·±×7LÑ7LÈqÐRSÐUY×U`ÑU`×UlÑUlÐNmÓnˆDŒOÜ 7¸¿¹È4Ï:É:Ô VˆÔØ×+Ñ+×7Ñ7ˆØ�;‰;×7Ò7Ø'+§z¡zØ%¤r§¡×'<Ñ'<¸qÀ+ÐPQÁ/ÐSW×S^ÑS^×SjÑSjÐ>kó(ˆDÔ$ô —z‘z t§{¡{×'FÑ'FÔGˆ�r!   Nc                 ó²  — | j                  |«      }|j                  \  }}}t        j                  | j                  |d| j
                  j                  f«      }|j                  |j                  «      }|�wt        j                  | j                  ||| j
                  j                  f«      }	|	j                  |j                  «      }	t        j                  |d¬«      }
|d|
z
  z  |	|
z  z   }t        j                  ||fd¬«      }| j
                  j                  r(|| j                  j                  |j                  «      z   }| j                  ||¬«      }|S )Nr   r)   ©Úaxis©rH   )r‰   r+   r5   Úbroadcast_tor‚   rc   rp   Úastyper,   rƒ   Úexpand_dimsÚconcatenaterŠ   r…   r�   )rR   rx   Úbool_masked_posrH   ry   rz   Úseq_lenr{   Ú
cls_tokensÚmask_tokensÚws              r"   rY   zFlaxBeitEmbeddings.__call__Ú   s)  € Ø×*Ñ*¨<Ó8ˆ
Ø!+×!1Ñ!1Ñˆ
�G˜Qä×%Ñ% d§n¡n°zÀ1ÀdÇkÁk×F]ÑF]Ð6^Ó_ˆ
Ø×&Ñ& z×'7Ñ'7Ó8ˆ
àÐ&Ü×*Ñ*¨4¯?©?¸ZÈÐRV×R]ÑR]×RiÑRiÐ<jÓkˆKØ%×,Ñ,¨Z×-=Ñ-=Ó>ˆKä—‘ °bÔ9ˆAØ# q¨1¡uÑ-°¸a±Ñ?ˆJä—_‘_ j°*Ð%=ÀAÔFˆ
à�;‰;×7Ò7Ø# d×&>Ñ&>×&EÑ&EÀj×FVÑFVÓ&WÑWˆJà—\‘\ *¸M�\ÓJˆ
ØÐr!   )NT)r   r   r   r   r   r\   r5   r~   r,   ru   rY   r    r!   r"   r€   r€   È   s(   … ÙAàÓØ—{‘{€Eˆ3�9‰9Ó"ò
Hôr!   r€   c                   ón   — e Zd ZU eed<   eeef   ed<   ej                  Z	ej                  ed<   d„ Z
d„ Zy)ÚFlaxBeitRelativePositionBiasrc   r#   r,   c                 ó   — d| j                   d   z  dz
  d| j                   d   z  dz
  z  dz   }| j                  dt        j                  j                  || j
                  j                  f«      | _        t        | j                   «      | _	        y )Nr&   r   r   r   Úrelative_position_bias_table)
r#   r‡   r]   rq   r3   rc   Únum_attention_headsrž   r>   r=   )rR   r7   s     r"   ru   z"FlaxBeitRelativePositionBias.setupö   sˆ   € Ø!" T×%5Ñ%5°aÑ%8Ñ!8¸1Ñ!<ÀÀT×EUÑEUÐVWÑEXÑAXÐ[\ÑA\Ñ ]Ð`aÑ aÐØ,0¯J©JØ*Ü�O‰O×!Ñ!Ø" D§K¡K×$CÑ$CÐDó-
ˆÔ)ô (DÀD×DTÑDTÓ'UˆÕ$r!   c                 ó*  — | j                   j                  d«      }| j                  d   | j                  d   z  dz   | j                  d   | j                  d   z  dz   df}| j                  |   j                  |«      }t	        j
                  |d«      S )Nr)   r   r   )r&   r   r   )r=   r1   r#   rž   r5   r2   )rR   Úindexr+   Úrelative_position_biass       r"   rY   z%FlaxBeitRelativePositionBias.__call__  s˜   € Ø×,Ñ,×4Ñ4°RÓ8ˆØ×!Ñ! !Ñ$ t×'7Ñ'7¸Ñ':Ñ:¸QÑ>À×@PÑ@PÐQRÑ@SÐVZ×VfÑVfÐghÑViÑ@iÐlmÑ@mÐoqÐrˆØ!%×!BÑ!BÀ5Ñ!I×!QÑ!QÐRWÓ!XÐÜ�}‰}Ð3°YÓ?Ð?r!   N)r   r   r   r   r\   r   Úintr5   r~   r,   ru   rY   r    r!   r"   rœ   rœ   ñ   s4   … ØÓØ�s˜C�x‘Ó Ø—{‘{€Eˆ3�9‰9Ó"ò	Vó@r!   rœ   c                   ó|   — e Zd ZU eed<   eeef   ed<   ej                  Z	ej                  ed<   d„ Z
	 d	dedefd„Zy)
ÚFlaxBeitSelfAttentionrc   r#   r,   c                 óT  — | j                   j                  | j                   j                  z  dk7  rPt        | j                   d«      s:t	        d| j                   j                  › d| j                   j                  › d�«      ‚t        j                  | j                   j                  | j                  t        j
                  j                  j                  | j                   j                  «      ¬«      | _        t        j                  | j                   j                  | j                  t        j
                  j                  j                  | j                   j                  «      d¬«      | _        t        j                  | j                   j                  | j                  t        j
                  j                  j                  | j                   j                  «      ¬«      | _        | j                  r2t!        | j                   | j                  | j                  ¬	«      | _        y d | _        y )
Nr   Úembedding_sizezThe hidden size z4 is not a multiple of the number of attention heads ú.)r,   ri   F)r,   ri   Úuse_bias©r#   r,   )rc   rp   rŸ   Úhasattrrw   r]   ÚDenser,   rN   rq   rr   rs   ÚqueryrB   Úvaluer#   rœ   r¢   ©rR   s    r"   ru   zFlaxBeitSelfAttention.setup  s�  € Ø�;‰;×"Ñ" T§[¡[×%DÑ%DÑDÈÒIÔRYØ�K‰KÐ)ôS
ô Ø" 4§;¡;×#:Ñ#:Ð";ð <ØŸ™×8Ñ8Ð9¸ð<óð ô
 —X‘XØ�K‰K×#Ñ#Ø—*‘*ÜŸ™×+Ñ+×2Ñ2°4·;±;×3PÑ3PÓQô
ˆŒ
ô
 —8‘8Ø�K‰K×#Ñ#Ø—*‘*ÜŸ™×+Ñ+×2Ñ2°4·;±;×3PÑ3PÓQØô	
ˆŒô —X‘XØ�K‰K×#Ñ#Ø—*‘*ÜŸ™×+Ñ+×2Ñ2°4·;±;×3PÑ3PÓQô
ˆŒ
ð ×Òô )¨¯©À$×BRÑBRÐZ^×ZdÑZdÔeð 	Õ#ð ð 	Õ#r!   NrH   Úoutput_attentionsc                 ób  — | j                   j                  | j                   j                  z  }| j                  |«      j	                  |j
                  d d | j                   j                  |fz   «      }| j                  |«      j	                  |j
                  d d | j                   j                  |fz   «      }| j                  |«      j	                  |j
                  d d | j                   j                  |fz   «      }d }	|s*| j                   j                  dkD  r| j                  d«      }	t        j                  d| j                  ¬«      }
| j                  �?t        j                  | j                  «       d«      }
|
j                  |j                  «      }
|�|
|j                  |
j                  «      z   }
t!        |||
|	| j                   j                  d|| j                  d ¬«	      }t        j"                  d||«      }|j	                  |j
                  d d d	z   «      }|r||f}|S |f}|S )
Nr&   rJ   r�   r„   r   T)ÚbiasÚdropout_rngÚdropout_rateÚbroadcast_dropoutrH   r,   Ú	precisionz...hqk,...khd->...qhd)r)   )rc   rp   rŸ   r­   r1   r+   r®   rB   Úattention_probs_dropout_probrM   r5   r6   r,   r¢   r”   r“   r
   Úeinsum)rR   Úhidden_statesr¢   rH   r°   Úhead_dimÚquery_statesÚvalue_statesÚ
key_statesr³   Úattention_biasÚattn_weightsÚattn_outputÚoutputss                 r"   rY   zFlaxBeitSelfAttention.__call__-  s  € ð —;‘;×*Ñ*¨d¯k©k×.MÑ.MÑMˆà—z‘z -Ó0×8Ñ8Ø×Ñ  Ð# t§{¡{×'FÑ'FÈÐ&QÑQó
ˆð —z‘z -Ó0×8Ñ8Ø×Ñ  Ð# t§{¡{×'FÑ'FÈÐ&QÑQó
ˆð —X‘X˜mÓ,×4Ñ4Ø×Ñ  Ð# t§{¡{×'FÑ'FÈÐ&QÑQó
ˆ
ð ˆÙ §¡×!IÑ!IÈCÒ!OØŸ-™-¨	Ó2ˆKäŸ™ 3¨d¯j©jÔ9ˆà×&Ñ&Ð2Ü Ÿ_™_¨T×-HÑ-HÓ-JÈAÓNˆNØ+×2Ñ2°<×3EÑ3EÓFˆNð "Ð-Ø+Ð.D×.KÑ.KÈN×L`ÑL`Ó.aÑaˆNä4ØØØØ#ØŸ™×AÑAØ"Ø'Ø—*‘*Øô

ˆô —j‘jÐ!8¸,ÈÓUˆØ!×)Ñ)¨+×*;Ñ*;¸B¸QÐ*?À%Ñ*GÓHˆá1B�; Ð-ˆØˆð JUÈˆØˆr!   ©NTF©r   r   r   r   r\   r   r£   r5   r~   r,   ru   r`   rY   r    r!   r"   r¥   r¥     sJ   … ØÓØ�s˜C�x‘Ó Ø—{‘{€Eˆ3�9‰9Ó"ò
ðB qvñ-ØIMð-Øimô-r!   r¥   c                   ób   — e Zd ZU eed<   ej                  Zej                  ed<   d„ Zdde	fd„Z
y)ÚFlaxBeitSelfOutputrc   r,   c                 óN  — t        j                  | j                  j                  t        j                   j
                  j                  | j                  j                  «      | j                  ¬«      | _	        t        j                  | j                  j                  ¬«      | _        y ©N©ri   r,   r†   ©r]   r¬   rc   rp   rN   rq   rr   rs   r,   Údenser‹   rŒ   r�   r¯   s    r"   ru   zFlaxBeitSelfOutput.setupa  ód   € Ü—X‘XØ�K‰K×#Ñ#ÜŸ™×+Ñ+×2Ñ2°4·;±;×3PÑ3PÓQØ—*‘*ô
ˆŒ
ô
 —z‘z t§{¡{×'FÑ'FÔGˆ�r!   rH   c                 óN   — | j                  |«      }| j                  ||¬«      }|S ©Nr‘   ©rÊ   r�   ©rR   r¹   rH   s      r"   rY   zFlaxBeitSelfOutput.__call__i  s(   € ØŸ
™
 =Ó1ˆØŸ™ ]À-˜ÓPˆØÐr!   NrZ   ©r   r   r   r   r\   r5   r~   r,   ru   r`   rY   r    r!   r"   rÅ   rÅ   ]  s,   … ØÓØ—{‘{€Eˆ3�9‰9Ó"òHñ°Tô r!   rÅ   c                   óx   — e Zd ZU eed<   eeef   ed<   ej                  Z	ej                  ed<   d„ Z
	 ddefd„Zy)	ÚFlaxBeitAttentionrc   r#   r,   c                 ó²   — t        | j                  | j                  | j                  ¬«      | _        t        | j                  | j                  ¬«      | _        y )Nr„   )r¥   rc   r#   r,   Ú	attentionrÅ   rX   r¯   s    r"   ru   zFlaxBeitAttention.setupt  s9   € Ü.¨t¯{©{¸D×<LÑ<LÐTX×T^ÑT^Ô_ˆŒÜ(¨¯©¸D¿J¹JÔGˆ�r!   Nr°   c                 ó|   — | j                  ||||¬«      }|d   }| j                  ||¬«      }|f}|r	||d   fz  }|S ©N©rH   r°   r   r‘   r   )rÔ   rX   )rR   r¹   r¢   rH   r°   Úattn_outputsrÀ   rÁ   s           r"   rY   zFlaxBeitAttention.__call__x  s_   € ð —~‘~ØÐ1ÀÐbsð &ó 
ˆð # 1‘oˆØ—k‘k +¸]�kÓKˆà�.ˆáØ˜ Q™Ð)Ñ)ˆGàˆr!   rÂ   rÃ   r    r!   r"   rÒ   rÒ   o  sB   … ØÓØ�s˜C�x‘Ó Ø—{‘{€Eˆ3�9‰9Ó"òHð
 inñØaeôr!   rÒ   c                   óZ   — e Zd ZU eed<   ej                  Zej                  ed<   d„ Zd„ Z	y)ÚFlaxBeitIntermediaterc   r,   c                 ó4  — t        j                  | j                  j                  t        j                   j
                  j                  | j                  j                  «      | j                  ¬«      | _	        t        | j                  j                     | _        y )NrÈ   )r]   r¬   rc   Úintermediate_sizerN   rq   rr   rs   r,   rÊ   r   Ú
hidden_actÚ
activationr¯   s    r"   ru   zFlaxBeitIntermediate.setup�  s`   € Ü—X‘XØ�K‰K×)Ñ)ÜŸ™×+Ñ+×2Ñ2°4·;±;×3PÑ3PÓQØ—*‘*ô
ˆŒ
ô
 ! §¡×!7Ñ!7Ñ8ˆ�r!   c                 óJ   — | j                  |«      }| j                  |«      }|S r@   )rÊ   rÞ   )rR   r¹   s     r"   rY   zFlaxBeitIntermediate.__call__•  s$   € ØŸ
™
 =Ó1ˆØŸ™¨Ó6ˆàÐr!   Nr}   r    r!   r"   rÚ   rÚ   ‰  s$   … ØÓØ—{‘{€Eˆ3�9‰9Ó"ò9ór!   rÚ   c                   ób   — e Zd ZU eed<   ej                  Zej                  ed<   d„ Zdde	fd„Z
y)ÚFlaxBeitOutputrc   r,   c                 óN  — t        j                  | j                  j                  t        j                   j
                  j                  | j                  j                  «      | j                  ¬«      | _	        t        j                  | j                  j                  ¬«      | _        y rÇ   rÉ   r¯   s    r"   ru   zFlaxBeitOutput.setup   rË   r!   rH   c                 óN   — | j                  |«      }| j                  ||¬«      }|S rÍ   rÎ   rÏ   s      r"   rY   zFlaxBeitOutput.__call__¨  s(   € ØŸ
™
 =Ó1ˆØŸ™ ]À-˜ÓPˆàÐr!   NrZ   rÐ   r    r!   r"   rá   rá   œ  s,   … ØÓØ—{‘{€Eˆ3�9‰9Ó"òHñ°Tô r!   rá   c                   ó†   — e Zd ZU eed<   eeef   ed<   eed<   ej                  Z
ej                  ed<   d„ Z	 d
dedefd	„Zy)ÚFlaxBeitLayerrc   r#   Údrop_path_rater,   c                 óz  — t        | j                  | j                  | j                  ¬«      | _        t        | j                  | j                  ¬«      | _        t        | j                  | j                  ¬«      | _        t        j                  | j                  j                  | j                  ¬«      | _        t        | j                  ¬«      | _        t        j                  | j                  j                  | j                  ¬«      | _        | j                  j"                  | _        | j$                  dkD  rw| j'                  dt(        | j                  j*                  | j$                  «      | _        | j'                  dt(        | j                  j*                  | j$                  «      | _        y d | _        d | _        y )Nr„   ©Úepsilonr,   r†   r   Úlambda_1Úlambda_2)rÒ   rc   r#   r,   rÔ   rÚ   Úintermediaterá   rX   r]   Ú	LayerNormÚlayer_norm_epsÚlayernorm_beforerF   ræ   Ú	drop_pathÚlayernorm_afterÚlayer_scale_init_valueÚinit_valuesr‡   rD   rp   rê   rë   r¯   s    r"   ru   zFlaxBeitLayer.setupµ  s&  € Ü*¨4¯;©;¸×8HÑ8HÐPT×PZÑPZÔ[ˆŒÜ0°·±ÀDÇJÁJÔOˆÔÜ$ T§[¡[¸¿
¹
ÔCˆŒÜ "§¡°T·[±[×5OÑ5OÐW[×WaÑWaÔ bˆÔÜ)¨t×/BÑ/BÔCˆŒÜ!Ÿ|™|°D·K±K×4NÑ4NÐVZ×V`ÑV`ÔaˆÔàŸ;™;×=Ñ=ˆÔØ×Ñ˜aÒØ ŸJ™J z´?ÀTÇ[Á[×E\ÑE\Ð_c×_oÑ_oÓpˆDŒMØ ŸJ™J z´?ÀTÇ[Á[×E\ÑE\Ð_c×_oÑ_oÓpˆD�Mà ˆDŒMØ ˆD�Mr!   NrH   r°   c                 ó  — | j                  | j                  |«      |||¬«      }|d   }| j                  �(| j                  j                  |j                  «      |z  }| j                  ||¬«      |z   }| j                  |«      }| j                  |«      }| j                  ||¬«      }| j                  �(| j                  j                  |j                  «      |z  }| j                  ||¬«      |z   }|f}|r	||d   fz  }|S rÖ   )
rÔ   rï   rê   r“   r,   rð   rñ   rì   rX   rë   )	rR   r¹   r¢   rH   r°   Úself_attention_outputsÚattention_outputÚlayer_outputrÁ   s	            r"   rY   zFlaxBeitLayer.__call__Å  s   € ð "&§¡Ø×!Ñ! -Ó0Ø"Ø'Ø/ð	 "0ó "
Ðð 2°!Ñ4Ðð �=‰=Ð$Ø#Ÿ}™}×3Ñ3Ð4D×4JÑ4JÓKÐN^Ñ^Ðð Ÿ™Ð'7À}˜ÓUÐXeÑeˆð ×+Ñ+¨MÓ:ˆà×(Ñ(¨Ó6ˆØ—{‘{ <¸}�{ÓMˆð �=‰=Ð$ØŸ=™=×/Ñ/°×0BÑ0BÓCÀlÑRˆLð —~‘~ lÀ-�~ÓPÐS`Ñ`ˆà�/ˆáØÐ.¨qÑ1Ð3Ñ3ˆGàˆr!   rÂ   )r   r   r   r   r\   r   r£   r[   r5   r~   r,   ru   r`   rY   r    r!   r"   rå   rå   ¯  sO   … ØÓØ�s˜C�x‘Ó ØÓØ—{‘{€Eˆ3�9‰9Ó"ò!ð" qvñ$ØIMð$Øimô$r!   rå   c            	       óÂ   — e Zd ZU eed<   eeef   ed<   ee   ed<   e	g e
j                  f   ed<   e
j                  Ze
j                  ed<   d„ Z	 	 	 	 ddeded	ed
efd„Zy)ÚFlaxBeitLayerCollectionrc   r#   Údrop_path_ratesr¢   r,   c                 ó&  — t        | j                  j                  «      D �cg c]^  }t        | j                  | j                  j                  r| j
                  nd | j                  |   t        |«      | j                  ¬«      ‘Œ` c}| _	        y c c}w )N)r#   ræ   Únamer,   )
Úrangerc   Únum_hidden_layersrå   Úuse_relative_position_biasr#   rú   Ústrr,   Úlayers)rR   Úis     r"   ru   zFlaxBeitLayerCollection.setupó  st   € ô ˜4Ÿ;™;×8Ñ8Ó9ö	
ð ô Ø—‘Ø04·±×0VÒ0V˜D×,Ò,Ð\`Ø#×3Ñ3°AÑ6Ü˜“VØ—j‘jöò	
ˆ�ùò 	
s   ¢A#BrH   r°   Úoutput_hidden_statesÚreturn_dictc                 ó4  — |rdnd }|rdnd }t        | j                  «      D ]H  \  }}	|r||fz  }| j                  �| j                  «       nd }
 |	||
||¬«      }|d   }|sŒ@||d   fz  }ŒJ |r||fz  }|f}|st        d„ |D «       «      S t	        |||¬«      S )Nr    r×   r   r   c              3   ó&   K  — | ]	  }|€Œ|–— Œ y ­wr@   r    )Ú.0Úvs     r"   ú	<genexpr>z3FlaxBeitLayerCollection.__call__.<locals>.<genexpr>  s   è ø€ Ò=˜q¨q©}œÑ=ùs   ‚Š)Úlast_hidden_stater¹   Ú
attentions)Ú	enumerater  r¢   Útupler   )rR   r¹   rH   r°   r  r  Úall_attentionsÚall_hidden_statesr  Úlayerr¢   Úlayer_outputsrÁ   s                r"   rY   z FlaxBeitLayerCollection.__call__ÿ  sÜ   € ñ  1™°dˆÙ"6™B¸DÐä! $§+¡+Ó.ò 	6‰HˆAˆuÙ#Ø! mÐ%5Ñ5Ð!ØFJ×FaÑFaÐFm T×%@Ñ%@Ô%BÐswÐ"Ù!ØÐ5À]ÐfwôˆMð *¨!Ñ,ˆMâ Ø =°Ñ#3Ð"5Ñ5‘ð	6ñ  Ø -Ð!1Ñ1Ðà Ð"ˆÙÜÑ= GÔ=Ó=Ð=ä"Ø+Ð;LÐYgô
ð 	
r!   N©TFFT)r   r   r   r   r\   r   r£   r   r[   r   r5   Úndarrayr~   r,   ru   r`   rY   r    r!   r"   rù   rù   ì  s…   … ØÓØ�s˜C�x‘Ó Ø˜%‘[Ó Ø$ R¨¯© _Ñ5Ó5Ø—{‘{€Eˆ3�9‰9Ó"ò

ð #Ø"'Ø%*Ø ñ!
ð ð!
ð  ð	!
ð
 #ð!
ð ô!
r!   rù   c            	       óŠ   — e Zd ZU eed<   eeef   ed<   ej                  Z	ej                  ed<   d„ Z
	 	 	 	 ddedededefd	„Zy
)ÚFlaxBeitEncoderrc   r#   r,   c                 óÌ  — | j                   j                  r1t        | j                   | j                  | j                  ¬«      | _        t        t        j                  d| j                   j                  | j                   j                  «      «      }t        | j                   | j                  || j                   j                  r| j
                  nd | j                  ¬«      | _        y )N)rc   r#   r,   r   )r#   rú   r¢   r,   )rc   Ú!use_shared_relative_position_biasrœ   r#   r,   r¢   Úlistr-   Úlinspaceræ   rþ   rù   r  )rR   rú   s     r"   ru   zFlaxBeitEncoder.setup(  sŸ   € Ø�;‰;×8Ò8Ü*FØ—{‘{°×0@Ñ0@ÈÏ
É
ô+ˆDÔ'ô
 œrŸ{™{¨1¨d¯k©k×.HÑ.HÈ$Ï+É+×JgÑJgÓhÓiˆÜ,Ø�K‰KØ×(Ñ(Ø+à�{‰{×<Ò<ð $(×#>Ò#>àØ—*‘*ô
ˆ�
r!   rH   r°   r  r  c                 ó.   — | j                  |||||¬«      S )N©rH   r°   r  r  )r  )rR   r¹   rH   r°   r  r  s         r"   rY   zFlaxBeitEncoder.__call__:  s)   € ð �z‰zØØ'Ø/Ø!5Ø#ð ó 
ð 	
r!   Nr  rÃ   r    r!   r"   r  r  #  sh   … ØÓØ�s˜C�x‘Ó Ø—{‘{€Eˆ3�9‰9Ó"ò
ð* #Ø"'Ø%*Ø ñ
ð ð
ð  ð	
ð
 #ð
ð ô
r!   r  c                   óv  ‡ — e Zd ZU dZeZdZdZdZe	j                  ed<   ddej                  dfded	ed
ej                  defˆ fd„Zddej&                  j(                  dededefd„Z eej5                  d«      «      	 	 	 	 	 	 	 ddedej&                  j(                  dedee   dee   dee   fd„«       Zˆ xZS )ÚFlaxBeitPreTrainedModelz†
    An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained
    models.
    Úbeitrx   NÚmodule_classr   Trc   Úseedr,   Ú_do_initc                 ó¦   •—  | j                   d||dœ|¤Ž}|€$d|j                  |j                  |j                  f}t        ‰| �  ||||||¬«       y )N)rc   r,   r   )Úinput_shaper   r,   r!  r    )r  rk   rj   ÚsuperÚ__init__)	rR   rc   r#  r   r,   r!  Úkwargsr^   Ú	__class__s	           €r"   r%  z FlaxBeitPreTrainedModel.__init__V  sc   ø€ ð #�×"Ñ"ÐH¨&¸ÑHÀÑHˆØÐØ˜f×/Ñ/°×1BÑ1BÀF×DWÑDWÐXˆKÜ‰Ñ˜ °[ÀtÐSXÐckÐÕlr!   rU   r#  Úparamsr$   c                 óê  — t        j                  || j                  ¬«      }t        j                  j                  |«      \  }}t        j                  j                  |«      \  }}|||dœ}| j                  j                  ||d¬«      d   }	|�dt        t        |	«      «      }	t        t        |«      «      }| j                  D ]
  }
|	|
   ||
<   Œ t        «       | _
        t        t        |«      «      S |	S )Nr„   )r(  r�   rK   F)r  r(  )r5   r3   r,   rN   rO   Úsplitr^   Úinitr   r	   Ú_missing_keysÚsetr   r   )rR   rU   r#  r(  rx   Ú
params_rngr³   Údroppath_rngÚrngsÚrandom_paramsÚmissing_keys              r"   Úinit_weightsz$FlaxBeitPreTrainedModel.init_weightsd  sß   € ä—y‘y °D·J±JÔ?ˆä"%§*¡*×"2Ñ"2°3Ó"7Ñˆ
�KÜ$'§J¡J×$4Ñ$4°[Ó$AÑ!ˆ�\Ø$°È,ÑWˆàŸ™×(Ñ(¨¨|ÈÐ(ÓOÐPXÑYˆàÐÜ(¬°-Ó)@ÓAˆMÜ!¤(¨6Ó"2Ó3ˆFØ#×1Ñ1ò A�Ø&3°KÑ&@��{Ò#ðAä!$£ˆDÔÜœ.¨Ó0Ó1Ð1à Ð r!   zbatch_size, sequence_lengthr³   Útrainr°   r  r  c	           
      óØ  — |�|n| j                   j                  }|�|n| j                   j                  }|�|n| j                   j                  }t	        j
                  |d«      }i }	|�,t        j                  j                  |«      \  }}
||	d<   |
|	d<   | j                  j                  d|xs | j                  it	        j                  |t        j                  ¬«      || ||||	¬«      S )N)r   r&   r   r   r�   rK   r(  r„   )r0  )rc   r°   r  r  r5   r2   rN   rO   r*  r^   Úapplyr(  r6   r~   )rR   rx   r–   r(  r³   r4  r°   r  r  r0  r/  s              r"   rY   z FlaxBeitPreTrainedModel.__call__x  sì   € ð 2CÐ1NÑ-ÐTX×T_ÑT_×TqÑTqÐà$8Ð$DÑ È$Ï+É+×JjÑJjð 	ð &1Ð%<‘kÀ$Ç+Á+×BYÑBYˆä—}‘} \°<Ó@ˆàˆØÐ"Ü(+¯
©
×(8Ñ(8¸Ó(EÑ%ˆK˜Ø)ˆD�‰OØ+ˆD�Ñà�{‰{× Ñ Ø�vÒ, §¡Ð-Ü�I‰I�l¬#¯+©+Ô6ØØˆIØØ ØØð !ó 	
ð 		
r!   r@   )NNNFNNN)r   r   r   r   r   Úconfig_classÚbase_model_prefixÚmain_input_namer  r]   ÚModuler\   r5   r~   r£   r,   r`   r%  rN   rO   ÚPRNGKeyr   r   r3  r   ÚBEIT_INPUTS_DOCSTRINGÚformatÚdictr   rY   Ú__classcell__)r'  s   @r"   r  r  K  s+  ø… ñð
 €LØÐØ$€OØ"€L�"—)‘)Ó"ð
 ØØŸ;™;Øñmàðmð ð	mð
 �y‰yðmð õmñ! §
¡
× 2Ñ 2ð !Àð !ÐPZð !Ðfpó !ñ( +Ð+@×+GÑ+GÐHeÓ+fÓgð ØØ*.ØØ,0Ø/3Ø&*ñ"
ð ð	"
ð
 —Z‘Z×'Ñ'ð"
ð ð"
ð $ D™>ð"
ð ' t™nð"
ð ˜d‘^ò"
ó hô"
r!   r  c                   óZ   — e Zd ZU eed<   ej                  Zej                  ed<   d„ Zd„ Z	y)ÚFlaxBeitPoolerrc   r,   c                 ó¦   — | j                   j                  r;t        j                  | j                   j                  | j
                  ¬«      | _        y y )Nrè   )rc   Úuse_mean_poolingr]   rí   rî   r,   Ú	layernormr¯   s    r"   ru   zFlaxBeitPooler.setup¢  s7   € Ø�;‰;×'Ò'ÜŸ\™\°$·+±+×2LÑ2LÐTX×T^ÑT^Ô_ˆD�Nð (r!   c                 ó°   — | j                   j                  r6|d d …dd …d d …f   }| j                  t        j                  |d¬«      «      }|S |d d …df   }|S )Nr   r�   r   )rc   rC  rD  r5   Úmean)rR   r¹   Úpatch_tokensÚpooled_outputs       r"   rY   zFlaxBeitPooler.__call__¦  sX   € Ø�;‰;×'Ò'à(ª¨A©B²¨Ñ2ˆLØ ŸN™N¬3¯8©8°LÀqÔ+IÓJˆMð
 Ðð *ª!¨Q¨$Ñ/ˆMàÐr!   Nr}   r    r!   r"   rA  rA  ž  s%   … ØÓØ—{‘{€Eˆ3�9‰9Ó"ò`ó	r!   rA  c            	       ó†   — e Zd ZU eed<   ej                  Zej                  ed<   dZe	ed<   d„ Z
	 	 	 	 	 dde	de	d	e	d
e	fd„Zy)ÚFlaxBeitModulerc   r,   TÚadd_pooling_layerc                 óî  — t        | j                  | j                  ¬«      | _        t	        | j                  | j                  j
                  j                  | j                  ¬«      | _        | j                  j                  s:t        j                  | j                  j                  | j                  ¬«      | _        | j                  r't        | j                  | j                  ¬«      | _        y d | _        y )Nr„   rª   rè   )r€   rc   r,   ry   r  r‰   rn   ÚencoderrC  r]   rí   rî   rD  rK  rA  Úpoolerr¯   s    r"   ru   zFlaxBeitModule.setup·  s�   € Ü,¨T¯[©[ÀÇ
Á
ÔKˆŒÜ&Ø�K‰K T§_¡_×%EÑ%E×%QÑ%QÐY]×YcÑYcô
ˆŒð �{‰{×+Ò+ÜŸ\™\°$·+±+×2LÑ2LÐTX×T^ÑT^Ô_ˆDŒNØGK×G]ÒG]”n T§[¡[¸¿
¹
ÔCˆ�Ðcgˆ�r!   NrH   r°   r  r  c                 ó`  — | j                  |||¬«      }| j                  |||||¬«      }|d   }| j                  j                  s| j	                  |«      }| j
                  r| j                  |«      nd }	|s|	€	|f|dd  z   S ||	f|dd  z   S t        ||	|j                  |j                  ¬«      S )Nr‘   r  r   r   )r
  Úpooler_outputr¹   r  )
ry   rM  rc   rC  rD  rK  rN  r   r¹   r  )
rR   rx   r–   rH   r°   r  r  r¹   rÁ   Úpooleds
             r"   rY   zFlaxBeitModule.__call__À  sÐ   € ð Ÿ™¨°oÐUb˜Ócˆà—,‘,ØØ'Ø/Ø!5Ø#ð ó 
ˆð   ™
ˆØ�{‰{×+Ò+Ø ŸN™N¨=Ó9ˆMØ/3×/EÒ/E�—‘˜]Ô+È4ˆáàˆ~Ø%Ð'¨'°!°"¨+Ñ5Ð5Ø! 6Ð*¨W°Q°R¨[Ñ8Ð8ä-Ø+Ø Ø!×/Ñ/Ø×)Ñ)ô	
ð 	
r!   )NTFFT)r   r   r   r   r\   r5   r~   r,   rK  r`   ru   rY   r    r!   r"   rJ  rJ  ²  si   … ØÓØ—{‘{€Eˆ3�9‰9Ó"Ø"Ð�tÓ"òhð Ø"Ø"'Ø%*Ø ñ"
ð ð	"
ð
  ð"
ð #ð"
ð ô"
r!   rJ  z^The bare Beit Model transformer outputting raw hidden-states without any specific head on top.c                   ó   — e Zd ZeZy)ÚFlaxBeitModelN)r   r   r   rJ  r  r    r!   r"   rS  rS  å  s	   „ ð
 "�Lr!   rS  aœ  
    Returns:

    Examples:

    ```python
    >>> from transformers import AutoImageProcessor, FlaxBeitModel
    >>> from PIL import Image
    >>> import requests

    >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
    >>> image = Image.open(requests.get(url, stream=True).raw)

    >>> image_processor = AutoImageProcessor.from_pretrained("microsoft/beit-base-patch16-224-pt22k-ft22k")
    >>> model = FlaxBeitModel.from_pretrained("microsoft/beit-base-patch16-224-pt22k-ft22k")

    >>> inputs = image_processor(images=image, return_tensors="np")
    >>> outputs = model(**inputs)
    >>> last_hidden_states = outputs.last_hidden_state
    ```
)Úoutput_typer7  c                   ón   — e Zd ZU eed<   ej                  Zej                  ed<   d„ Z	 	 	 	 	 	 dde	fd„Z
y)Ú$FlaxBeitForMaskedImageModelingModulerc   r,   c                 ó²  — t        | j                  d| j                  ¬«      | _        t	        j
                  | j                  j                  | j                  ¬«      | _        t	        j                  | j                  j                  t        j                  j                  j                  | j                  j                  «      | j                  ¬«      | _        y )NF)rK  r,   rè   rÈ   )rJ  rc   r,   r  r]   rí   rî   rD  r¬   Ú
vocab_sizerN   rq   rr   rs   Úlm_headr¯   s    r"   ru   z*FlaxBeitForMaskedImageModelingModule.setup  s…   € Ü" 4§;¡;À%ÈtÏzÉzÔZˆŒ	ô Ÿ™¨d¯k©k×.HÑ.HÐPT×PZÑPZÔ[ˆŒÜ—x‘xØ�K‰K×"Ñ"ÜŸ™×+Ñ+×2Ñ2°4·;±;×3PÑ3PÓQØ—*‘*ô
ˆ�r!   NrH   c                 ó"  — |�|n| j                   j                  }| j                  ||||||¬«      }|d   }| j                  |«      }| j	                  |d d …dd …f   «      }	|s|	f|dd  z   }
|
S t        |	|j                  |j                  ¬«      S )Nr  r   r   r&   ©Úlogitsr¹   r  )rc   Úuse_return_dictr  rD  rY  r   r¹   r  )rR   rx   r–   rH   r°   r  r  rÁ   Úsequence_outputÚprediction_scoresrX   s              r"   rY   z-FlaxBeitForMaskedImageModelingModule.__call__  s¯   € ð &1Ð%<‘kÀ$Ç+Á+×B]ÑB]ˆà—)‘)ØØØ'Ø/Ø!5Ø#ð ó 
ˆð " !™*ˆØŸ.™.¨Ó9ˆØ ŸL™L¨º¸A¹B¸Ñ)?Ó@ÐáØ'Ð)¨G°A°B¨KÑ7ˆFØˆMä!Ø$Ø!×/Ñ/Ø×)Ñ)ô
ð 	
r!   ©NNTNNNrÐ   r    r!   r"   rV  rV    sB   … ØÓØ—{‘{€Eˆ3�9‰9Ó"ò	
ð ØØ"ØØ!Øñ 
ð ô	 
r!   rV  zYBeit Model transformer with a 'language' modeling head on top (to predict visual tokens).c                   ó   — e Zd ZeZy)ÚFlaxBeitForMaskedImageModelingN)r   r   r   rV  r  r    r!   r"   rb  rb  9  s	   „ ð
 8�Lr!   rb  a?  
    bool_masked_pos (`numpy.ndarray` of shape `(batch_size, num_patches)`):
        Boolean masked positions. Indicates which patches are masked (1) and which aren't (0).

    Returns:

    Examples:

    ```python
    >>> from transformers import AutoImageProcessor, BeitForMaskedImageModeling
    >>> from PIL import Image
    >>> import requests

    >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
    >>> image = Image.open(requests.get(url, stream=True).raw)

    >>> image_processor = AutoImageProcessor.from_pretrained("microsoft/beit-base-patch16-224-pt22k")
    >>> model = BeitForMaskedImageModeling.from_pretrained("microsoft/beit-base-patch16-224-pt22k")

    >>> inputs = image_processor(images=image, return_tensors="np")
    >>> outputs = model(**inputs)
    >>> logits = outputs.logits
    ```
c                   ón   — e Zd ZU eed<   ej                  Zej                  ed<   d„ Z	 	 	 	 	 	 dde	fd„Z
y)Ú$FlaxBeitForImageClassificationModulerc   r,   c                 ó>  — t        | j                  | j                  d¬«      | _        t	        j
                  | j                  j                  t        j                  j                  j                  | j                  j                  «      | j                  ¬«      | _        y )NT)rc   r,   rK  rÈ   )rJ  rc   r,   r  r]   r¬   Ú
num_labelsrN   rq   rr   rs   Ú
classifierr¯   s    r"   ru   z*FlaxBeitForImageClassificationModule.setupd  sa   € Ü"¨$¯+©+¸T¿Z¹ZÐ[_Ô`ˆŒ	ÜŸ(™(Ø�K‰K×"Ñ"ÜŸ™×+Ñ+×2Ñ2°4·;±;×3PÑ3PÓQØ—*‘*ô
ˆ�r!   NrH   c                 óì   — |�|n| j                   j                  }| j                  |||||¬«      }|d   }| j                  |«      }	|s|	f|dd  z   }
|
S t	        |	|j
                  |j                  ¬«      S )Nr  r   r&   r[  )rc   r]  r  rg  r   r¹   r  )rR   rx   r–   rH   r°   r  r  rÁ   rH  r\  rX   s              r"   rY   z-FlaxBeitForImageClassificationModule.__call__l  s‘   € ð &1Ð%<‘kÀ$Ç+Á+×B]ÑB]ˆà—)‘)ØØ'Ø/Ø!5Ø#ð ó 
ˆð   ™
ˆØ—‘ Ó/ˆáØ�Y ¨¨ Ñ,ˆFØˆMä+ØØ!×/Ñ/Ø×)Ñ)ô
ð 	
r!   r`  rÐ   r    r!   r"   rd  rd  `  sB   … ØÓØ—{‘{€Eˆ3�9‰9Ó"ò
ð ØØ"ØØ!Øñ
ð ô	
r!   rd  z¶
    Beit Model transformer with an image classification head on top (a linear layer on top of the average of the final
    hidden states of the patch tokens) e.g. for ImageNet.
    c                   ó   — e Zd ZeZy)ÚFlaxBeitForImageClassificationN)r   r   r   rd  r  r    r!   r"   rj  rj  �  s	   „ ð 8�Lr!   rj  aM  
    Returns:

    Example:

    ```python
    >>> from transformers import AutoImageProcessor, FlaxBeitForImageClassification
    >>> from PIL import Image
    >>> import requests

    >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
    >>> image = Image.open(requests.get(url, stream=True).raw)

    >>> image_processor = AutoImageProcessor.from_pretrained("microsoft/beit-base-patch16-224")
    >>> model = FlaxBeitForImageClassification.from_pretrained("microsoft/beit-base-patch16-224")

    >>> inputs = image_processor(images=image, return_tensors="np")
    >>> outputs = model(**inputs)
    >>> logits = outputs.logits
    >>> # model predicts one of the 1000 ImageNet classes
    >>> predicted_class_idx = logits.argmax(-1).item()
    >>> print("Predicted class:", model.config.id2label[predicted_class_idx])
    ```
)rj  rb  rS  r  )IÚtypingr   r   r   r   ÚflaxÚ
flax.linenÚlinenr]   rN   Ú	jax.numpyÚnumpyr5   r-   Úflax.core.frozen_dictr   r   r	   Úflax.linen.attentionr
   Úflax.traverse_utilr   r   Úmodeling_flax_outputsr   r   r   r   Úmodeling_flax_utilsr   r   r   r   Úutilsr   r   Úconfiguration_beitr   ÚstructÚ	dataclassr   ÚBEIT_START_DOCSTRINGr<  r£   r  r>   r~   rD   r:  rF   rb   r€   rœ   r¥   rÅ   rÒ   rÚ   rá   rå   rù   r  r  rA  rJ  rS  ÚFLAX_BEIT_MODEL_DOCSTRINGrV  rb  ÚFLAX_BEIT_MLM_DOCSTRINGrd  rj  ÚFLAX_BEIT_CLASSIF_DOCSTRINGÚ__all__r    r!   r"   ú<module>r     s¹  ð÷" 3Ó 2ã Ý Û 
Ý Û ß >Ñ >Ý >ß ;÷ó ÷ó ÷ QÝ *ð ‡�×ÑôÐ%Có ó ðð,!Ð ðFÐ ð".¨e°C¸°H©oð .À#Ç+Á+ó .ð0 .1¯[©[ó *ô�r—y‘yô ô*C˜bŸi™iô Cô@&˜Ÿ™ô &ôR@ 2§9¡9ô @ô.R˜BŸI™Iô Rôj˜Ÿ™ô ô$˜Ÿ	™	ô ô4˜2Ÿ9™9ô ô&�R—Y‘Yô ô&:�B—I‘Iô :ôz4
˜bŸi™iô 4
ôn%
�b—i‘iô %
ôPP
Ð1ô P
ôf�R—Y‘Yô ô(0
�R—Y‘Yô 0
ñf ØdØóô"Ð+ó "ó	ð"ðÐ ñ, ˜Ð(AÔ BÙ   Ð<ZÐisÕ tô/
¨2¯9©9ô /
ñd Ø_Øóô8Ð%<ó 8ó	ð8ðÐ ñ2 Ð7Ð9PÔ QÙ  Ø"Ð0BÐQ[õô
*
¨2¯9©9ô *
ñZ ðð óô8Ð%<ó 8óð8ðÐ ñ2 Ð7Ð9TÔ UÙ  Ø"Ð0LÐ[eõò
�r!   