Ë
    S^(hLy  ã                   ó  — d Z ddlZddlZddlmZmZ ddlmZ	 ddl
Z
ddlmZ ddlmZmZmZ ddlmZ ddlmZmZ ddlmZmZmZ dd	lmZmZmZmZ dd
l m!Z!m"Z" ddl#m$Z$ dZ%dZ& G d„ de	jN                  «      Z( G d„ de	jN                  «      Z) G d„ de	jN                  «      Z* G d„ de	jN                  «      Z+ G d„ de	jN                  «      Z,ejZ                  fd„Z. G d„ de	jN                  «      Z/ G d„ de	jN                  «      Z0 G d„ de	jN                  «      Z1 G d „ d!e	jN                  «      Z2 G d"„ d#e	jN                  «      Z3 G d$„ d%e	jN                  «      Z4 G d&„ d'e	jN                  «      Z5 G d(„ d)e«      Z6 G d*„ d+e	jN                  «      Z7 e!d,e%«       G d-„ d.e6«      «       Z8d/Z9 ee8e9«        ee8ee$¬0«        G d1„ d2e	jN                  «      Z: e!d3e%«       G d4„ d5e6«      «       Z;d6Z< ee;e<«        ee;ee$¬0«       g d7¢Z=y)8zFlax DINOv2 model.é    N)ÚOptionalÚTuple)Ú
FrozenDictÚfreezeÚunfreeze)Údot_product_attention_weights)Úflatten_dictÚunflatten_dicté   )ÚFlaxBaseModelOutputÚFlaxBaseModelOutputWithPoolingÚFlaxSequenceClassifierOutput)ÚACT2FNÚFlaxPreTrainedModelÚ append_replace_return_docstringsÚoverwrite_call_docstring)Úadd_start_docstringsÚ%add_start_docstrings_to_model_forwardé   )ÚDinov2Configaþ  

    This model inherits from [`FlaxPreTrainedModel`]. Check the superclass documentation for the generic methods the
    library implements for all its model (such as downloading, saving and converting weights from PyTorch models)

    This model is also a
    [flax.linen.Module](https://flax.readthedocs.io/en/latest/api_reference/flax.linen/module.html) subclass. Use it as
    a regular Flax linen Module and refer to the Flax documentation for all matter related to general usage and
    behavior.

    Finally, this model supports inherent JAX features such as:

    - [Just-In-Time (JIT) compilation](https://jax.readthedocs.io/en/latest/jax.html#just-in-time-compilation-jit)
    - [Automatic Differentiation](https://jax.readthedocs.io/en/latest/jax.html#automatic-differentiation)
    - [Vectorization](https://jax.readthedocs.io/en/latest/jax.html#vectorization-vmap)
    - [Parallelization](https://jax.readthedocs.io/en/latest/jax.html#parallelization-pmap)

    Parameters:
        config ([`Dinov2Config`]): Model configuration class with all the parameters of the model.
            Initializing with a config file does not load the weights associated with the model, only the
            configuration. Check out the [`~FlaxPreTrainedModel.from_pretrained`] method to load the model weights.
        dtype (`jax.numpy.dtype`, *optional*, defaults to `jax.numpy.float32`):
            The data type of the computation. Can be one of `jax.numpy.float32`, `jax.numpy.float16` (on GPUs) and
            `jax.numpy.bfloat16` (on TPUs).

            This can be used to enable mixed-precision training or half-precision inference on GPUs or TPUs. If
            specified all the computation will be performed with the given `dtype`.

            **Note that this only specifies the dtype of the computation and does not influence the dtype of model
            parameters.**

            If you wish to change the dtype of the model parameters, see [`~FlaxPreTrainedModel.to_fp16`] and
            [`~FlaxPreTrainedModel.to_bf16`].
a
  
    Args:
        pixel_values (`numpy.ndarray` of shape `(batch_size, num_channels, height, width)`):
            Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See [`Dinov2ImageProcessor.__call__`]
            for details.

        output_attentions (`bool`, *optional*):
            Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
            tensors for more detail.
        output_hidden_states (`bool`, *optional*):
            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
            more detail.
        return_dict (`bool`, *optional*):
            Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
c                   óZ   — e Zd ZU eed<   ej                  Zej                  ed<   d„ Zd„ Z	y)ÚFlaxDinov2PatchEmbeddingsÚconfigÚdtypec                 ór  — | j                   j                  }| j                   j                  }t        |t        j
                  j                  «      r|n||f}t        |t        j
                  j                  «      r|n||f}|d   |d   z  |d   |d   z  z  }|| _        | j                   j                  | _        t        j                  | j                   j                  ||d| j                  t        j                  j                  j                  | j                   j                   dz  dd«      ¬«      | _        y )Nr   r   ÚVALIDé   Úfan_inÚtruncated_normal)Úkernel_sizeÚstridesÚpaddingr   Úkernel_init)r   Ú
image_sizeÚ
patch_sizeÚ
isinstanceÚcollectionsÚabcÚIterableÚnum_patchesÚnum_channelsÚnnÚConvÚhidden_sizer   ÚjaxÚinitializersÚvariance_scalingÚinitializer_rangeÚ
projection)Úselfr$   r%   r*   s       úm/var/www/skyplay_api_hub/venv/lib/python3.12/site-packages/transformers/models/dinov2/modeling_flax_dinov2.pyÚsetupzFlaxDinov2PatchEmbeddings.setup_   sý   € Ø—[‘[×+Ñ+ˆ
Ø—[‘[×+Ñ+ˆ
Ü#-¨j¼+¿/¹/×:RÑ:RÔ#S‘ZÐZdÐfpÐYqˆ
Ü#-¨j¼+¿/¹/×:RÑ:RÔ#S‘ZÐZdÐfpÐYqˆ
Ø! !‘}¨
°1©Ñ5¸*ÀQ¹-È:ÐVWÉ=Ñ:XÑYˆà&ˆÔØ ŸK™K×4Ñ4ˆÔÜŸ'™'Ø�K‰K×#Ñ#Ø"ØØØ—*‘*ÜŸ™×+Ñ+×<Ñ<Ø—‘×-Ñ-¨qÑ0°(Ð<Nóô	
ˆ�ó    c                 óÊ   — |j                   d   }|| j                  k7  rt        d«      ‚| j                  |«      }|j                   \  }}}}t	        j
                  ||d|f«      S )NéÿÿÿÿzeMake sure that the channel dimension of the pixel values match with the one set in the configuration.)Úshaper+   Ú
ValueErrorr3   ÚjnpÚreshape)r4   Úpixel_valuesr+   Ú
embeddingsÚ
batch_sizeÚ_Úchannelss          r5   Ú__call__z"FlaxDinov2PatchEmbeddings.__call__t   sl   € Ø#×)Ñ)¨"Ñ-ˆØ˜4×,Ñ,Ò,ÜØwóð ð —_‘_ \Ó2ˆ
Ø%/×%5Ñ%5Ñ"ˆ
�A�q˜(Ü�{‰{˜:¨
°B¸Ð'AÓBÐBr7   N©
Ú__name__Ú
__module__Ú__qualname__r   Ú__annotations__r<   Úfloat32r   r6   rC   © r7   r5   r   r   [   s%   … ØÓØ—{‘{€Eˆ3�9‰9Ó"ò
ó*Cr7   r   c                   óf   — e Zd ZU dZeed<   ej                  Zej                  ed<   d„ Z	d„ Z
dd„Zy)	ÚFlaxDinov2Embeddingsz7Construct the CLS token, position and patch embeddings.r   r   c                 ó¤  — | j                  dt        j                  j                  j	                  | j
                  j                  dz  dd«      dd| j
                  j                  f«      | _        | j
                  j                  rn| j                  dt        j                  j                  j	                  | j
                  j                  dz  dd«      d| j
                  j                  f«      | _
        t        | j
                  | j                  ¬«      | _        | j                  j                  }| j                  dt        j                  j                  j	                  | j
                  j                  dz  dd«      d|dz   | j
                  j                  f«      | _        t        j                   | j
                  j"                  ¬	«      | _        y )
NÚ	cls_tokenr   r   r   r   Ú
mask_token©r   Úposition_embeddings©Úrate)Úparamr/   r,   r0   r1   r   r2   r.   rN   Úuse_mask_tokenrO   r   r   Úpatch_embeddingsr*   rQ   ÚDropoutÚhidden_dropout_probÚdropout)r4   r*   s     r5   r6   zFlaxDinov2Embeddings.setup…   sY  € ØŸ™ØÜ�F‰F×Ñ×0Ñ0°·±×1NÑ1NÐPQÑ1QÐS[Ð]oÓpØ��4—;‘;×*Ñ*Ð+ó
ˆŒð
 �;‰;×%Ò%Ø"Ÿj™jØÜ—‘×#Ñ#×4Ñ4°T·[±[×5RÑ5RÐTUÑ5UÐW_ÐasÓtØ�D—K‘K×+Ñ+Ð,óˆDŒOô
 !:¸$¿+¹+ÈTÏZÉZÔ XˆÔØ×+Ñ+×7Ñ7ˆØ#'§:¡:Ø!Ü�F‰F×Ñ×0Ñ0°·±×1NÑ1NÐPQÑ1QÐS[Ð]oÓpØ�˜a‘ §¡×!8Ñ!8Ð9ó$
ˆÔ ô
 —z‘z t§{¡{×'FÑ'FÔGˆ�r7   c           	      ó6  — |j                   d   dz
  }|j                   d   dz
  }||k(  r||k(  r|S |d d …df   }|d d …dd …f   }	|j                   d   }
||j                  z  }||j                  z  }|dz   |dz   }}|	j                  dt        t	        j
                  |«      «      t        t	        j
                  |«      «      |
f«      }	t        j                  |	d«      }	|	j                  }t        j                  |t	        j
                  |«      z  «      }t        j                  |t	        j
                  |«      z  «      }t        j                  ||gt        j                  ¬«      }t        j                  ddgt        j                  ¬«      }t        j                  j                  |	j                  t        j                  «      |	j                   d   |	j                   d   ||fd||d	d
¬«      }	|	j                  |«      }	t        j                  |	d«      j                  |j                   d   d|
f«      }	t        j                  |	|j                   d   ddf«      }t        j                  ||j                   d   ddf«      }t        j                   ||fd¬«      S )Nr   r   r9   gš™™™™™¹?)r   r   r   r   rP   ç        )r   r   ÚbicubicF)r:   Úspatial_dimsÚscaleÚtranslationÚmethodÚ	antialias©r   r   r   r   ©Úaxis)r:   r%   r=   ÚintÚmathÚsqrtr<   Ú	transposer   rI   Úarrayr/   ÚimageÚscale_and_translateÚastypeÚtileÚconcatenate)r4   r   Úhidden_statesÚheightÚwidthrQ   r*   Únum_positionsÚclass_pos_embedÚpatch_pos_embedÚdimÚhÚwÚtarget_dtypeÚnew_height_ratioÚnew_width_ratior^   r_   Úpatch_pos_embed_expandedÚclass_pos_embed_expandeds                       r5   Úinterpolate_pos_encodingz-FlaxDinov2Embeddings.interpolate_pos_encodingš   sc  € Ø#×)Ñ)¨!Ñ,¨qÑ0ˆØ+×1Ñ1°!Ñ4°qÑ8ˆØ˜-Ò'¨F°eªOØ&Ð&Ø-ªa°¨dÑ3ˆØ-ªa°±¨eÑ4ˆØ×!Ñ! "Ñ%ˆà�f×'Ñ'Ñ'ˆØ�V×&Ñ&Ñ&ˆØ˜C™  S¡�ˆà)×1Ñ1Ø””D—I‘I˜mÓ,Ó-¬s´4·9±9¸]Ó3KÓ/LÈcÐRó
ˆô Ÿ-™-¨¸ÓFˆØ&×,Ñ,ˆÜŸ;™; v´·	±	¸-Ó0HÑ'HÓIÐÜŸ+™+ e¬d¯i©i¸Ó.FÑ&FÓGˆä—	‘	Ð+¨_Ð=ÄSÇ[Á[ÔQˆÜ—i‘i  c 
´#·+±+Ô>ˆäŸ)™)×7Ñ7Ø×"Ñ"¤3§;¡;Ó/Ø"×(Ñ(¨Ñ+¨_×-BÑ-BÀ1Ñ-EÀqÈ!ÐLØØØ#ØØð 8ó 
ˆð *×0Ñ0°Ó>ˆÜŸ-™-¨¸ÓF×NÑNÐPc×PiÑPiÐjkÑPlÐnpÐruÐOvÓwˆÜ#&§8¡8¨O¸m×>QÑ>QÐRSÑ>TÐVWÐYZÐ=[Ó#\Ð Ü#&§8¡8¨O¸m×>QÑ>QÐRSÑ>TÐVWÐYZÐ=[Ó#\Ð ä�‰Ð 8Ð:RÐSÐZ[Ô\Ð\r7   c                 óþ  — |j                   d   }| j                  j                  j                  }|j                   d   |j                   d   }}| j                  |j	                  |«      «      }t        j                  | j                  |d| j                  j                  f«      }t        j                  ||fd¬«      }|| j                  | j                  |||| j                  «      z   }| j                  ||¬«      }|S )Nr   r   r   rc   ©Údeterministic)r:   rV   r3   r   rl   r<   Úbroadcast_torN   r   r.   rn   r}   rQ   rY   )	r4   r>   r€   r@   rx   rp   rq   r?   Ú
cls_tokenss	            r5   rC   zFlaxDinov2Embeddings.__call__Â   så   € Ø!×'Ñ'¨Ñ*ˆ
Ø×,Ñ,×7Ñ7×=Ñ=ˆØ$×*Ñ*¨1Ñ-¨|×/AÑ/AÀ!Ñ/D�ˆà×*Ñ*¨<×+>Ñ+>¸|Ó+LÓMˆ
ä×%Ñ% d§n¡n°zÀ1ÀdÇkÁk×F]ÑF]Ð6^Ó_ˆ
Ü—_‘_ j°*Ð%=ÀAÔFˆ
à $×"?Ñ"?Ø�K‰K˜ V¨U°D×4LÑ4Ló#
ñ 
ˆ
ð —\‘\ *¸M�\ÓJˆ
ØÐr7   N©T)rE   rF   rG   Ú__doc__r   rH   r<   rI   r   r6   r}   rC   rJ   r7   r5   rL   rL      s/   … ÙAàÓØ—{‘{€Eˆ3�9‰9Ó"òHò*&]ôPr7   rL   c                   óf   — e Zd ZU eed<   ej                  Zej                  ed<   d„ Zdde	de	fd„Z
y)	ÚFlaxDinov2SelfAttentionr   r   c           	      óà  — | j                   j                  | j                   j                  z  dk7  rt        d«      ‚t	        j
                  | j                   j                  | j                  t        j                  j                  j                  | j                   j                  dz  dd¬«      | j                   j                  ¬«      | _        t	        j
                  | j                   j                  | j                  t        j                  j                  j                  | j                   j                  dz  dd¬«      | j                   j                  ¬«      | _        t	        j
                  | j                   j                  | j                  t        j                  j                  j                  | j                   j                  dz  dd¬«      | j                   j                  ¬«      | _        y )Nr   z‡`config.hidden_size`: {self.config.hidden_size} has to be a multiple of `config.num_attention_heads`: {self.config.num_attention_heads}r   r   r   )ÚmodeÚdistribution)r   r#   Úuse_bias)r   r.   Únum_attention_headsr;   r,   ÚDenser   r/   r0   r1   r2   Úqkv_biasÚqueryÚkeyÚvalue©r4   s    r5   r6   zFlaxDinov2SelfAttention.setupÙ   sp  € Ø�;‰;×"Ñ" T§[¡[×%DÑ%DÑDÈÒIÜð5óð ô
 —X‘XØ�K‰K×#Ñ#Ø—*‘*ÜŸ™×+Ñ+×<Ñ<Ø—‘×-Ñ-¨qÑ0°xÐN`ð =ó ð —[‘[×)Ñ)ô
ˆŒ
ô —8‘8Ø�K‰K×#Ñ#Ø—*‘*ÜŸ™×+Ñ+×<Ñ<Ø—‘×-Ñ-¨qÑ0°xÐN`ð =ó ð —[‘[×)Ñ)ô
ˆŒô —X‘XØ�K‰K×#Ñ#Ø—*‘*ÜŸ™×+Ñ+×<Ñ<Ø—‘×-Ñ-¨qÑ0°xÐN`ð =ó ð —[‘[×)Ñ)ô
ˆ�
r7   r€   Úoutput_attentionsc           
      óH  — | j                   j                  | j                   j                  z  }| j                  |«      j	                  |j
                  d d | j                   j                  |fz   «      }| j                  |«      j	                  |j
                  d d | j                   j                  |fz   «      }| j                  |«      j	                  |j
                  d d | j                   j                  |fz   «      }d }|s*| j                   j                  dkD  r| j                  d«      }t        |||| j                   j                  d|| j                  d ¬«      }	t        j                  d|	|«      }
|
j	                  |
j
                  d d dz   «      }
|r|
|	f}|S |
f}|S )Nr   r[   rY   T)Údropout_rngÚdropout_rateÚbroadcast_dropoutr€   r   Ú	precisionz...hqk,...khd->...qhd)r9   )r   r.   r‹   rŽ   r=   r:   r�   r�   Úattention_probs_dropout_probÚmake_rngr   r   r<   Úeinsum)r4   ro   r€   r’   Úhead_dimÚquery_statesÚvalue_statesÚ
key_statesr”   Úattn_weightsÚattn_outputÚoutputss               r5   rC   z FlaxDinov2SelfAttention.__call__ù   sŒ  € Ø—;‘;×*Ñ*¨d¯k©k×.MÑ.MÑMˆà—z‘z -Ó0×8Ñ8Ø×Ñ  Ð# t§{¡{×'FÑ'FÈÐ&QÑQó
ˆð —z‘z -Ó0×8Ñ8Ø×Ñ  Ð# t§{¡{×'FÑ'FÈÐ&QÑQó
ˆð —X‘X˜mÓ,×4Ñ4Ø×Ñ  Ð# t§{¡{×'FÑ'FÈÐ&QÑQó
ˆ
ð ˆÙ §¡×!IÑ!IÈCÒ!OØŸ-™-¨	Ó2ˆKä4ØØØ#ØŸ™×AÑAØ"Ø'Ø—*‘*Øô	
ˆô —j‘jÐ!8¸,ÈÓUˆØ!×)Ñ)¨+×*;Ñ*;¸B¸QÐ*?À%Ñ*GÓHˆá1B�; Ð-ˆØˆð JUÈˆØˆr7   N©TF©rE   rF   rG   r   rH   r<   rI   r   r6   ÚboolrC   rJ   r7   r5   r†   r†   Õ   s4   … ØÓØ—{‘{€Eˆ3�9‰9Ó"ò
ñ@ °Tð  ÐUYô  r7   r†   c                   ób   — e Zd ZU eed<   ej                  Zej                  ed<   d„ Zdde	fd„Z
y)ÚFlaxDinov2SelfOutputr   r   c                 óX  — t        j                  | j                  j                  t        j                   j
                  j                  | j                  j                  dz  dd«      | j                  ¬«      | _	        t        j                  | j                  j                  ¬«      | _        y )Nr   r   r   ©r#   r   rR   )r,   rŒ   r   r.   r/   r0   r1   r2   r   ÚdenserW   rX   rY   r‘   s    r5   r6   zFlaxDinov2SelfOutput.setup!  ss   € Ü—X‘XØ�K‰K×#Ñ#ÜŸ™×+Ñ+×<Ñ<Ø—‘×-Ñ-¨qÑ0°(Ð<Nóð —*‘*ô
ˆŒ
ô —z‘z t§{¡{×'FÑ'FÔGˆ�r7   r€   c                 óN   — | j                  |«      }| j                  ||¬«      }|S )Nr   )r©   rY   )r4   ro   Úinput_tensorr€   s       r5   rC   zFlaxDinov2SelfOutput.__call__+  s(   € ØŸ
™
 =Ó1ˆØŸ™ ]À-˜ÓPˆØÐr7   Nrƒ   r£   rJ   r7   r5   r¦   r¦     s,   … ØÓØ—{‘{€Eˆ3�9‰9Ó"òHñÀ4ô r7   r¦   c                   ób   — e Zd ZU eed<   ej                  Zej                  ed<   d„ Zdde	fd„Z
y)ÚFlaxDinov2Attentionr   r   c                 óœ   — t        | j                  | j                  ¬«      | _        t	        | j                  | j                  ¬«      | _        y ©NrP   )r†   r   r   Ú	attentionr¦   Úoutputr‘   s    r5   r6   zFlaxDinov2Attention.setup6  s.   € Ü0°·±ÀDÇJÁJÔOˆŒÜ*¨4¯;©;¸d¿j¹jÔIˆ�r7   r’   c                 ó|   — | j                  |||¬«      }|d   }| j                  |||¬«      }|f}|r	||d   fz  }|S )N©r€   r’   r   r   r   )r°   r±   )r4   ro   r€   r’   Úattn_outputsr    r¡   s          r5   rC   zFlaxDinov2Attention.__call__:  sU   € Ø—~‘~ mÀ=Ðdu�~ÓvˆØ" 1‘oˆØŸ™ K°Èm˜Ó\ˆà Ð"ˆáØ˜ Q™Ð)Ñ)ˆGàˆr7   Nr¢   r£   rJ   r7   r5   r­   r­   2  s,   … ØÓØ—{‘{€Eˆ3�9‰9Ó"òJñ
ÈTô 
r7   r­   c                 ó4   — t        j                  ||«      |z  S ©N)r<   Úones)r�   r:   r^   r   s       r5   Úones_with_scaler¸   G  s   € Ü�8‰8�E˜5Ó! EÑ)Ð)r7   c                   óZ   — e Zd ZU eed<   ej                  Zej                  ed<   d„ Zd„ Z	y)ÚFlaxDinov2LayerScaler   r   c                 ó   — | j                   j                  | j                  dt        j                  j
                  j                  | j                   j                  f«      z  | _        | j                  | j                   j                  z  | _        y )NÚlambda1)	r   Úlayerscale_valuerT   r/   r,   r0   r·   r.   r¼   r‘   s    r5   r6   zFlaxDinov2LayerScale.setupO  se   € Ø—{‘{×3Ñ3°d·j±jØÜ�F‰F×Ñ×$Ñ$Ø�[‰[×$Ñ$Ð&ó7
ñ 
ˆŒð
 —|‘| d§k¡k×&BÑ&BÑBˆ�r7   c                 ó    — | j                   |z  S r¶   )r¼   ©r4   ro   s     r5   rC   zFlaxDinov2LayerScale.__call__W  s   € Ø�|‰|˜mÑ+Ð+r7   NrD   rJ   r7   r5   rº   rº   K  s%   … ØÓØ—{‘{€Eˆ3�9‰9Ó"òCó,r7   rº   c                   ób   — e Zd ZU dZeed<   ej                  j                  dde	e
   fd„«       Zy)ÚFlaxDinov2DropPathzXDrop paths (Stochastic Depth) per sample (when applied in main path of residual blocks).rS   r€   c                 óN  — | j                   dk(  r|S d| j                   z
  }|r|S |j                  d   fd|j                  dz
  z  z   }| j                  d«      }|t        j
                  j                  |||j                  ¬«      z   }t        j                  |«      }||z  |z  }|S )Nr[   g      ð?r   )r   r   Údroppath)r:   r   )
rS   r:   Úndimr™   r/   ÚrandomÚuniformr   r<   Úfloor)	r4   Úinputsr€   Ú	keep_probr:   ÚrngÚrandom_tensorÚbinary_tensorr±   s	            r5   rC   zFlaxDinov2DropPath.__call__a  s¡   € à�9‰9˜ÒØˆMØ˜$Ÿ)™)‘Oˆ	ÙØˆMà—\‘\ !‘_Ð&¨°·±¸q±Ñ)AÑAˆEØ—-‘- 
Ó+ˆCØ%¬¯
©
×(:Ñ(:¸3ÀeÐSY×S_ÑS_Ð(:Ó(`Ñ`ˆMÜŸI™I mÓ4ˆMØ˜iÑ'¨-Ñ7ˆFØˆMr7   Nrƒ   )rE   rF   rG   r„   ÚfloatrH   r,   ÚmoduleÚcompactr   r¤   rC   rJ   r7   r5   rÁ   rÁ   \  s1   … Ùbà
ƒKà‡Y�Y×Ññ¨h°t©nò ó ñr7   rÁ   c                   óZ   — e Zd ZU eed<   ej                  Zej                  ed<   d„ Zd„ Z	y)ÚFlaxDinov2MLPr   r   c                 óâ  — t        j                  | j                  j                  | j                  j                  z  t
        j                   j                  j                  | j                  j                  dz  dd«      | j                  ¬«      | _
        t        j                  | j                  j                  t
        j                   j                  j                  | j                  j                  dz  dd«      | j                  ¬«      | _        t        | j                  j                  t        «      r#t        | j                  j                     | _        y | j                  j                  | _        y )Nr   r   r   r¨   )r,   rŒ   r   r.   Ú	mlp_ratior/   r0   r1   r2   r   Úfc1Úfc2r&   Ú
hidden_actÚstrr   Úactr‘   s    r5   r6   zFlaxDinov2MLP.setupu  sú   € Ü—8‘8Ø�K‰K×#Ñ# d§k¡k×&;Ñ&;Ñ;ÜŸ™×+Ñ+×<Ñ<Ø—‘×-Ñ-¨qÑ0°(Ð<Nóð —*‘*ô
ˆŒô —8‘8Ø�K‰K×#Ñ#ÜŸ™×+Ñ+×<Ñ<Ø—‘×-Ñ-¨qÑ0°(Ð<Nóð —*‘*ô
ˆŒô �d—k‘k×,Ñ,¬cÔ2Ü˜dŸk™k×4Ñ4Ñ5ˆD�Hà—{‘{×-Ñ-ˆD�Hr7   c                 ól   — | j                  |«      }| j                  |«      }| j                  |«      }|S r¶   )rÔ   rØ   rÕ   r¿   s     r5   rC   zFlaxDinov2MLP.__call__‰  s2   € ØŸ™ Ó/ˆØŸ™ Ó/ˆØŸ™ Ó/ˆØÐr7   NrD   rJ   r7   r5   rÑ   rÑ   q  s$   … ØÓØ—{‘{€Eˆ3�9‰9Ó"ò.ó(r7   rÑ   c                   óZ   — e Zd ZU eed<   ej                  Zej                  ed<   d„ Zd„ Z	y)ÚFlaxDinov2SwiGLUFFNr   r   c                 ó‚  — t        | j                  j                  | j                  j                  z  «      }t        | j                  dz  dz  «      dz   dz  dz  }t        j                  d|z  t        j
                  j                  j                  | j                  j                  dz  dd«      | j                  ¬«      | _        t        j                  | j                  j                  t        j
                  j                  j                  | j                  j                  dz  dd«      | j                  ¬«      | _        y )Nr   r   é   é   r   r   r¨   )re   r   r.   rÓ   Úhidden_featuresr,   rŒ   r/   r0   r1   r2   r   Ú
weights_inÚweights_out)r4   rß   s     r5   r6   zFlaxDinov2SwiGLUFFN.setup”  sö   € Ü˜dŸk™k×5Ñ5¸¿¹×8MÑ8MÑMÓNˆÜ˜t×3Ñ3°aÑ7¸!Ñ;Ó<¸qÑ@ÀQÑFÈÑJˆäŸ(™(Ø�ÑÜŸ™×+Ñ+×<Ñ<Ø—‘×-Ñ-¨qÑ0°(Ð<Nóð —*‘*ô
ˆŒô Ÿ8™8Ø�K‰K×#Ñ#ÜŸ™×+Ñ+×<Ñ<Ø—‘×-Ñ-¨qÑ0°(Ð<Nóð —*‘*ô
ˆÕr7   c                 ó¬   — | j                  |«      }t        j                  |dd¬«      \  }}t        j                  |«      |z  }| j                  |«      S )Nr   r9   rc   )rà   r<   Úsplitr,   Úsilurá   )r4   ro   Úx1Úx2Úhiddens        r5   rC   zFlaxDinov2SwiGLUFFN.__call__§  sI   € ØŸ™¨Ó6ˆÜ—‘˜=¨!°"Ô5‰ˆˆBÜ—‘˜“˜rÑ!ˆØ×Ñ Ó'Ð'r7   NrD   rJ   r7   r5   rÛ   rÛ   �  s$   … ØÓØ—{‘{€Eˆ3�9‰9Ó"ò
ó&(r7   rÛ   c                   óf   — e Zd ZU eed<   ej                  Zej                  ed<   d„ Zdde	de	fd„Z
y)	ÚFlaxDinov2Layerr   r   c                 óÞ  — t        j                  | j                  j                  | j                  ¬«      | _        t        | j                  | j                  ¬«      | _        t        | j                  | j                  ¬«      | _	        t        | j                  j                  «      | _        t        j                  | j                  j                  | j                  ¬«      | _        | j                  j                  r't        | j                  | j                  ¬«      | _        n&t#        | j                  | j                  ¬«      | _        t        | j                  | j                  ¬«      | _        y )N©Úepsilonr   rP   )r,   Ú	LayerNormr   Úlayer_norm_epsr   Únorm1r­   r°   rº   Úlayer_scale1rÁ   Údrop_path_rateÚ	drop_pathÚnorm2Úuse_swiglu_ffnrÛ   ÚmlprÑ   Úlayer_scale2r‘   s    r5   r6   zFlaxDinov2Layer.setup²  sÕ   € Ü—\‘\¨$¯+©+×*DÑ*DÈDÏJÉJÔWˆŒ
Ü,¨T¯[©[ÀÇ
Á
ÔKˆŒÜ0°·±ÀDÇJÁJÔOˆÔÜ+¨D¯K©K×,FÑ,FÓGˆŒÜ—\‘\¨$¯+©+×*DÑ*DÈDÏJÉJÔWˆŒ
à�;‰;×%Ò%Ü*¨4¯;©;¸d¿j¹jÔIˆD�Hä$ T§[¡[¸¿
¹
ÔCˆDŒHä0°·±ÀDÇJÁJÔOˆÕr7   r€   r’   c                 óD  — | j                  | j                  |«      ||¬«      }|d   }| j                  |«      }|dd  }| j                  |«      |z   }| j	                  |«      }| j                  |«      }| j                  |«      }| j                  |«      |z   }|f|z   }|S )Nr³   r   r   )r°   rï   rð   rò   ró   rõ   rö   )r4   ro   r€   r’   Úself_attention_outputsÚattention_outputr¡   Úlayer_outputs           r5   rC   zFlaxDinov2Layer.__call__À  s½   € Ø!%§¡Ø�J‰J�}Ó%Ø'Ø/ð "0ó "
Ðð 2°!Ñ4Ðà×,Ñ,Ð-=Ó>Ðà(¨¨Ð,ˆð Ÿ™Ð'7Ó8¸=ÑHˆð —z‘z -Ó0ˆØ—x‘x Ó-ˆØ×(Ñ(¨Ó6ˆð —~‘~ lÓ3°mÑCˆà�/ GÑ+ˆàˆr7   Nr¢   r£   rJ   r7   r5   ré   ré   ®  s4   … ØÓØ—{‘{€Eˆ3�9‰9Ó"òPñ°Tð ÐUYô r7   ré   c            	       óv   — e Zd ZU eed<   ej                  Zej                  ed<   d„ Z	 	 	 	 d
de	de	de	de	fd„Z
y	)ÚFlaxDinov2LayerCollectionr   r   c           	      óÄ   — t        | j                  j                  «      D �cg c]-  }t        | j                  t	        |«      | j
                  ¬«      ‘Œ/ c}| _        y c c}w )N)Únamer   )Úranger   Únum_hidden_layersré   r×   r   Úlayers)r4   Úis     r5   r6   zFlaxDinov2LayerCollection.setupâ  sE   € äQVÐW[×WbÑWb×WtÑWtÓQuö
ØLMŒO˜DŸK™K¬c°!«f¸D¿J¹JÖGò
ˆ�ùò 
s   ¢2Ar€   r’   Úoutput_hidden_statesÚreturn_dictc                 óö   — |rdnd }|rdnd }t        | j                  «      D ])  \  }}	|r||fz  } |	|||¬«      }
|
d   }|sŒ!||
d   fz  }Œ+ |r||fz  }|f}|st        d„ |D «       «      S t        |||¬«      S )NrJ   r³   r   r   c              3   ó&   K  — | ]	  }|€Œ|–— Œ y ­wr¶   rJ   )Ú.0Úvs     r5   ú	<genexpr>z5FlaxDinov2LayerCollection.__call__.<locals>.<genexpr>  s   è ø€ Ò=˜q¨q©}œÑ=ùs   ‚Š)Úlast_hidden_statero   Ú
attentions)Ú	enumerater  Útupler   )r4   ro   r€   r’   r  r  Úall_attentionsÚall_hidden_statesr  ÚlayerÚlayer_outputsr¡   s               r5   rC   z"FlaxDinov2LayerCollection.__call__ç  s·   € ñ  1™°dˆÙ"6™B¸DÐä! $§+¡+Ó.ò 		6‰HˆAˆuÙ#Ø! mÐ%5Ñ5Ð!á! -¸}Ð`qÔrˆMà)¨!Ñ,ˆMâ Ø =°Ñ#3Ð"5Ñ5‘ð		6ñ  Ø -Ð!1Ñ1Ðà Ð"ˆÙÜÑ= GÔ=Ó=Ð=ä"Ø+Ð;LÐYgô
ð 	
r7   N©TFFTr£   rJ   r7   r5   rü   rü   Þ  sZ   … ØÓØ—{‘{€Eˆ3�9‰9Ó"ò
ð #Ø"'Ø%*Ø ñ
ð ð
ð  ð	
ð
 #ð
ð ô
r7   rü   c            	       óv   — e Zd ZU eed<   ej                  Zej                  ed<   d„ Z	 	 	 	 d
de	de	de	de	fd„Z
y	)ÚFlaxDinov2Encoderr   r   c                 óP   — t        | j                  | j                  ¬«      | _        y r¯   )rü   r   r   r  r‘   s    r5   r6   zFlaxDinov2Encoder.setup  s   € Ü.¨t¯{©{À$Ç*Á*ÔMˆ�
r7   r€   r’   r  r  c                 ó.   — | j                  |||||¬«      S )N©r€   r’   r  r  )r  )r4   ro   r€   r’   r  r  s         r5   rC   zFlaxDinov2Encoder.__call__  s)   € ð �z‰zØØ'Ø/Ø!5Ø#ð ó 
ð 	
r7   Nr  r£   rJ   r7   r5   r  r  
  s[   … ØÓØ—{‘{€Eˆ3�9‰9Ó"òNð #Ø"'Ø%*Ø ñ
ð ð
ð  ð	
ð
 #ð
ð ô
r7   r  c                   ót  ‡ — e Zd ZU dZeZdZdZdZe	j                  ed<   ddej                  dfded	ed
ej                  defˆ fd„Zddej&                  j(                  dededefd„Z eej5                  d«      «      	 	 	 	 	 	 ddedej&                  j(                  dedee   dee   dee   fd„«       Zˆ xZS )ÚFlaxDinov2PreTrainedModelz†
    An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained
    models.
    Údinov2r>   NÚmodule_classr   Tr   Úseedr   Ú_do_initc                 ó¦   •—  | j                   d||dœ|¤Ž}|€$d|j                  |j                  |j                  f}t        ‰| �  ||||||¬«       y )N©r   r   r   )Úinput_shaper  r   r  rJ   )r  r$   r+   ÚsuperÚ__init__)	r4   r   r   r  r   r  ÚkwargsrÎ   Ú	__class__s	           €r5   r"  z"FlaxDinov2PreTrainedModel.__init__-  sc   ø€ ð #�×"Ñ"ÐH¨&¸ÑHÀÑHˆØÐØ˜f×/Ñ/°×1BÑ1BÀF×DWÑDWÐXˆKÜ‰Ñ˜ °[ÀtÐSXÐckÐÕlr7   rÊ   r   ÚparamsÚreturnc                 óê  — t        j                  || j                  ¬«      }t        j                  j                  |«      \  }}t        j                  j                  |«      \  }}|||dœ}| j                  j                  ||d¬«      d   }	|�dt        t        |	«      «      }	t        t        |«      «      }| j                  D ]
  }
|	|
   ||
<   Œ t        «       | _
        t        t        |«      «      S |	S )NrP   )r%  rY   rÃ   F)r  r%  )r<   Úzerosr   r/   rÅ   rã   rÎ   Úinitr	   r   Ú_missing_keysÚsetr   r
   )r4   rÊ   r   r%  r>   Ú
params_rngr”   Údroppath_rngÚrngsÚrandom_paramsÚmissing_keys              r5   Úinit_weightsz&FlaxDinov2PreTrainedModel.init_weights;  sß   € ä—y‘y °D·J±JÔ?ˆä"%§*¡*×"2Ñ"2°3Ó"7Ñˆ
�KÜ$'§J¡J×$4Ñ$4°[Ó$AÑ!ˆ�\Ø$°È,ÑWˆàŸ™×(Ñ(¨¨|ÈÐ(ÓOÐPXÑYˆàÐÜ(¬°-Ó)@ÓAˆMÜ!¤(¨6Ó"2Ó3ˆFØ#×1Ñ1ò A�Ø&3°KÑ&@��{Ò#ðAä!$£ˆDÔÜœ.¨Ó0Ó1Ð1à Ð r7   zbatch_size, sequence_lengthr”   Útrainr’   r  r  c           	      óÖ  — |�|n| j                   j                  }|�|n| j                   j                  }|�|n| j                   j                  }t	        j
                  |d«      }i }|�,t        j                  j                  |«      \  }}	||d<   |	|d<   | j                  j                  d|xs | j                  it	        j                  |t        j                  ¬«      | ||||¬«      S )Nrb   rY   rÃ   r%  rP   )r.  )r   r’   r  r  r<   rh   r/   rÅ   rã   rÎ   Úapplyr%  ri   rI   )
r4   r>   r%  r”   r2  r’   r  r  r.  r-  s
             r5   rC   z"FlaxDinov2PreTrainedModel.__call__O  sé   € ð 2CÐ1NÑ-ÐTX×T_ÑT_×TqÑTqÐà$8Ð$DÑ È$Ï+É+×JjÑJjð 	ð &1Ð%<‘kÀ$Ç+Á+×BYÑBYˆä—}‘} \°<Ó@ˆàˆØÐ"Ü(+¯
©
×(8Ñ(8¸Ó(EÑ%ˆK˜Ø)ˆD�‰OØ+ˆD�Ñà�{‰{× Ñ Ø�vÒ, §¡Ð-Ü�I‰I�l¬#¯+©+Ô6ØˆIØØ ØØð !ó 
ð 	
r7   r¶   )NNFNNN)rE   rF   rG   r„   r   Úconfig_classÚbase_model_prefixÚmain_input_namer  r,   ÚModulerH   r<   rI   re   r   r¤   r"  r/   rÅ   ÚPRNGKeyr   r   r1  r   ÚDINOV2_INPUTS_DOCSTRINGÚformatÚdictr   rC   Ú__classcell__)r$  s   @r5   r  r  "  s(  ø… ñð
  €LØ ÐØ$€OØ"€L�"—)‘)Ó"ð
 ØØŸ;™;Øñmàðmð ð	mð
 �y‰yðmð õmñ! §
¡
× 2Ñ 2ð !Àð !ÐPZð !Ðfpó !ñ( +Ð+B×+IÑ+IÐJgÓ+hÓið Ø*.ØØ,0Ø/3Ø&*ñ 
ð ð 
ð —Z‘Z×'Ñ'ð	 
ð
 ð 
ð $ D™>ð 
ð ' t™nð 
ð ˜d‘^ò 
ó jô 
r7   r  c            	       óv   — e Zd ZU eed<   ej                  Zej                  ed<   d„ Z	 	 	 	 d
de	de	de	de	fd„Z
y	)ÚFlaxDinov2Moduler   r   c                 ó  — t        | j                  | j                  ¬«      | _        t	        | j                  | j                  ¬«      | _        t        j                  | j                  j                  | j                  ¬«      | _	        y )NrP   rë   )
rL   r   r   r?   r  Úencoderr,   rí   rî   Ú	layernormr‘   s    r5   r6   zFlaxDinov2Module.setupw  sQ   € Ü.¨t¯{©{À$Ç*Á*ÔMˆŒÜ(¨¯©¸D¿J¹JÔGˆŒÜŸ™¨d¯k©k×.HÑ.HÐPT×PZÑPZÔ[ˆ�r7   r€   r’   r  r  c                 óú   — | j                  ||¬«      }| j                  |||||¬«      }|d   }| j                  |«      }|d d …dd d …f   }	|s||	f}
|
|dd  z   S t        ||	|j                  |j
                  ¬«      S )Nr   r  r   r   )r
  Úpooler_outputro   r  )r?   rA  rB  r   ro   r  )r4   r>   r€   r’   r  r  ro   Úencoder_outputsÚsequence_outputÚpooled_outputÚhead_outputss              r5   rC   zFlaxDinov2Module.__call__|  s¦   € ð Ÿ™¨ÀM˜ÓRˆàŸ,™,ØØ'Ø/Ø!5Ø#ð 'ó 
ˆð *¨!Ñ,ˆØŸ.™.¨Ó9ˆØ'ª¨1ªa¨Ñ0ˆáØ+¨]Ð;ˆLØ /°!°"Ð"5Ñ5Ð5ä-Ø-Ø'Ø)×7Ñ7Ø&×1Ñ1ô	
ð 	
r7   Nr  r£   rJ   r7   r5   r?  r?  s  s[   … ØÓØ—{‘{€Eˆ3�9‰9Ó"ò\ð #Ø"'Ø%*Ø ñ
ð ð
ð  ð	
ð
 #ð
ð ô
r7   r?  z`The bare Dinov2 Model transformer outputting raw hidden-states without any specific head on top.c                   ó   — e Zd ZeZy)ÚFlaxDinov2ModelN)rE   rF   rG   r?  r  rJ   r7   r5   rJ  rJ  ž  s	   „ ð
 $�Lr7   rJ  ar  
    Returns:

    Examples:

    ```python
    >>> from transformers import AutoImageProcessor, FlaxDinov2Model
    >>> from PIL import Image
    >>> import requests

    >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
    >>> image = Image.open(requests.get(url, stream=True).raw)

    >>> image_processor = AutoImageProcessor.from_pretrained("facebook/dinov2-base")
    >>> model = FlaxDinov2Model.from_pretrained("facebook/dinov2-base")

    >>> inputs = image_processor(images=image, return_tensors="np")
    >>> outputs = model(**inputs)
    >>> last_hidden_states = outputs.last_hidden_state
    ```
)Úoutput_typer5  c                   ól   — e Zd ZU eed<   ej                  Zej                  ed<   d„ Z	 	 	 	 	 dde	fd„Z
y)Ú&FlaxDinov2ForImageClassificationModuler   r   c           	      óF  — t        | j                  | j                  ¬«      | _        t	        j
                  | j                  j                  | j                  t        j                  j                  j                  | j                  j                  dz  dd«      ¬«      | _        y )Nr  r   r   r   )r   r#   )r?  r   r   r  r,   rŒ   Ú
num_labelsr/   r0   r1   r2   Ú
classifierr‘   s    r5   r6   z,FlaxDinov2ForImageClassificationModule.setupÆ  sk   € Ü&¨d¯k©kÀÇÁÔLˆŒÜŸ(™(Ø�K‰K×"Ñ"Ø—*‘*ÜŸ™×+Ñ+×<Ñ<Ø—‘×-Ñ-¨qÑ0°(Ð<Nóô
ˆ�r7   Nr€   c                 óf  — |�|n| j                   j                  }| j                  |||||¬«      }|d   }|d d …df   }|d d …dd …f   }	t        j                  ||	j                  d¬«      gd¬«      }
| j                  |
«      }|s|f|dd  z   }|S t        ||j                  |j                  ¬«      S )Nr  r   r   rc   r9   r   )Úlogitsro   r  )
r   Úuse_return_dictr  r<   rn   ÚmeanrP  r   ro   r  )r4   r>   r€   r’   r  r  r¡   ro   rN   Úpatch_tokensÚlinear_inputrR  r±   s                r5   rC   z/FlaxDinov2ForImageClassificationModule.__call__Ð  sÓ   € ð &1Ð%<‘kÀ$Ç+Á+×B]ÑB]ˆà—+‘+ØØ'Ø/Ø!5Ø#ð ó 
ˆð   ™
ˆà!¢! Q $Ñ'ˆ	Ø$¢Q¨© UÑ+ˆÜ—‘¨	°<×3DÑ3DÈ!Ð3DÓ3LÐ'MÐTVÔWˆà—‘ Ó.ˆáØ�Y ¨¨ Ñ,ˆFØˆMä+ØØ!×/Ñ/Ø×)Ñ)ô
ð 	
r7   )NTNNNr£   rJ   r7   r5   rM  rM  Â  s?   … ØÓØ—{‘{€Eˆ3�9‰9Ó"ò
ð Ø"ØØ!Øñ"
ð ô"
r7   rM  z§
    Dinov2 Model transformer with an image classification head on top (a linear layer on top of the final hidden state of
    the [CLS] token) e.g. for ImageNet.
    c                   ó   — e Zd ZeZy)Ú FlaxDinov2ForImageClassificationN)rE   rF   rG   rM  r  rJ   r7   r5   rX  rX  õ  s	   „ ð :�Lr7   rX  a“  
    Returns:

    Example:

    ```python
    >>> from transformers import AutoImageProcessor, FlaxDinov2ForImageClassification
    >>> from PIL import Image
    >>> import jax
    >>> import requests

    >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
    >>> image = Image.open(requests.get(url, stream=True).raw)

    >>> image_processor = AutoImageProcessor.from_pretrained("facebook/dinov2-base-imagenet1k-1-layer")
    >>> model = FlaxDinov2ForImageClassification.from_pretrained("facebook/dinov2-base-imagenet1k-1-layer", from_pt=True)

    >>> inputs = image_processor(images=image, return_tensors="np")
    >>> outputs = model(**inputs)
    >>> logits = outputs.logits

    >>> # model predicts one of the 1000 ImageNet classes
    >>> predicted_class_idx = jax.numpy.argmax(logits, axis=-1)
    >>> print("Predicted class:", model.config.id2label[predicted_class_idx.item()])
    ```
)rX  rJ  r  )>r„   Úcollections.abcr'   rf   Útypingr   r   Ú
flax.linenÚlinenr,   r/   Ú	jax.numpyÚnumpyr<   Úflax.core.frozen_dictr   r   r   Úflax.linen.attentionr   Úflax.traverse_utilr	   r
   Úmodeling_flax_outputsr   r   r   Úmodeling_flax_utilsr   r   r   r   Úutilsr   r   Úconfiguration_dinov2r   ÚDINOV2_START_DOCSTRINGr:  r8  r   rL   r†   r¦   r­   rI   r¸   rº   rÁ   rÑ   rÛ   ré   rü   r  r  r?  rJ  ÚFLAX_VISION_MODEL_DOCSTRINGrM  rX  Ú$FLAX_VISION_CLASSIFICATION_DOCSTRINGÚ__all__rJ   r7   r5   ú<module>rj     s  ðñ ã Û ß "å Û 
Ý ß >Ñ >Ý >ß ;ç vÑ v÷ó ÷ QÝ .ð!Ð ðFÐ ô"!C §	¡	ô !CôHR˜2Ÿ9™9ô RôlD˜bŸi™iô DôP˜2Ÿ9™9ô ô*˜"Ÿ)™)ô ð* .1¯[©[ó *ô,˜2Ÿ9™9ô ,ô"˜Ÿ™ô ô*�B—I‘Iô ô>(˜"Ÿ)™)ô (ô<,�b—i‘iô ,ô`(
 §	¡	ô (
ôX
˜Ÿ	™	ô 
ô0N
Ð 3ô N
ôb(
�r—y‘yô (
ñV ØfØóô$Ð/ó $ó	ð$ðÐ ñ, ˜Ð*EÔ FÙ  ØÐ!?Èlõô
0
¨R¯Y©Yô 0
ñf ðð óô:Ð'@ó :óð:ð(Ð $ñ6 Ð9Ð;_Ô `Ù  Ø$Ð2NÐ]iõò
 _�r7   