Ë
    S^(h>ª  ã            	       óò  — d Z ddlZddlZddlmZ ddlmZmZm	Z	 ddl
Z
ddlZ
ddl
mZ ddlmZmZmZ ddlmZ dd	lmZ dd
lmZ ddlmZmZmZmZmZmZ ddlmZ ddl m!Z!  ejD                  e#«      Z$dZ%dZ&g d¢Z'dZ(dZ)e G d„ de«      «       Z*e G d„ de«      «       Z+e G d„ de«      «       Z,e G d„ de«      «       Z- G d„ dej\                  «      Z/ G d„ dej\                  «      Z0dAde
jb                  d e2d!e3d"e
jb                  fd#„Z4 G d$„ d%ej\                  «      Z5 G d&„ d'ej\                  «      Z6 G d(„ d)ej\                  «      Z7 G d*„ d+ej\                  «      Z8 G d,„ d-ej\                  «      Z9 G d.„ d/ej\                  «      Z: G d0„ d1e«      Z;d2Z<d3Z= ed4e<«       G d5„ d6e;«      «       Z> ed7e<«       G d8„ d9e;«      «       Z? ed:e<«       G d;„ d<e;«      «       Z@ ed=e<«       G d>„ d?e;e«      «       ZAg d@¢ZBy)BzPyTorch FocalNet model.é    N)Ú	dataclass)ÚOptionalÚTupleÚUnion)Únn)ÚBCEWithLogitsLossÚCrossEntropyLossÚMSELossé   )ÚACT2FN)ÚBackboneOutput)ÚPreTrainedModel)ÚModelOutputÚadd_code_sample_docstringsÚadd_start_docstringsÚ%add_start_docstrings_to_model_forwardÚloggingÚreplace_return_docstrings)ÚBackboneMixiné   )ÚFocalNetConfigr   zmicrosoft/focalnet-tiny)r   é1   i   ztabby, tabby catc                   ó–   — e Zd ZU dZdZeej                     ed<   dZ	ee
ej                        ed<   dZee
ej                        ed<   y)ÚFocalNetEncoderOutputaÄ  
    FocalNet encoder's outputs, with potential hidden states.

    Args:
        last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
            Sequence of hidden-states at the output of the last layer of the model.
        hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
            Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each stage) of
            shape `(batch_size, sequence_length, hidden_size)`.

            Hidden-states of the model at the output of each layer plus the initial embedding outputs.

        reshaped_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
            Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each stage) of
            shape `(batch_size, hidden_size, height, width)`.

            Hidden-states of the model at the output of each layer plus the initial embedding outputs reshaped to
            include the spatial dimensions.
    NÚlast_hidden_stateÚhidden_statesÚreshaped_hidden_states)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r   ÚtorchÚFloatTensorÚ__annotations__r   r   r   © ó    úl/var/www/skyplay_api_hub/venv/lib/python3.12/site-packages/transformers/models/focalnet/modeling_focalnet.pyr   r   8   sT   … ñð( 6:Ð�x × 1Ñ 1Ñ2Ó9Ø8<€M�8˜E %×"3Ñ"3Ñ4Ñ5Ó<ØAEÐ˜H U¨5×+<Ñ+<Ñ%=Ñ>ÔEr&   r   c                   ó¾   — e Zd ZU dZdZeej                     ed<   dZ	eej                     ed<   dZ
eeej                        ed<   dZeeej                        ed<   y)ÚFocalNetModelOutputa¦  
    FocalNet model's outputs that also contains a pooling of the last hidden states.

    Args:
        last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
            Sequence of hidden-states at the output of the last layer of the model.
        pooler_output (`torch.FloatTensor` of shape `(batch_size, hidden_size)`, *optional*, returned when `add_pooling_layer=True` is passed):
            Average pooling of the last layer hidden-state.
        hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
            Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each stage) of
            shape `(batch_size, sequence_length, hidden_size)`.

            Hidden-states of the model at the output of each layer plus the initial embedding outputs.
        reshaped_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
            Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each stage) of
            shape `(batch_size, hidden_size, height, width)`.

            Hidden-states of the model at the output of each layer plus the initial embedding outputs reshaped to
            include the spatial dimensions.
    Nr   Úpooler_outputr   r   )r   r   r    r!   r   r   r"   r#   r$   r*   r   r   r   r%   r&   r'   r)   r)   S   si   … ñð* 6:Ð�x × 1Ñ 1Ñ2Ó9Ø15€M�8˜E×-Ñ-Ñ.Ó5Ø8<€M�8˜E %×"3Ñ"3Ñ4Ñ5Ó<ØAEÐ˜H U¨5×+<Ñ+<Ñ%=Ñ>ÔEr&   r)   c                   ó¾   — e Zd ZU dZdZeej                     ed<   dZ	eej                     ed<   dZ
eeej                        ed<   dZeeej                        ed<   y)Ú!FocalNetMaskedImageModelingOutputa  
    FocalNet masked image model outputs.

    Args:
        loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `bool_masked_pos` is provided):
            Masked image modeling (MLM) loss.
        reconstruction (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`):
            Reconstructed pixel values.
        hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
            Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each stage) of
            shape `(batch_size, sequence_length, hidden_size)`.

            Hidden-states of the model at the output of each layer plus the initial embedding outputs.
        reshaped_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
            Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each stage) of
            shape `(batch_size, hidden_size, height, width)`.

            Hidden-states of the model at the output of each layer plus the initial embedding outputs reshaped to
            include the spatial dimensions.
    NÚlossÚreconstructionr   r   )r   r   r    r!   r-   r   r"   r#   r$   r.   r   r   r   r%   r&   r'   r,   r,   p   sh   … ñð* )-€Dˆ(�5×$Ñ$Ñ
%Ó,Ø26€N�H˜U×.Ñ.Ñ/Ó6Ø8<€M�8˜E %×"3Ñ"3Ñ4Ñ5Ó<ØAEÐ˜H U¨5×+<Ñ+<Ñ%=Ñ>ÔEr&   r,   c                   ó¾   — e Zd ZU dZdZeej                     ed<   dZ	eej                     ed<   dZ
eeej                        ed<   dZeeej                        ed<   y)ÚFocalNetImageClassifierOutputaS  
    FocalNet outputs for image classification.

    Args:
        loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
            Classification (or regression if config.num_labels==1) loss.
        logits (`torch.FloatTensor` of shape `(batch_size, config.num_labels)`):
            Classification (or regression if config.num_labels==1) scores (before SoftMax).
        hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
            Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each stage) of
            shape `(batch_size, sequence_length, hidden_size)`.

            Hidden-states of the model at the output of each layer plus the initial embedding outputs.
        reshaped_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
            Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each stage) of
            shape `(batch_size, hidden_size, height, width)`.

            Hidden-states of the model at the output of each layer plus the initial embedding outputs reshaped to
            include the spatial dimensions.
    Nr-   Úlogitsr   r   )r   r   r    r!   r-   r   r"   r#   r$   r1   r   r   r   r%   r&   r'   r0   r0   �   sh   … ñð* )-€Dˆ(�5×$Ñ$Ñ
%Ó,Ø*.€FˆH�U×&Ñ&Ñ'Ó.Ø8<€M�8˜E %×"3Ñ"3Ñ4Ñ5Ó<ØAEÐ˜H U¨5×+<Ñ+<Ñ%=Ñ>ÔEr&   r0   c                   óŠ   ‡ — e Zd ZdZdˆ fd„	Z	 ddeej                     deej                     de	ej                     fd„Zˆ xZS )	ÚFocalNetEmbeddingszX
    Construct the patch embeddings and layernorm. Optionally, also the mask token.
    c           	      ó  •— t         ‰| �  «        t        ||j                  |j                  |j
                  |j                  |j                  d¬«      | _        | j                  j                  | _
        |r4t        j                  t        j                  dd|j                  «      «      nd | _        t        j                   |j                  |j"                  ¬«      | _        t        j&                  |j(                  «      | _        y )NT)ÚconfigÚ
image_sizeÚ
patch_sizeÚnum_channelsÚ	embed_dimÚuse_conv_embedÚis_stemr   ©Úeps)ÚsuperÚ__init__ÚFocalNetPatchEmbeddingsr6   r7   r8   r9   r:   Úpatch_embeddingsÚ	grid_sizeÚ
patch_gridr   Ú	Parameterr"   ÚzerosÚ
mask_tokenÚ	LayerNormÚlayer_norm_epsÚnormÚDropoutÚhidden_dropout_probÚdropout)Úselfr5   Úuse_mask_tokenÚ	__class__s      €r'   r?   zFocalNetEmbeddings.__init__¯   s¿   ø€ Ü‰ÑÔä 7ØØ×(Ñ(Ø×(Ñ(Ø×,Ñ,Ø×&Ñ&Ø!×0Ñ0Øô!
ˆÔð ×/Ñ/×9Ñ9ˆŒÙO]œ"Ÿ,™,¤u§{¡{°1°a¸×9IÑ9IÓ'JÔKÐcgˆŒä—L‘L ×!1Ñ!1°v×7LÑ7LÔMˆŒ	Ü—z‘z &×"<Ñ"<Ó=ˆ�r&   Úpixel_valuesÚbool_masked_posÚreturnc                 ó8  — | j                  |«      \  }}| j                  |«      }|j                  «       \  }}}|�K| j                  j	                  ||d«      }|j                  d«      j                  |«      }	|d|	z
  z  ||	z  z   }| j                  |«      }||fS )Néÿÿÿÿç      ð?)rA   rI   ÚsizerF   ÚexpandÚ	unsqueezeÚtype_asrL   )
rM   rP   rQ   Ú
embeddingsÚoutput_dimensionsÚ
batch_sizeÚseq_lenÚ_Úmask_tokensÚmasks
             r'   ÚforwardzFocalNetEmbeddings.forwardÁ   s¦   € ð )-×(=Ñ(=¸lÓ(KÑ%ˆ
Ð%Ø—Y‘Y˜zÓ*ˆ
Ø!+§¡Ó!2Ñˆ
�G˜QàÐ&ØŸ/™/×0Ñ0°¸WÀbÓIˆKà"×,Ñ,¨RÓ0×8Ñ8¸ÓEˆDØ# s¨T¡zÑ2°[À4Ñ5GÑGˆJà—\‘\ *Ó-ˆ
ØÐ,Ð,Ð,r&   )F©N)r   r   r    r!   r?   r   r"   r#   Ú
BoolTensorr   ÚTensorra   Ú__classcell__©rO   s   @r'   r3   r3   ª   sQ   ø„ ñõ>ð& hlñ-Ø$ U×%6Ñ%6Ñ7ð-ØJRÐSX×ScÑScÑJdð-à	ˆu�|‰|Ñ	÷-r&   r3   c                   óz   ‡ — e Zd Z	 	 	 dˆ fd„	Zd„ Zdeej                     deej                  ee
   f   fd„Zˆ xZS )r@   c	                 ód  •— t         ‰| �  «        t        |t        j                  j
                  «      r|n||f}t        |t        j                  j
                  «      r|n||f}|d   |d   z  |d   |d   z  z  }	|| _        || _        || _        |	| _	        |d   |d   z  |d   |d   z  f| _
        |r/|rd}
d}d}nd}
d}d}t        j                  |||
||¬«      | _        nt        j                  ||||¬«      | _        |r't        j                  ||j                  ¬	«      | _        y d | _        y )
Nr   r   é   é   é   r   )Úkernel_sizeÚstrideÚpadding)rl   rm   r<   )r>   r?   Ú
isinstanceÚcollectionsÚabcÚIterabler6   r7   r8   Únum_patchesrB   r   ÚConv2dÚ
projectionrG   rH   rI   )rM   r5   r6   r7   r8   r9   Úadd_normr:   r;   rs   rl   rn   rm   rO   s                €r'   r?   z FocalNetPatchEmbeddings.__init__Ó   s7  ø€ ô 	‰ÑÔÜ#-¨j¼+¿/¹/×:RÑ:RÔ#S‘ZÐZdÐfpÐYqˆ
Ü#-¨j¼+¿/¹/×:RÑ:RÔ#S‘ZÐZdÐfpÐYqˆ
Ø! !‘}¨
°1©Ñ5¸*ÀQ¹-È:ÐVWÉ=Ñ:XÑYˆØ$ˆŒØ$ˆŒØ(ˆÔØ&ˆÔØ$ Q™-¨:°a©=Ñ8¸*ÀQ¹-È:ÐVWÉ=Ñ:XÐYˆŒááØ�Ø�Ø‘à�Ø�Ø�Ü Ÿi™iØ˜i°[ÈÐY`ôˆD�Oô !Ÿi™i¨°iÈZÐ`jÔkˆDŒOáÜŸ™ Y°F×4IÑ4IÔJˆD�IàˆD�Ir&   c                 ón  — || j                   d   z  dk7  rDd| j                   d   || j                   d   z  z
  f}t        j                  j                  ||«      }|| j                   d   z  dk7  rFddd| j                   d   || j                   d   z  z
  f}t        j                  j                  ||«      }|S )Nr   r   )r7   r   Ú
functionalÚpad)rM   rP   ÚheightÚwidthÚ
pad_valuess        r'   Ú	maybe_padz!FocalNetPatchEmbeddings.maybe_padý   s±   € Ø�4—?‘? 1Ñ%Ñ%¨Ò*Ø˜TŸ_™_¨QÑ/°%¸$¿/¹/È!Ñ:LÑ2LÑLÐMˆJÜŸ=™=×,Ñ,¨\¸:ÓFˆLØ�D—O‘O AÑ&Ñ&¨!Ò+Ø˜Q  4§?¡?°1Ñ#5¸ÀÇÁÐQRÑASÑ8SÑ#SÐTˆJÜŸ=™=×,Ñ,¨\¸:ÓFˆLØÐr&   rP   rR   c                 óN  — |j                   \  }}}}|| j                  k7  rt        d«      ‚| j                  |||«      }| j	                  |«      }|j                   \  }}}}||f}|j                  d«      j                  dd«      }| j                  �| j                  |«      }||fS )NzeMake sure that the channel dimension of the pixel values match with the one set in the configuration.rj   r   )Úshaper8   Ú
ValueErrorr}   ru   ÚflattenÚ	transposerI   )rM   rP   r^   r8   rz   r{   rZ   r[   s           r'   ra   zFocalNetPatchEmbeddings.forward  sµ   € Ø)5×);Ñ);Ñ&ˆˆ<˜ Ø˜4×,Ñ,Ò,ÜØwóð ð —~‘~ l°F¸EÓBˆØ—_‘_ \Ó2ˆ
Ø(×.Ñ.Ñˆˆ1ˆf�eØ# U˜OÐØ×'Ñ'¨Ó*×4Ñ4°Q¸Ó:ˆ
à�9‰9Ð ØŸ™ :Ó.ˆJàÐ,Ð,Ð,r&   )FFF)r   r   r    r?   r}   r   r"   r#   r   rd   Úintra   re   rf   s   @r'   r@   r@   Ò   sL   ø„ ð ØØõ(òTð- H¨U×->Ñ->Ñ$?ð -ÀEÈ%Ï,É,ÐX]Ð^aÑXbÐJbÑDc÷ -r&   r@   ÚinputÚ	drop_probÚtrainingrR   c                 ó  — |dk(  s|s| S d|z
  }| j                   d   fd| j                  dz
  z  z   }|t        j                  || j                  | j
                  ¬«      z   }|j                  «        | j                  |«      |z  }|S )aF  
    Drop paths (Stochastic Depth) per sample (when applied in main path of residual blocks).

    Comment by Ross Wightman: This is the same as the DropConnect impl I created for EfficientNet, etc networks,
    however, the original name is misleading as 'Drop Connect' is a different form of dropout in a separate paper...
    See discussion: https://github.com/tensorflow/tpu/issues/494#issuecomment-532968956 ... I've opted for changing the
    layer and argument names to 'drop path' rather than mix DropConnect as a layer name and use 'survival rate' as the
    argument.
    ç        r   r   )r   )ÚdtypeÚdevice)r   Úndimr"   Úrandr‰   rŠ   Úfloor_Údiv)r„   r…   r†   Ú	keep_probr   Úrandom_tensorÚoutputs          r'   Ú	drop_pathr’     s�   € ð �CÒ™xØˆØ�I‘€IØ�[‰[˜‰^Ð ¨¯
©
°Q©Ñ 7Ñ7€EØ¤§
¡
¨5¸¿¹ÈEÏLÉLÔ YÑY€MØ×ÑÔØ�Y‰Y�yÓ! MÑ1€FØ€Mr&   c                   óx   ‡ — e Zd ZdZd	dee   ddfˆ fd„Zdej                  dej                  fd„Z	de
fd„Zˆ xZS )
ÚFocalNetDropPathzXDrop paths (Stochastic Depth) per sample (when applied in main path of residual blocks).Nr…   rR   c                 ó0   •— t         ‰| �  «        || _        y rb   )r>   r?   r…   )rM   r…   rO   s     €r'   r?   zFocalNetDropPath.__init__2  s   ø€ Ü‰ÑÔØ"ˆ�r&   r   c                 óD   — t        || j                  | j                  «      S rb   )r’   r…   r†   )rM   r   s     r'   ra   zFocalNetDropPath.forward6  s   € Ü˜¨¯©¸¿¹ÓFÐFr&   c                 ó8   — dj                  | j                  «      S )Nzp={})Úformatr…   ©rM   s    r'   Ú
extra_reprzFocalNetDropPath.extra_repr9  s   € Ø�}‰}˜TŸ^™^Ó,Ð,r&   rb   )r   r   r    r!   r   Úfloatr?   r"   rd   ra   Ústrrš   re   rf   s   @r'   r”   r”   /  sG   ø„ Ùbñ# (¨5¡/ð #¸Tõ #ðG U§\¡\ð G°e·l±ló Gð-˜C÷ -r&   r”   c                   ó&   ‡ — e Zd Zdˆ fd„	Zd„ Zˆ xZS )ÚFocalNetModulationc                 ó  •— t         ‰	| �  «        || _        |j                  |   | _        |j
                  |   | _        || _        |j                  | _        |j                  | _	        t        j                  |d|z  | j                  dz   z   |¬«      | _        t        j                  ||dd|¬«      | _        t        j                  «       | _        t        j                  ||«      | _        t        j$                  |«      | _        t        j(                  «       | _        g | _        t/        | j                  «      D ]—  }| j                  |z  | j                  z   }| j*                  j1                  t        j2                  t        j                  |||d||dz  d¬«      t        j                  «       «      «       | j,                  j1                  |«       Œ™ | j                  r't        j4                  ||j6                  ¬«      | _        y y )Nrj   r   )Úbias)rl   rm   r    F)rl   rm   Úgroupsrn   r    r<   )r>   r?   ÚdimÚfocal_windowsÚfocal_windowÚfocal_levelsÚfocal_levelÚfocal_factorÚ use_post_layernorm_in_modulationÚnormalize_modulatorr   ÚLinearÚprojection_inrt   Úprojection_contextÚGELUÚ
activationÚprojection_outrJ   Úprojection_dropoutÚ
ModuleListÚfocal_layersÚkernel_sizesÚrangeÚappendÚ
SequentialrG   rH   Ú	layernorm)
rM   r5   Úindexr¢   r§   r    r°   Úkrl   rO   s
            €r'   r?   zFocalNetModulation.__init__>  s¬  ø€ Ü‰ÑÔàˆŒØ"×0Ñ0°Ñ7ˆÔØ!×.Ñ.¨uÑ5ˆÔØ(ˆÔØ06×0WÑ0WˆÔ-Ø#)×#=Ñ#=ˆÔ äŸY™Y s¨A°©G°t×7GÑ7GÈ!Ñ7KÑ,LÐSWÔXˆÔÜ"$§)¡)¨C°À!ÈAÐTXÔ"YˆÔäŸ'™'›)ˆŒÜ Ÿi™i¨¨SÓ1ˆÔÜ"$§*¡*Ð-?Ó"@ˆÔÜŸM™M›OˆÔàˆÔÜ�t×'Ñ'Ó(ò 
	2ˆAØ×+Ñ+¨aÑ/°$×2CÑ2CÑCˆKØ×Ñ×$Ñ$Ü—‘Ü—I‘IØ˜S¨kÀ!ÈCÐYdÐhiÑYiÐpuôô —G‘G“Ió	ôð ×Ñ×$Ñ$ [Õ1ð
	2ð ×0Ò0ÜŸ\™\¨#°6×3HÑ3HÔIˆD�Nð 1r&   c                 ó,  — |j                   d   }| j                  |«      j                  dddd«      j                  «       }t	        j
                  |||| j                  dz   fd«      \  }}}d}t        | j                  «      D ]+  } | j                  |   |«      }|||dd…||dz   …f   z  z   }Œ- | j                  |j                  dd¬«      j                  dd¬«      «      }	||	|dd…| j                  d…f   z  z   }| j                  r|| j                  dz   z  }| j                  |«      }
||
z  }|j                  dddd«      j                  «       }| j                  r| j                  |«      }| j                  |«      }| j!                  |«      }|S )	zˆ
        Args:
            hidden_state:
                Input features with shape of (batch_size, height, width, num_channels)
        rT   r   r   r   rj   NT)Úkeepdim)r   r«   ÚpermuteÚ
contiguousr"   Úsplitr¦   r´   r²   r®   Úmeanr©   r¬   r¨   r·   r¯   r°   )rM   Úhidden_stater8   ÚxÚqÚctxÚgatesÚctx_allÚlevelÚ
ctx_globalÚ	modulatorÚx_outs               r'   ra   zFocalNetModulation.forward_  s­  € ð $×)Ñ)¨"Ñ-ˆð ×Ñ˜|Ó,×4Ñ4°Q¸¸1¸aÓ@×KÑKÓMˆÜŸ™ A¨°lÀD×DTÑDTÐWXÑDXÐ'YÐ[\Ó]‰ˆˆ3�ð ˆÜ˜4×+Ñ+Ó,ò 	BˆEØ*�$×#Ñ# EÑ*¨3Ó/ˆCØ  eªA¨u°u¸q±yÐ/@Ð,@Ñ&AÑ AÑA‰Gð	Bð —_‘_ S§X¡X¨a¸ XÓ%>×%CÑ%CÀAÈtÐ%CÓ%TÓUˆ
Ø˜J¨ªq°$×2BÑ2BÑ2DÐ/DÑ)EÑEÑEˆð ×#Ò#Ø ×!1Ñ!1°AÑ!5Ñ6ˆGð ×+Ñ+¨GÓ4ˆ	Ø�I‘ˆØ—‘˜a  A qÓ)×4Ñ4Ó6ˆØ×0Ò0Ø—N‘N 5Ó)ˆEð ×#Ñ# EÓ*ˆØ×'Ñ'¨Ó.ˆØˆr&   )rj   Trˆ   ©r   r   r    r?   ra   re   rf   s   @r'   rž   rž   =  s   ø„ õJöB"r&   rž   c                   ó&   ‡ — e Zd Zdˆ fd„	Zd„ Zˆ xZS )ÚFocalNetMlpc                 ó
  •— t         ‰| �  «        |xs |}|xs |}t        j                  ||«      | _        t
        |j                     | _        t        j                  ||«      | _        t        j                  |«      | _
        y rb   )r>   r?   r   rª   Úfc1r   Ú
hidden_actr®   Úfc2rJ   Údrop)rM   r5   Úin_featuresÚhidden_featuresÚout_featuresrÑ   rO   s         €r'   r?   zFocalNetMlp.__init__…  sh   ø€ Ü‰ÑÔØ#Ò2 {ˆØ)Ò8¨[ˆÜ—9‘9˜[¨/Ó:ˆŒÜ  ×!2Ñ!2Ñ3ˆŒÜ—9‘9˜_¨lÓ;ˆŒÜ—J‘J˜tÓ$ˆ�	r&   c                 ó°   — | j                  |«      }| j                  |«      }| j                  |«      }| j                  |«      }| j                  |«      }|S rb   )rÎ   r®   rÑ   rÐ   )rM   rÀ   s     r'   ra   zFocalNetMlp.forwardŽ  sN   € Ø—x‘x Ó-ˆØ—‘ |Ó4ˆØ—y‘y Ó.ˆØ—x‘x Ó-ˆØ—y‘y Ó.ˆØÐr&   )NNrˆ   rÊ   rf   s   @r'   rÌ   rÌ   „  s   ø„ õ%ör&   rÌ   c                   ó*   ‡ — e Zd ZdZdˆ fd„	Zd„ Zˆ xZS )ÚFocalNetLayeraƒ  Focal Modulation Network layer (block).

    Args:
        config (`FocalNetConfig`):
            Model config.
        index (`int`):
            Layer index.
        dim (`int`):
            Number of input channels.
        input_resolution (`Tuple[int]`):
            Input resulotion.
        drop_path (`float`, *optional*, defaults to 0.0):
            Stochastic depth rate.
    c                 óH  •— t         ‰| �  «        || _        || _        || _        |j
                  | _        |j                  | _        t        j                  ||j                  ¬«      | _        t        |||| j                  ¬«      | _        |dkD  rt        |«      nt        j                  «       | _        t        j                  ||j                  ¬«      | _        t%        ||j&                  z  «      }t)        |||| j                  ¬«      | _        d| _        d| _        |j0                  ryt        j2                  |j4                  t7        j8                  |«      z  d¬«      | _        t        j2                  |j4                  t7        j8                  |«      z  d¬«      | _        y y )Nr<   )r5   r¸   r¢   r°   rˆ   )r5   rÒ   rÓ   rÑ   rU   T)Úrequires_grad)r>   r?   r5   r¢   Úinput_resolutionrK   rÑ   Úuse_post_layernormr   rG   rH   Únorm1rž   Ú
modulationr”   ÚIdentityr’   Únorm2rƒ   Ú	mlp_ratiorÌ   ÚmlpÚgamma_1Úgamma_2Úuse_layerscalerD   Úlayerscale_valuer"   Úones)rM   r5   r¸   r¢   rÚ   r’   Úmlp_hidden_dimrO   s          €r'   r?   zFocalNetLayer.__init__§  sE  ø€ Ü‰ÑÔàˆŒð ˆŒØ 0ˆÔð ×.Ñ.ˆŒ	Ø"(×";Ñ";ˆÔä—\‘\ #¨6×+@Ñ+@ÔAˆŒ
Ü,ØØØØ#Ÿy™yô	
ˆŒð 9BÀCºÔ)¨)Ô4ÌRÏ[É[Ë]ˆŒÜ—\‘\ #¨6×+@Ñ+@ÔAˆŒ
Ü˜S 6×#3Ñ#3Ñ3Ó4ˆÜ f¸#È~Ðdh×dmÑdmÔnˆŒàˆŒØˆŒØ× Ò ÜŸ<™<¨×(?Ñ(?Ä%Ç*Á*ÈcÓBSÑ(SÐcgÔhˆDŒLÜŸ<™<¨×(?Ñ(?Ä%Ç*Á*ÈcÓBSÑ(SÐcgÔhˆD�Lð !r&   c           	      ó:  — |\  }}|j                   \  }}}|}| j                  r|n| j                  |«      }|j                  ||||«      }| j	                  |«      j                  |||z  |«      }| j                  s|n| j                  |«      }|| j                  | j                  |z  «      z   }|| j                  | j                  | j                  r | j                  | j                  |«      «      n| j                  | j                  |«      «      z  «      z   }|S rb   )
r   rÛ   rÜ   ÚviewrÝ   r’   râ   rã   rß   rá   )	rM   rÀ   Úinput_dimensionsrz   r{   r\   r^   r8   Úshortcuts	            r'   ra   zFocalNetLayer.forwardÇ  s  € Ø(‰ˆ�Ø&2×&8Ñ&8Ñ#ˆ
�A�|Øˆð (,×'>Ò'>‘|ÀDÇJÁJÈ|ÓD\ˆØ#×(Ñ(¨°V¸UÀLÓQˆØ—‘ |Ó4×9Ñ9¸*ÀfÈuÁnÐVbÓcˆØ+/×+BÒ+B‘|ÈÏ
É
ÐS_ÓH`ˆð   $§.¡.°·±ÀÑ1LÓ"MÑMˆØ# d§n¡nØ�L‰LØ59×5LÒ5Lˆt�z‰z˜$Ÿ(™( <Ó0Ô1ÐRV×RZÑRZÐ[_×[eÑ[eÐfrÓ[sÓRtñvó'
ñ 
ˆð
 Ðr&   )rˆ   )r   r   r    r!   r?   ra   re   rf   s   @r'   r×   r×   —  s   ø„ ñõiö@r&   r×   c                   ój   ‡ — e Zd Zˆ fd„Zdej
                  deeef   deej
                     fd„Zˆ xZ	S )ÚFocalNetStagec                 ó’  •— t         ‰| �  «        || _        t        |j                  «      | _        t        | j
                  «      D �cg c]  }|j                  d|z  z  ‘Œ }}||   }|| j
                  dz
  k  r||dz      nd }|| j
                  dz
  k  rt        nd }t        j                  d|j                  t        |j                  «      «      D �	cg c]  }	|	j                  «       ‘Œ }
}	|
t        |j                  d | «      t        |j                  d |dz    «       }t        j                  t        |j                  |   «      D �cg c]'  }t!        ||||t#        |t$        «      r||   n|¬«      ‘Œ) c}«      | _        |�' |||d||d|j(                  d¬«      | _        d| _        y d | _        d| _        y c c}w c c}	w c c}w )Nrj   r   r   )r5   r¸   r¢   rÚ   r’   TF)r5   r6   r7   r8   r9   rv   r:   r;   )r>   r?   r5   ÚlenÚdepthsÚ
num_stagesr´   r9   r@   r"   ÚlinspaceÚdrop_path_rateÚsumÚitemr   r±   r×   ro   ÚlistÚlayersr:   Ú
downsampleÚpointing)rM   r5   r¸   rÚ   Úir9   r¢   Úout_dimrø   rÁ   Údprr’   rO   s               €r'   r?   zFocalNetStage.__init__Ý  s³  ø€ Ü‰ÑÔàˆŒÜ˜fŸm™mÓ,ˆŒä8=¸d¿o¹oÓ8NÖO°1�V×%Ñ%¨¨A©Ó.ÐOˆ	ÐOØ˜ÑˆØ+0°4·?±?ÀQÑ3FÒ+F�)˜E A™IÒ&ÈTˆØ16¸¿¹È1Ñ9LÒ1LÕ,ÐSWˆ
ô "'§¡°°6×3HÑ3HÌ#ÈfÏmÉmÓJ\Ó!]Ö^˜Aˆq�v‰v�xÐ^ˆÐ^Øœ˜FŸM™M¨&¨5Ð1Ó2´S¸¿¹À{ÈÐQRÉÐ9SÓ5TÐUˆ	ä—m‘mô ˜vŸ}™}¨UÑ3Ó4ö	ð ô Ø!ØØØ%5Ü.8¸ÄDÔ.I˜i¨šlÈyöò	ó
ˆŒð Ð!Ù(ØØ+ØØ Ø!ØØ%×4Ñ4Øô	ˆDŒOð ˆ�ð #ˆDŒOàˆ�ùòI Pùò _ùò	s   ÁF:ÃF?Å
,Gr   rê   rR   c                 ó  — |\  }}| j                   D ]  } |||«      }Œ |}| j                  �K|\  }}|j                  dd«      j                  |j                  d   d||«      }| j                  |«      \  }}n||||f}|||f}|S )Nr   rj   r   rT   )r÷   rø   r‚   Úreshaper   )	rM   r   rê   rz   r{   Úlayer_moduleÚ!hidden_states_before_downsamplingr[   Ústage_outputss	            r'   ra   zFocalNetStage.forward	  s´   € Ø(‰ˆ�Ø ŸK™Kò 	JˆLÙ(¨Ð8HÓI‰Mð	Jð -:Ð)Ø�?‰?Ð&Ø,‰MˆF�EØ)×3Ñ3°A°qÓ9×AÑAØ1×7Ñ7¸Ñ:¸BÀÈóˆMð 04¯©¸}Ó/MÑ,ˆMÑ,ð "(¨°¸Ð >Ðà&Ð(IÐK\Ð]ˆàÐr&   )
r   r   r    r?   r"   rd   r   rƒ   ra   re   rf   s   @r'   rí   rí   Ü  s=   ø„ ô*ðX U§\¡\ð ÀUÈ3ÐPSÈ8Á_ð ÐY^Ð_d×_kÑ_kÑYl÷ r&   rí   c                   ó€   ‡ — e Zd Zˆ fd„Z	 	 	 d	dej
                  deeef   dee	   dee	   dee	   de
eef   fd„Zˆ xZS )
ÚFocalNetEncoderc                 ó2  •— t         ‰| �  «        t        |j                  «      | _        || _        t        j                  t        | j                  «      D �cg c]$  }t        |||d   d|z  z  |d   d|z  z  f¬«      ‘Œ& c}«      | _
        d| _        y c c}w )Nr   rj   r   )r5   r¸   rÚ   F)r>   r?   rï   rð   rñ   r5   r   r±   r´   rí   ÚstagesÚgradient_checkpointing)rM   r5   rB   Úi_layerrO   s       €r'   r?   zFocalNetEncoder.__init__  s—   ø€ Ü‰ÑÔÜ˜fŸm™mÓ,ˆŒØˆŒä—m‘mô  % T§_¡_Ó5öð ô Ø!Ø!Ø&/°¡l°q¸'±zÑ&BÀIÈaÁLÐUVÐX_ÑU_ÑD`Ð%aöòó	
ˆŒð ',ˆÕ#ùòs   Á)Br   rê   Úoutput_hidden_statesÚ(output_hidden_states_before_downsamplingÚreturn_dictrR   c                 ó  — |rdnd }|rdnd }|rE|j                   \  }}	}
 |j                  |g|¢|
‘­Ž }|j                  dddd«      }||fz  }||fz  }t        | j                  «      D ]ú  \  }}| j
                  r*| j                  r| j                  |j                  ||«      }n	 |||«      }|d   }|d   }|d   }|d   |d   f}|rP|rN|j                   \  }}	}
 |j                  |g|d   |d   f¢|
‘­Ž }|j                  dddd«      }||fz  }||fz  }Œ°|sŒ³|rŒ¶|j                   \  }}	}
 |j                  |g|¢|
‘­Ž }|j                  dddd«      }||fz  }||fz  }Œü |st        d„ ||fD «       «      S t        |||¬	«      S )
Nr%   r   r   r   rj   éþÿÿÿrT   c              3   ó&   K  — | ]	  }|€Œ|–— Œ y ­wrb   r%   )Ú.0Úvs     r'   ú	<genexpr>z*FocalNetEncoder.forward.<locals>.<genexpr>g  s   è ø€ ÒX˜qÈ!É-œÑXùs   ‚Š)r   r   r   )r   ré   r¼   Ú	enumerater  r  r†   Ú_gradient_checkpointing_funcÚ__call__Útupler   )rM   r   rê   r  r	  r
  Úall_hidden_statesÚall_reshaped_hidden_statesr\   r^   Úhidden_sizeÚreshaped_hidden_staterú   Ústage_moduler  r   r[   s                    r'   ra   zFocalNetEncoder.forward1  s9  € ñ #7™B¸DÐÙ+?¡RÀTÐ"áØ)6×)<Ñ)<Ñ&ˆJ˜˜;à$6 M×$6Ñ$6°zÐ$bÐDTÐ$bÐVaÒ$bÐ!Ø$9×$AÑ$AÀ!ÀQÈÈ1Ó$MÐ!Ø -Ð!1Ñ1ÐØ&Ð+@Ð*BÑBÐ&ä(¨¯©Ó5ò  	G‰OˆAˆ|Ø×*Ò*¨t¯}ª}Ø $× AÑ AØ ×)Ñ)Ø!Ø$ó!‘ñ !-¨]Ð<LÓ M�à)¨!Ñ,ˆMØ0=¸aÑ0@Ð-Ø -¨aÑ 0Ðà 1°"Ñ 5Ð7HÈÑ7LÐMÐá#Ñ(PØ-N×-TÑ-TÑ*�
˜A˜{ð )OÐ(I×(NÑ(NØð)Ø"3°AÑ"6Ð8IÈ!Ñ8LÐ!Mð)ØOZò)Ð%ð )>×(EÑ(EÀaÈÈAÈqÓ(QÐ%Ø!Ð&GÐ%IÑIÐ!Ø*Ð/DÐ.FÑFÑ*Ú%Ò.VØ-:×-@Ñ-@Ñ*�
˜A˜{à(:¨×(:Ñ(:¸:Ð(fÐHXÐ(fÐZeÒ(fÐ%Ø(=×(EÑ(EÀaÈÈAÈqÓ(QÐ%Ø! mÐ%5Ñ5Ð!Ø*Ð/DÐ.FÑFÑ*ðA 	GñD ÜÑX ]Ð4EÐ$FÔXÓXÐXä$Ø+Ø+Ø#=ô
ð 	
r&   )FFT)r   r   r    r?   r"   rd   r   rƒ   r   Úboolr   r   ra   re   rf   s   @r'   r  r    su   ø„ ô,ð, 05ØCHØ&*ñ<
à—|‘|ð<
ð    S ™/ð<
ð ' t™nð	<
ð
 3;¸4±.ð<
ð ˜d‘^ð<
ð 
ˆuÐ+Ð+Ñ	,÷<
r&   r  c                   ó,   — e Zd ZdZeZdZdZdZdgZ	d„ Z
y)ÚFocalNetPreTrainedModelz†
    An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained
    models.
    ÚfocalnetrP   Trí   c                 ó²  — t        |t        j                  t        j                  f«      rm|j                  j
                  j                  d| j                  j                  ¬«       |j                  �%|j                  j
                  j                  «        yyt        |t        j                  «      rJ|j                  j
                  j                  «        |j                  j
                  j                  d«       yt        |t        «      r2|j                  �%|j                  j
                  j                  «        yyt        |t        «      rŠ| j                  j                   rs|j"                  j
                  j                  | j                  j$                  «       |j&                  j
                  j                  | j                  j$                  «       yyy)zInitialize the weightsrˆ   )r¿   ÚstdNrU   )ro   r   rª   rt   ÚweightÚdataÚnormal_r5   Úinitializer_ranger    Úzero_rG   Úfill_r3   rF   r×   rä   râ   rå   rã   )rM   Úmodules     r'   Ú_init_weightsz%FocalNetPreTrainedModel._init_weights|  s<  € ä�fœrŸy™y¬"¯)©)Ð4Ô5ð �M‰M×Ñ×&Ñ&¨C°T·[±[×5RÑ5RÐ&ÔSØ�{‰{Ð&Ø—‘× Ñ ×&Ñ&Õ(ð 'ä˜¤§¡Ô-Ø�K‰K×Ñ×"Ñ"Ô$Ø�M‰M×Ñ×$Ñ$ SÕ)Ü˜Ô 2Ô3Ø× Ñ Ð,Ø×!Ñ!×&Ñ&×,Ñ,Õ.ð -ä˜¤Ô.Ø�{‰{×)Ò)Ø—‘×#Ñ#×)Ñ)¨$¯+©+×*FÑ*FÔGØ—‘×#Ñ#×)Ñ)¨$¯+©+×*FÑ*FÕGð *ð /r&   N)r   r   r    r!   r   Úconfig_classÚbase_model_prefixÚmain_input_nameÚsupports_gradient_checkpointingÚ_no_split_modulesr'  r%   r&   r'   r  r  p  s.   „ ñð
 "€LØ"ÐØ$€OØ&*Ð#Ø(Ð)ÐóHr&   r  aK  
    This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class. Use
    it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and
    behavior.

    Parameters:
        config ([`FocalNetConfig`]): Model configuration class with all the parameters of the model.
            Initializing with a config file does not load the weights associated with the model, only the
            configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
aB  
    Args:
        pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`):
            Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See
            [`AutoImageProcessor.__call__`] for details.

        output_hidden_states (`bool`, *optional*):
            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
            more detail.
        return_dict (`bool`, *optional*):
            Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
zVThe bare FocalNet Model outputting raw hidden-states without any specific head on top.c                   óÌ   ‡ — e Zd Zdˆ fd„	Zd„ Z ee«       eee	e
de¬«      	 	 	 	 ddeej                     deej                     dee   dee   d	eee	f   f
d
„«       «       Zˆ xZS )ÚFocalNetModelc                 óê  •— t         ‰| �  |«       || _        t        |j                  «      | _        t        |j                  d| j
                  dz
  z  z  «      | _        t        ||¬«      | _
        t        || j                  j                  «      | _        t        j                  | j                  |j                   ¬«      | _        |rt        j$                  d«      nd | _        | j)                  «        y )Nrj   r   )rN   r<   )r>   r?   r5   rï   rð   rñ   rƒ   r9   Únum_featuresr3   rZ   r  rC   Úencoderr   rG   rH   r·   ÚAdaptiveAvgPool1dÚpoolerÚ	post_init)rM   r5   Úadd_pooling_layerrN   rO   s       €r'   r?   zFocalNetModel.__init__®  s³   ø€ Ü‰Ñ˜Ô ØˆŒÜ˜fŸm™mÓ,ˆŒÜ × 0Ñ 0°1¸¿¹È1Ñ9LÑ3MÑ MÓNˆÔä,¨VÀNÔSˆŒÜ& v¨t¯©×/IÑ/IÓJˆŒäŸ™ d×&7Ñ&7¸V×=RÑ=RÔSˆŒÙ1B”b×*Ñ*¨1Ô-ÈˆŒð 	�‰Õr&   c                 ó.   — | j                   j                  S rb   )rZ   rA   r™   s    r'   Úget_input_embeddingsz"FocalNetModel.get_input_embeddings½  s   € Ø�‰×/Ñ/Ð/r&   Úvision)Ú
checkpointÚoutput_typer(  ÚmodalityÚexpected_outputrP   rQ   r  r
  rR   c                 óò  — |�|n| j                   j                  }|�|n| j                   j                  }|€t        d«      ‚| j	                  ||¬«      \  }}| j                  ||||¬«      }|d   }| j                  |«      }d}	| j                  �7| j                  |j                  dd«      «      }	t        j                  |	d«      }	|s||	f|dd z   }
|
S t        ||	|j                  |j                  ¬«      S )	z¿
        bool_masked_pos (`torch.BoolTensor` of shape `(batch_size, num_patches)`):
            Boolean masked positions. Indicates which patches are masked (1) and which aren't (0).
        Nz You have to specify pixel_values)rQ   ©r  r
  r   r   rj   )r   r*   r   r   )r5   r  Úuse_return_dictr€   rZ   r1  r·   r3  r‚   r"   r�   r)   r   r   )rM   rP   rQ   r  r
  Úembedding_outputrê   Úencoder_outputsÚsequence_outputÚpooled_outputr‘   s              r'   ra   zFocalNetModel.forwardÀ  s  € ð( %9Ð$DÑ È$Ï+É+×JjÑJjð 	ð &1Ð%<‘kÀ$Ç+Á+×B]ÑB]ˆàÐÜÐ?Ó@Ð@à-1¯_©_¸\Ð[j¨_Ó-kÑ*ÐÐ*àŸ,™,ØØØ!5Ø#ð	 'ó 
ˆð *¨!Ñ,ˆØŸ.™.¨Ó9ˆàˆØ�;‰;Ð"Ø ŸK™K¨×(AÑ(AÀ!ÀQÓ(GÓHˆMÜ!ŸM™M¨-¸Ó;ˆMáØ% }Ð5¸ÈÈÐ8KÑKˆFàˆMä"Ø-Ø'Ø)×7Ñ7Ø#2×#IÑ#Iô	
ð 	
r&   )TF©NNNN)r   r   r    r?   r7  r   ÚFOCALNET_INPUTS_DOCSTRINGr   Ú_CHECKPOINT_FOR_DOCr)   Ú_CONFIG_FOR_DOCÚ_EXPECTED_OUTPUT_SHAPEr   r"   r#   rc   r  r   r   ra   re   rf   s   @r'   r.  r.  ©  s«   ø„ õ
ò0ñ +Ð+DÓEÙØ&Ø'Ø$ØØ.ôð 59Ø6:Ø/3Ø&*ñ.
à˜u×0Ñ0Ñ1ð.
ð " %×"2Ñ"2Ñ3ð.
ð ' t™nð	.
ð
 ˜d‘^ð.
ð 
ˆuÐ)Ð)Ñ	*ò.
óó Fô.
r&   r.  a|  FocalNet Model with a decoder on top for masked image modeling.

    This follows the same implementation as in [SimMIM](https://arxiv.org/abs/2111.09886).

    <Tip>

    Note that we provide a script to pre-train this model on custom data in our [examples
    directory](https://github.com/huggingface/transformers/tree/main/examples/pytorch/image-pretraining).

    </Tip>
    c                   ó¾   ‡ — e Zd Zˆ fd„Z ee«       eee¬«      	 	 	 	 d	de	e
j                     de	e
j                     de	e   de	e   deeef   f
d„«       «       Zˆ xZS )
ÚFocalNetForMaskedImageModelingc                 ó¾  •— t         ‰| �  |«       t        |dd¬«      | _        t	        |j
                  «      | _        t        |j                  d| j                  dz
  z  z  «      }t        j                  t        j                  ||j                  dz  |j                  z  d¬«      t        j                  |j                  «      «      | _        | j!                  «        y )NFT)r5  rN   rj   r   )Úin_channelsÚout_channelsrl   )r>   r?   r.  r  rï   rð   rñ   rƒ   r9   r   r¶   rt   Úencoder_strider8   ÚPixelShuffleÚdecoderr4  )rM   r5   r0  rO   s      €r'   r?   z'FocalNetForMaskedImageModeling.__init__  s¯   ø€ Ü‰Ñ˜Ô ä% fÀÐVZÔ[ˆŒä˜fŸm™mÓ,ˆŒÜ˜6×+Ñ+¨a°D·O±OÀaÑ4GÑ.HÑHÓIˆÜ—}‘}Ü�I‰IØ(°v×7LÑ7LÈaÑ7OÐRX×ReÑReÑ7eÐstôô �O‰O˜F×1Ñ1Ó2ó	
ˆŒð 	�‰Õr&   ©r:  r(  rP   rQ   r  r
  rR   c                 ó´  — |�|n| j                   j                  }| j                  ||||¬«      }|d   }|j                  dd«      }|j                  \  }}}	t        j                  |	dz  «      x}
}|j                  |||
|«      }| j                  |«      }d}|��| j                   j                  | j                   j                  z  }|j                  d||«      }|j                  | j                   j                  d«      j                  | j                   j                  d«      j                  d«      j                  «       }t        j                  j!                  ||d¬	«      }||z  j#                  «       |j#                  «       d
z   z  | j                   j$                  z  }|s|f|dd z   }|�|f|z   S |S t'        |||j(                  |j*                  ¬«      S )aQ  
        bool_masked_pos (`torch.BoolTensor` of shape `(batch_size, num_patches)`):
            Boolean masked positions. Indicates which patches are masked (1) and which aren't (0).

        Returns:

        Examples:
        ```python
        >>> from transformers import AutoImageProcessor, FocalNetConfig, FocalNetForMaskedImageModeling
        >>> import torch
        >>> from PIL import Image
        >>> import requests

        >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
        >>> image = Image.open(requests.get(url, stream=True).raw)

        >>> image_processor = AutoImageProcessor.from_pretrained("microsoft/focalnet-base-simmim-window6-192")
        >>> config = FocalNetConfig()
        >>> model = FocalNetForMaskedImageModeling(config)

        >>> num_patches = (model.config.image_size // model.config.patch_size) ** 2
        >>> pixel_values = image_processor(images=image, return_tensors="pt").pixel_values
        >>> # create random boolean mask of shape (batch_size, num_patches)
        >>> bool_masked_pos = torch.randint(low=0, high=2, size=(1, num_patches)).bool()

        >>> outputs = model(pixel_values, bool_masked_pos=bool_masked_pos)
        >>> loss, reconstructed_pixel_values = outputs.loss, outputs.logits
        >>> list(reconstructed_pixel_values.shape)
        [1, 3, 192, 192]
        ```N)rQ   r  r
  r   r   rj   g      à?rT   Únone)Ú	reductiongñhãˆµøä>)r-   r.   r   r   )r5   r?  r  r‚   r   ÚmathÚfloorrþ   rP  r6   r7   Úrepeat_interleaverX   r½   r   rx   Úl1_lossrô   r8   r,   r   r   )rM   rP   rQ   r  r
  ÚoutputsrB  r\   r8   Úsequence_lengthrz   r{   Úreconstructed_pixel_valuesÚmasked_im_lossrV   r`   Úreconstruction_lossr‘   s                     r'   ra   z&FocalNetForMaskedImageModeling.forward  sÖ  € ðN &1Ð%<‘kÀ$Ç+Á+×B]ÑB]ˆà—-‘-ØØ+Ø!5Ø#ð	  ó 
ˆð " !™*ˆà)×3Ñ3°A°qÓ9ˆØ4C×4IÑ4IÑ1ˆ
�L /ÜŸ™ O°SÑ$8Ó9Ð9ˆ�Ø)×1Ñ1°*¸lÈFÐTYÓZˆð &*§\¡\°/Ó%BÐ"àˆØÑ&Ø—;‘;×)Ñ)¨T¯[©[×-CÑ-CÑCˆDØ-×5Ñ5°b¸$ÀÓEˆOà×1Ñ1°$·+±+×2HÑ2HÈ!ÓLß"Ñ" 4§;¡;×#9Ñ#9¸1Ó=ß‘˜1“ß‘“ð	 ô #%§-¡-×"7Ñ"7¸ÐF`ÐlrÐ"7Ó"sÐØ1°DÑ8×=Ñ=Ó?À4Ç8Á8Ã:ÐPTÑCTÑUÐX\×XcÑXc×XpÑXpÑpˆNáØ0Ð2°W¸Q¸R°[Ñ@ˆFØ3AÐ3M�^Ð%¨Ñ.ÐYÐSYÐYä0ØØ5Ø!×/Ñ/Ø#*×#AÑ#Aô	
ð 	
r&   rD  )r   r   r    r?   r   rE  r   r,   rG  r   r"   r#   rc   r  r   r   ra   re   rf   s   @r'   rJ  rJ  ù  s£   ø„ ôñ" +Ð+DÓEÙÐ+LÐ[jÔkð 59Ø6:Ø/3Ø&*ñN
à˜u×0Ñ0Ñ1ðN
ð " %×"2Ñ"2Ñ3ðN
ð ' t™nð	N
ð
 ˜d‘^ðN
ð 
ˆuÐ7Ð7Ñ	8òN
ó ló FôN
r&   rJ  z…
    FocalNet Model with an image classification head on top (a linear layer on top of the pooled output) e.g. for
    ImageNet.
    c                   óÂ   ‡ — e Zd Zˆ fd„Z ee«       eeee	e
¬«      	 	 	 	 d	deej                     deej                     dee   dee   deeef   f
d„«       «       Zˆ xZS )
ÚFocalNetForImageClassificationc                 ó>  •— t         ‰| �  |«       |j                  | _        t        |«      | _        |j                  dkD  r4t        j                  | j                  j                  |j                  «      nt        j                  «       | _	        | j                  «        y )Nr   )r>   r?   Ú
num_labelsr.  r  r   rª   r0  rÞ   Ú
classifierr4  ©rM   r5   rO   s     €r'   r?   z'FocalNetForImageClassification.__init__u  sx   ø€ Ü‰Ñ˜Ô à ×+Ñ+ˆŒÜ% fÓ-ˆŒð IO×HYÑHYÐ\]ÒH]ŒB�I‰I�d—m‘m×0Ñ0°&×2CÑ2CÔDÔce×cnÑcnÓcpð 	Œð
 	�‰Õr&   )r9  r:  r(  r<  rP   Úlabelsr  r
  rR   c                 ó  — |�|n| j                   j                  }| j                  |||¬«      }|d   }| j                  |«      }d}|��‡| j                   j                  €�| j
                  dk(  rd| j                   _        nl| j
                  dkD  rL|j                  t        j                  k(  s|j                  t        j                  k(  rd| j                   _        nd| j                   _        | j                   j                  dk(  rIt        «       }	| j
                  dk(  r& |	|j                  «       |j                  «       «      }nŒ |	||«      }n‚| j                   j                  dk(  r=t        «       }	 |	|j                  d| j
                  «      |j                  d«      «      }n,| j                   j                  dk(  rt        «       }	 |	||«      }|s|f|dd z   }
|�|f|
z   S |
S t        |||j                   |j"                  ¬	«      S )
aŠ  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the image classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        Nr>  r   Ú
regressionÚsingle_label_classificationÚmulti_label_classificationrT   rj   )r-   r1   r   r   )r5   r?  r  rb  Úproblem_typera  r‰   r"   Úlongrƒ   r
   Úsqueezer	   ré   r   r0   r   r   )rM   rP   rd  r  r
  rY  rC  r1   r-   Úloss_fctr‘   s              r'   ra   z&FocalNetForImageClassification.forwardƒ  sÈ  € ð( &1Ð%<‘kÀ$Ç+Á+×B]ÑB]ˆà—-‘-ØØ!5Ø#ð  ó 
ˆð   ™
ˆà—‘ Ó/ˆàˆØÑØ�{‰{×'Ñ'Ð/Ø—?‘? aÒ'Ø/;�D—K‘KÕ,Ø—_‘_ qÒ(¨f¯l©l¼e¿j¹jÒ.HÈFÏLÉLÔ\a×\eÑ\eÒLeØ/L�D—K‘KÕ,à/K�D—K‘KÔ,à�{‰{×'Ñ'¨<Ò7Ü"›9�Ø—?‘? aÒ'Ù# F§N¡NÓ$4°f·n±nÓ6FÓG‘Dá# F¨FÓ3‘DØ—‘×)Ñ)Ð-JÒJÜ+Ó-�Ù §¡¨B°·±Ó @À&Ç+Á+ÈbÃ/ÓR‘Ø—‘×)Ñ)Ð-IÒIÜ,Ó.�Ù ¨Ó/�áØ�Y ¨¨ Ñ,ˆFØ)-Ð)9�T�G˜fÑ$ÐE¸vÐEä,ØØØ!×/Ñ/Ø#*×#AÑ#Aô	
ð 	
r&   rD  )r   r   r    r?   r   rE  r   Ú_IMAGE_CLASS_CHECKPOINTr0   rG  Ú_IMAGE_CLASS_EXPECTED_OUTPUTr   r"   r#   Ú
LongTensorr  r   r   ra   re   rf   s   @r'   r_  r_  l  s£   ø„ ôñ +Ð+DÓEÙØ*Ø1Ø$Ø4ô	ð 59Ø-1Ø/3Ø&*ñ9
à˜u×0Ñ0Ñ1ð9
ð ˜×)Ñ)Ñ*ð9
ð ' t™nð	9
ð
 ˜d‘^ð9
ð 
ˆuÐ3Ð3Ñ	4ò9
óó Fô9
r&   r_  zG
    FocalNet backbone, to be used with frameworks like X-Decoder.
    c                   ó’   ‡ — e Zd Zdefˆ fd„Z ee«       eee	¬«      	 	 d	de
j                  dee   dee   defd„«       «       Zˆ xZS )
ÚFocalNetBackboner5   c                 óÀ   •— t         ‰| �  |«       t         ‰| �	  |«       |j                  g|j                  z   | _        t        |«      | _        | j                  «        y rb   )	r>   r?   Ú_init_backboner9   Úhidden_sizesr0  r.  r  r4  rc  s     €r'   r?   zFocalNetBackbone.__init__Í  sQ   ø€ Ü‰Ñ˜Ô Ü‰Ñ˜vÔ&à#×-Ñ-Ð.°×1DÑ1DÑDˆÔÜ% fÓ-ˆŒð 	�‰Õr&   rQ  rP   r  r
  rR   c                 óŽ  — |�|n| j                   j                  }|�|n| j                   j                  }| j                  |dd¬«      }|j                  }d}t        | j                  «      D ]  \  }}|| j                  v sŒ|||   fz  }Œ |s|f}	|r|	|j                  fz  }	|	S t        ||r|j                  d¬«      S dd¬«      S )a|  
        Returns:

        Examples:

        ```python
        >>> from transformers import AutoImageProcessor, AutoBackbone
        >>> import torch
        >>> from PIL import Image
        >>> import requests

        >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
        >>> image = Image.open(requests.get(url, stream=True).raw)

        >>> processor = AutoImageProcessor.from_pretrained("microsoft/focalnet-tiny-lrf")
        >>> model = AutoBackbone.from_pretrained("microsoft/focalnet-tiny-lrf")

        >>> inputs = processor(image, return_tensors="pt")
        >>> outputs = model(**inputs)
        ```NTr>  r%   )Úfeature_mapsr   Ú
attentions)
r5   r?  r  r  r   r  Ústage_namesrÔ   r   r   )
rM   rP   r  r
  rY  r   rv  ÚidxÚstager‘   s
             r'   ra   zFocalNetBackbone.forward×  sî   € ð8 &1Ð%<‘kÀ$Ç+Á+×B]ÑB]ˆà$8Ð$DÑ È$Ï+É+×JjÑJjð 	ð —-‘- À4ÐUY�-ÓZˆà×6Ñ6ˆàˆÜ# D×$4Ñ$4Ó5ò 	6‰JˆC�Ø˜×)Ñ)Ò)Ø ¨sÑ!3Ð 5Ñ5‘ð	6ñ Ø"�_ˆFÙ#Ø˜7×0Ñ0Ð2Ñ2�ØˆMäØ%Ù3G˜'×/Ñ/Øô
ð 	
àMQØô
ð 	
r&   )NN)r   r   r    r   r?   r   rE  r   r   rG  r"   rd   r   r  ra   re   rf   s   @r'   rq  rq  Æ  st   ø„ ð˜~õ ñ +Ð+DÓEÙ¨>ÈÔXð 04Ø&*ñ	2
à—l‘lð2
ð ' t™nð2
ð ˜d‘^ð	2
ð
 
ò2
ó Yó Fô2
r&   rq  )r_  rJ  rq  r.  r  )rˆ   F)Cr!   Úcollections.abcrp   rU  Údataclassesr   Útypingr   r   r   r"   Útorch.utils.checkpointr   Útorch.nnr   r	   r
   Úactivationsr   Úmodeling_outputsr   Úmodeling_utilsr   Úutilsr   r   r   r   r   r   Úutils.backbone_utilsr   Úconfiguration_focalnetr   Ú
get_loggerr   ÚloggerrG  rF  rH  rm  rn  r   r)   r,   r0   ÚModuler3   r@   rd   r›   r  r’   r”   rž   rÌ   r×   rí   r  r  ÚFOCALNET_START_DOCSTRINGrE  r.  rJ  r_  rq  Ú__all__r%   r&   r'   ú<module>r‹     s‰  ðñ ã Û Ý !ß )Ñ )ã Û Ý ß AÑ Aå !Ý .Ý -÷÷ õ 2Ý 2ð 
ˆ×	Ñ	˜HÓ	%€ð #€ð 0Ð Ú%Ð ð 4Ð Ø1Ð ð ôF˜Kó Fó ðFð4 ôF˜+ó Fó ðFð8 ôF¨ó Fó ðFð8 ôF Kó Fó ðFô8%-˜Ÿ™ô %-ôPD-˜bŸi™iô D-ñP�U—\‘\ð ¨eð ÀTð ÐV[×VbÑVbó ô*-�r—y‘yô -ôD˜Ÿ™ô DôN�"—)‘)ô ô&B�B—I‘Iô BôJ?�B—I‘Iô ?ôDO
�b—i‘iô O
ôdH˜oô Hð@	Ð ðÐ ñ Ø\ØóôI
Ð+ó I
ó	ðI
ñX ð
ð óôb
Ð%<ó b
óðb
ñJ ðð óôP
Ð%<ó P
óðP
ñf ðð ó	ô?
Ð.°ó ?
óð?
òD�r&   