Ë
    T^(hë ã            	       óÊ  — d Z ddlZddlZddlZddlmZ ddlmZm	Z	m
Z
 ddlZddlZddlmZmZ ddlmZmZmZ ddlmZ dd	lmZ dd
lmZ ddlmZmZmZ ddlmZmZm Z m!Z!m"Z"m#Z#m$Z$ ddl%m&Z& ddl'm(Z(  e"jR                  e*«      Z+dZ,dZ-g d¢Z.dZ/dZ0e G d„ de«      «       Z1e G d„ de«      «       Z2e G d„ de«      «       Z3e G d„ de«      «       Z4d„ Z5d„ Z6dLdej                  de7d e8d!ej                  fd"„Z9 G d#„ d$ejt                  «      Z; G d%„ d&ejt                  «      Z< G d'„ d(ejt                  «      Z= G d)„ d*ejt                  «      Z> G d+„ d,ejt                  «      Z? G d-„ d.ejt                  «      Z@ G d/„ d0ejt                  «      ZA G d1„ d2ejt                  «      ZB G d3„ d4ejt                  «      ZC G d5„ d6ejt                  «      ZD G d7„ d8ejt                  «      ZE G d9„ d:ejt                  «      ZF G d;„ d<e«      ZGd=ZHd>ZI e d?eH«       G d@„ dAeG«      «       ZJ e dBeH«       G dC„ dDeG«      «       ZK e dEeH«       G dF„ dGeG«      «       ZL e dHeH«       G dI„ dJeGe&«      «       ZMg dK¢ZNy)Mz!PyTorch Swinv2 Transformer model.é    N)Ú	dataclass)ÚOptionalÚTupleÚUnion)ÚTensorÚnn)ÚBCEWithLogitsLossÚCrossEntropyLossÚMSELossé   )ÚACT2FN)ÚBackboneOutput)ÚPreTrainedModel)Ú find_pruneable_heads_and_indicesÚmeshgridÚprune_linear_layer)ÚModelOutputÚadd_code_sample_docstringsÚadd_start_docstringsÚ%add_start_docstrings_to_model_forwardÚloggingÚreplace_return_docstringsÚ	torch_int)ÚBackboneMixiné   )ÚSwinv2Configr   z(microsoft/swinv2-tiny-patch4-window8-256)r   é@   i   zEgyptian catc                   óÐ   — e Zd ZU dZdZeej                     ed<   dZ	ee
ej                  df      ed<   dZee
ej                  df      ed<   dZee
ej                  df      ed<   y)ÚSwinv2EncoderOutputa‚  
    Swinv2 encoder's outputs, with potential hidden states and attentions.

    Args:
        last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
            Sequence of hidden-states at the output of the last layer of the model.
        hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
            Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each stage) of
            shape `(batch_size, sequence_length, hidden_size)`.

            Hidden-states of the model at the output of each layer plus the initial embedding outputs.
        attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
            Tuple of `torch.FloatTensor` (one for each stage) of shape `(batch_size, num_heads, sequence_length,
            sequence_length)`.

            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
            heads.
        reshaped_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
            Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each stage) of
            shape `(batch_size, hidden_size, height, width)`.

            Hidden-states of the model at the output of each layer plus the initial embedding outputs reshaped to
            include the spatial dimensions.
    NÚlast_hidden_state.Úhidden_statesÚ
attentionsÚreshaped_hidden_states)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r    r   ÚtorchÚFloatTensorÚ__annotations__r!   r   r"   r#   © ó    úh/var/www/skyplay_api_hub/venv/lib/python3.12/site-packages/transformers/models/swinv2/modeling_swinv2.pyr   r   >   s}   … ñð2 6:Ð�x × 1Ñ 1Ñ2Ó9Ø=A€M�8˜E %×"3Ñ"3°SÐ"8Ñ9Ñ:ÓAØ:>€J�˜˜u×0Ñ0°#Ð5Ñ6Ñ7Ó>ØFJÐ˜H U¨5×+<Ñ+<¸cÐ+AÑ%BÑCÔJr,   r   c                   óø   — e Zd ZU dZdZeej                     ed<   dZ	eej                     ed<   dZ
eeej                  df      ed<   dZeeej                  df      ed<   dZeeej                  df      ed<   y)	ÚSwinv2ModelOutputaV  
    Swinv2 model's outputs that also contains a pooling of the last hidden states.

    Args:
        last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
            Sequence of hidden-states at the output of the last layer of the model.
        pooler_output (`torch.FloatTensor` of shape `(batch_size, hidden_size)`, *optional*, returned when `add_pooling_layer=True` is passed):
            Average pooling of the last layer hidden-state.
        hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
            Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each stage) of
            shape `(batch_size, sequence_length, hidden_size)`.

            Hidden-states of the model at the output of each layer plus the initial embedding outputs.
        attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
            Tuple of `torch.FloatTensor` (one for each stage) of shape `(batch_size, num_heads, sequence_length,
            sequence_length)`.

            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
            heads.
        reshaped_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
            Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each stage) of
            shape `(batch_size, hidden_size, height, width)`.

            Hidden-states of the model at the output of each layer plus the initial embedding outputs reshaped to
            include the spatial dimensions.
    Nr    Úpooler_output.r!   r"   r#   )r$   r%   r&   r'   r    r   r(   r)   r*   r0   r!   r   r"   r#   r+   r,   r-   r/   r/   `   s’   … ñð6 6:Ð�x × 1Ñ 1Ñ2Ó9Ø15€M�8˜E×-Ñ-Ñ.Ó5Ø=A€M�8˜E %×"3Ñ"3°SÐ"8Ñ9Ñ:ÓAØ:>€J�˜˜u×0Ñ0°#Ð5Ñ6Ñ7Ó>ØFJÐ˜H U¨5×+<Ñ+<¸cÐ+AÑ%BÑCÔJr,   r/   c                   ó  — e Zd ZU dZdZeej                     ed<   dZ	eej                     ed<   dZ
eeej                  df      ed<   dZeeej                  df      ed<   dZeeej                  df      ed<   ed	„ «       Zy)
ÚSwinv2MaskedImageModelingOutputaÉ  
    Swinv2 masked image model outputs.

    Args:
        loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `bool_masked_pos` is provided):
            Masked image modeling (MLM) loss.
        reconstruction (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`):
            Reconstructed pixel values.
        hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
            Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each stage) of
            shape `(batch_size, sequence_length, hidden_size)`.

            Hidden-states of the model at the output of each layer plus the initial embedding outputs.
        attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
            Tuple of `torch.FloatTensor` (one for each stage) of shape `(batch_size, num_heads, sequence_length,
            sequence_length)`.

            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
            heads.
        reshaped_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
            Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each stage) of
            shape `(batch_size, hidden_size, height, width)`.

            Hidden-states of the model at the output of each layer plus the initial embedding outputs reshaped to
            include the spatial dimensions.
    NÚlossÚreconstruction.r!   r"   r#   c                 óN   — t        j                  dt        «       | j                  S )Nzžlogits attribute is deprecated and will be removed in version 5 of Transformers. Please use the reconstruction attribute to retrieve the final output instead.)ÚwarningsÚwarnÚFutureWarningr4   ©Úselfs    r-   Úlogitsz&Swinv2MaskedImageModelingOutput.logits©   s%   € ä�‰ð]äô	
ð
 ×"Ñ"Ð"r,   )r$   r%   r&   r'   r3   r   r(   r)   r*   r4   r!   r   r"   r#   Úpropertyr;   r+   r,   r-   r2   r2   …   s£   … ñð6 )-€Dˆ(�5×$Ñ$Ñ
%Ó,Ø26€N�H˜U×.Ñ.Ñ/Ó6Ø=A€M�8˜E %×"3Ñ"3°SÐ"8Ñ9Ñ:ÓAØ:>€J�˜˜u×0Ñ0°#Ð5Ñ6Ñ7Ó>ØFJÐ˜H U¨5×+<Ñ+<¸cÐ+AÑ%BÑCÓJàñ#ó ñ#r,   r2   c                   óø   — e Zd ZU dZdZeej                     ed<   dZ	eej                     ed<   dZ
eeej                  df      ed<   dZeeej                  df      ed<   dZeeej                  df      ed<   y)	ÚSwinv2ImageClassifierOutputa  
    Swinv2 outputs for image classification.

    Args:
        loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
            Classification (or regression if config.num_labels==1) loss.
        logits (`torch.FloatTensor` of shape `(batch_size, config.num_labels)`):
            Classification (or regression if config.num_labels==1) scores (before SoftMax).
        hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
            Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each stage) of
            shape `(batch_size, sequence_length, hidden_size)`.

            Hidden-states of the model at the output of each layer plus the initial embedding outputs.
        attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
            Tuple of `torch.FloatTensor` (one for each stage) of shape `(batch_size, num_heads, sequence_length,
            sequence_length)`.

            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
            heads.
        reshaped_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
            Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each stage) of
            shape `(batch_size, hidden_size, height, width)`.

            Hidden-states of the model at the output of each layer plus the initial embedding outputs reshaped to
            include the spatial dimensions.
    Nr3   r;   .r!   r"   r#   )r$   r%   r&   r'   r3   r   r(   r)   r*   r;   r!   r   r"   r#   r+   r,   r-   r>   r>   ³   s‘   … ñð6 )-€Dˆ(�5×$Ñ$Ñ
%Ó,Ø*.€FˆH�U×&Ñ&Ñ'Ó.Ø=A€M�8˜E %×"3Ñ"3°SÐ"8Ñ9Ñ:ÓAØ:>€J�˜˜u×0Ñ0°#Ð5Ñ6Ñ7Ó>ØFJÐ˜H U¨5×+<Ñ+<¸cÐ+AÑ%BÑCÔJr,   r>   c                 óÌ   — | j                   \  }}}}| j                  |||z  |||z  ||«      } | j                  dddddd«      j                  «       j                  d|||«      }|S )z2
    Partitions the given input into windows.
    r   r   r   é   é   é   éÿÿÿÿ©ÚshapeÚviewÚpermuteÚ
contiguous)Úinput_featureÚwindow_sizeÚ
batch_sizeÚheightÚwidthÚnum_channelsÚwindowss          r-   Úwindow_partitionrP   Ù   s}   € ð /<×.AÑ.AÑ+€J�˜˜|Ø!×&Ñ&Ø�F˜kÑ)¨;¸ÀÑ8LÈkÐ[gó€Mð ×#Ñ# A q¨!¨Q°°1Ó5×@Ñ@ÓB×GÑGÈÈKÐYdÐfrÓs€GØ€Nr,   c                 óÈ   — | j                   d   }| j                  d||z  ||z  |||«      } | j                  dddddd«      j                  «       j                  d|||«      } | S )z?
    Merges windows to produce higher resolution features.
    rC   r   r   r   r@   rA   rB   rD   )rO   rJ   rL   rM   rN   s        r-   Úwindow_reverserR   æ   sn   € ð —=‘= Ñ$€LØ�l‰l˜2˜v¨Ñ4°e¸{Ñ6JÈKÐYdÐfrÓs€GØ�o‰o˜a  A q¨!¨QÓ/×:Ñ:Ó<×AÑAÀ"ÀfÈeÐUaÓb€GØ€Nr,   ÚinputÚ	drop_probÚtrainingÚreturnc                 ó  — |dk(  s|s| S d|z
  }| j                   d   fd| j                  dz
  z  z   }|t        j                  || j                  | j
                  ¬«      z   }|j                  «        | j                  |«      |z  }|S )aF  
    Drop paths (Stochastic Depth) per sample (when applied in main path of residual blocks).

    Comment by Ross Wightman: This is the same as the DropConnect impl I created for EfficientNet, etc networks,
    however, the original name is misleading as 'Drop Connect' is a different form of dropout in a separate paper...
    See discussion: https://github.com/tensorflow/tpu/issues/494#issuecomment-532968956 ... I've opted for changing the
    layer and argument names to 'drop path' rather than mix DropConnect as a layer name and use 'survival rate' as the
    argument.
    ç        r   r   )r   )ÚdtypeÚdevice)rE   Úndimr(   ÚrandrY   rZ   Úfloor_Údiv)rS   rT   rU   Ú	keep_probrE   Úrandom_tensorÚoutputs          r-   Ú	drop_pathrb   ñ   s�   € ð �CÒ™xØˆØ�I‘€IØ�[‰[˜‰^Ð ¨¯
©
°Q©Ñ 7Ñ7€EØ¤§
¡
¨5¸¿¹ÈEÏLÉLÔ YÑY€MØ×ÑÔØ�Y‰Y�yÓ! MÑ1€FØ€Mr,   c                   óx   ‡ — e Zd ZdZd	dee   ddfˆ fd„Zdej                  dej                  fd„Z	de
fd„Zˆ xZS )
ÚSwinv2DropPathzXDrop paths (Stochastic Depth) per sample (when applied in main path of residual blocks).NrT   rV   c                 ó0   •— t         ‰| �  «        || _        y ©N)ÚsuperÚ__init__rT   )r:   rT   Ú	__class__s     €r-   rh   zSwinv2DropPath.__init__	  s   ø€ Ü‰ÑÔØ"ˆ�r,   r!   c                 óD   — t        || j                  | j                  «      S rf   )rb   rT   rU   ©r:   r!   s     r-   ÚforwardzSwinv2DropPath.forward  s   € Ü˜¨¯©¸¿¹ÓFÐFr,   c                 ó8   — dj                  | j                  «      S )Nzp={})ÚformatrT   r9   s    r-   Ú
extra_reprzSwinv2DropPath.extra_repr  s   € Ø�}‰}˜TŸ^™^Ó,Ð,r,   rf   )r$   r%   r&   r'   r   Úfloatrh   r(   r   rl   Ústrro   Ú__classcell__©ri   s   @r-   rd   rd     sG   ø„ Ùbñ# (¨5¡/ð #¸Tõ #ðG U§\¡\ð G°e·l±ló Gð-˜C÷ -r,   rd   c            
       óÐ   ‡ — e Zd ZdZdˆ fd„	Zdej                  dededej                  fd„Z	 	 dde	ej                     d	e	ej                     d
edeej                     fd„Zˆ xZS )ÚSwinv2EmbeddingszW
    Construct the patch and position embeddings. Optionally, also the mask token.
    c                 ó~  •— t         ‰| �  «        t        |«      | _        | j                  j                  }| j                  j
                  | _        |r4t        j                  t        j                  dd|j                  «      «      nd | _        |j                  r=t        j                  t        j                  d|dz   |j                  «      «      | _        nd | _        t        j                  |j                  «      | _        t        j"                  |j$                  «      | _        |j(                  | _        || _        y )Nr   )rg   rh   ÚSwinv2PatchEmbeddingsÚpatch_embeddingsÚnum_patchesÚ	grid_sizeÚ
patch_gridr   Ú	Parameterr(   ÚzerosÚ	embed_dimÚ
mask_tokenÚuse_absolute_embeddingsÚposition_embeddingsÚ	LayerNormÚnormÚDropoutÚhidden_dropout_probÚdropoutÚ
patch_sizeÚconfig)r:   rˆ   Úuse_mask_tokenry   ri   s       €r-   rh   zSwinv2Embeddings.__init__  sâ   ø€ Ü‰ÑÔä 5°fÓ =ˆÔØ×+Ñ+×7Ñ7ˆØ×/Ñ/×9Ñ9ˆŒÙO]œ"Ÿ,™,¤u§{¡{°1°a¸×9IÑ9IÓ'JÔKÐcgˆŒà×)Ò)Ü')§|¡|´E·K±KÀÀ;ÐQRÁ?ÐTZ×TdÑTdÓ4eÓ'fˆDÕ$à'+ˆDÔ$ä—L‘L ×!1Ñ!1Ó2ˆŒ	Ü—z‘z &×"<Ñ"<Ó=ˆŒØ ×+Ñ+ˆŒØˆ�r,   Ú
embeddingsrL   rM   rV   c                 ó¦  — |j                   d   dz
  }| j                  j                   d   dz
  }t        j                  j	                  «       s||k(  r||k(  r| j                  S | j                  dd…dd…f   }| j                  dd…dd…f   }|j                   d   }|| j
                  z  }	|| j
                  z  }
t        |dz  «      }|j                  d|||«      }|j                  dddd«      }t        j                  j                  ||	|
fdd	¬
«      }|j                  dddd«      j                  dd|«      }t        j                  ||fd¬«      S )a   
        This method allows to interpolate the pre-trained position encodings, to be able to use the model on higher resolution
        images. This method is also adapted to support torch.jit tracing.

        Adapted from:
        - https://github.com/facebookresearch/dino/blob/de9ee3df6cf39fac952ab558447af1fa1365362a/vision_transformer.py#L174-L194, and
        - https://github.com/facebookresearch/dinov2/blob/e1277af2ba9496fbadf7aec6eba56e8d882d1e35/dinov2/models/vision_transformer.py#L179-L211
        r   NrC   ç      à?r   r   r@   ÚbicubicF)ÚsizeÚmodeÚalign_corners©Údim)rE   r�   r(   ÚjitÚ
is_tracingr‡   r   ÚreshaperG   r   Ú
functionalÚinterpolaterF   Úcat)r:   rŠ   rL   rM   ry   Únum_positionsÚclass_pos_embedÚpatch_pos_embedr’   Ú
new_heightÚ	new_widthÚsqrt_num_positionss               r-   Úinterpolate_pos_encodingz)Swinv2Embeddings.interpolate_pos_encoding-  s`  € ð !×&Ñ& qÑ)¨AÑ-ˆØ×0Ñ0×6Ñ6°qÑ9¸AÑ=ˆô �y‰y×#Ñ#Ô%¨+¸Ò*FÈ6ÐUZÊ?Ø×+Ñ+Ð+à×2Ñ2²1°b°q°b°5Ñ9ˆØ×2Ñ2²1°a±b°5Ñ9ˆà×Ñ˜rÑ"ˆà˜tŸ™Ñ.ˆ
Ø˜TŸ_™_Ñ,ˆ	ä& }°cÑ'9Ó:ÐØ)×1Ñ1°!Ð5GÐI[Ð]`ÓaˆØ)×1Ñ1°!°Q¸¸1Ó=ˆäŸ-™-×3Ñ3ØØ˜iÐ(ØØð	 4ó 
ˆð *×1Ñ1°!°Q¸¸1Ó=×BÑBÀ1ÀbÈ#ÓNˆä�y‰y˜/¨?Ð;ÀÔCÐCr,   Úpixel_valuesÚbool_masked_posrŸ   c                 óÂ  — |j                   \  }}}}| j                  |«      \  }}	| j                  |«      }|j                  «       \  }
}}|�K| j                  j                  |
|d«      }|j                  d«      j                  |«      }|d|z
  z  ||z  z   }| j                  �(|r|| j                  |||«      z   }n|| j                  z   }| j                  |«      }||	fS )NrC   ç      ð?)rE   rx   rƒ   rŽ   r   ÚexpandÚ	unsqueezeÚtype_asr�   rŸ   r†   )r:   r    r¡   rŸ   Ú_rN   rL   rM   rŠ   Úoutput_dimensionsrK   Úseq_lenÚmask_tokensÚmasks                 r-   rl   zSwinv2Embeddings.forwardU  sô   € ð *6×);Ñ);Ñ&ˆˆ<˜ Ø(,×(=Ñ(=¸lÓ(KÑ%ˆ
Ð%Ø—Y‘Y˜zÓ*ˆ
Ø!+§¡Ó!2Ñˆ
�G˜QàÐ&ØŸ/™/×0Ñ0°¸WÀbÓIˆKà"×,Ñ,¨RÓ0×8Ñ8¸ÓEˆDØ# s¨T¡zÑ2°[À4Ñ5GÑGˆJà×#Ñ#Ð/Ù'Ø'¨$×*GÑ*GÈ
ÐTZÐ\aÓ*bÑb‘
à'¨$×*BÑ*BÑB�
à—\‘\ *Ó-ˆ
àÐ,Ð,Ð,r,   )F©NF)r$   r%   r&   r'   rh   r(   r   ÚintrŸ   r   r)   Ú
BoolTensorÚboolr   rl   rr   rs   s   @r-   ru   ru     s’   ø„ ñõð&&D°5·<±<ð &DÈð &DÐUXð &DÐ]b×]iÑ]ió &DðV 7;Ø).ñ	-à˜u×0Ñ0Ñ1ð-ð " %×"2Ñ"2Ñ3ð-ð #'ð	-ð
 
ˆu�|‰|Ñ	÷-r,   ru   c                   óv   ‡ — e Zd ZdZˆ fd„Zd„ Zdeej                     de	ej                  e	e   f   fd„Zˆ xZS )rw   zì
    This class turns `pixel_values` of shape `(batch_size, num_channels, height, width)` into the initial
    `hidden_states` (patch embeddings) of shape `(batch_size, seq_length, hidden_size)` to be consumed by a
    Transformer.
    c                 ó  •— t         ‰| �  «        |j                  |j                  }}|j                  |j
                  }}t        |t        j                  j                  «      r|n||f}t        |t        j                  j                  «      r|n||f}|d   |d   z  |d   |d   z  z  }|| _        || _        || _        || _
        |d   |d   z  |d   |d   z  f| _        t        j                  ||||¬«      | _        y )Nr   r   )Úkernel_sizeÚstride)rg   rh   Ú
image_sizer‡   rN   r~   Ú
isinstanceÚcollectionsÚabcÚIterablery   rz   r   ÚConv2dÚ
projection)r:   rˆ   r´   r‡   rN   Úhidden_sizery   ri   s          €r-   rh   zSwinv2PatchEmbeddings.__init__y  sû   ø€ Ü‰ÑÔØ!'×!2Ñ!2°F×4EÑ4E�Jˆ
Ø$*×$7Ñ$7¸×9IÑ9I�kˆÜ#-¨j¼+¿/¹/×:RÑ:RÔ#S‘ZÐZdÐfpÐYqˆ
Ü#-¨j¼+¿/¹/×:RÑ:RÔ#S‘ZÐZdÐfpÐYqˆ
Ø! !‘}¨
°1©Ñ5¸*ÀQ¹-È:ÐVWÉ=Ñ:XÑYˆØ$ˆŒØ$ˆŒØ(ˆÔØ&ˆÔØ$ Q™-¨:°a©=Ñ8¸*ÀQ¹-È:ÐVWÉ=Ñ:XÐYˆŒäŸ)™) L°+È:Ð^hÔiˆ�r,   c                 ón  — || j                   d   z  dk7  rDd| j                   d   || j                   d   z  z
  f}t        j                  j                  ||«      }|| j                   d   z  dk7  rFddd| j                   d   || j                   d   z  z
  f}t        j                  j                  ||«      }|S )Nr   r   )r‡   r   r–   Úpad)r:   r    rL   rM   Ú
pad_valuess        r-   Ú	maybe_padzSwinv2PatchEmbeddings.maybe_padˆ  s±   € Ø�4—?‘? 1Ñ%Ñ%¨Ò*Ø˜TŸ_™_¨QÑ/°%¸$¿/¹/È!Ñ:LÑ2LÑLÐMˆJÜŸ=™=×,Ñ,¨\¸:ÓFˆLØ�D—O‘O AÑ&Ñ&¨!Ò+Ø˜Q  4§?¡?°1Ñ#5¸ÀÇÁÐQRÑASÑ8SÑ#SÐTˆJÜŸ=™=×,Ñ,¨\¸:ÓFˆLØÐr,   r    rV   c                 óà   — |j                   \  }}}}| j                  |||«      }| j                  |«      }|j                   \  }}}}||f}|j                  d«      j	                  dd«      }||fS )Nr@   r   )rE   r¿   rº   ÚflattenÚ	transpose)r:   r    r§   rN   rL   rM   rŠ   r¨   s           r-   rl   zSwinv2PatchEmbeddings.forward‘  s}   € Ø)5×);Ñ);Ñ&ˆˆ<˜ à—~‘~ l°F¸EÓBˆØ—_‘_ \Ó2ˆ
Ø(×.Ñ.Ñˆˆ1ˆf�eØ# U˜OÐØ×'Ñ'¨Ó*×4Ñ4°Q¸Ó:ˆ
àÐ,Ð,Ð,r,   )r$   r%   r&   r'   rh   r¿   r   r(   r)   r   r   r­   rl   rr   rs   s   @r-   rw   rw   r  sF   ø„ ñôjòð	- H¨U×->Ñ->Ñ$?ð 	-ÀEÈ%Ï,É,ÐX]Ð^aÑXbÐJbÑDc÷ 	-r,   rw   c            	       ó²   ‡ — e Zd ZdZej
                  fdee   dedej                  ddfˆ fd„Z	d„ Z
d	ej                  d
eeef   dej                  fd„Zˆ xZS )ÚSwinv2PatchMerginga'  
    Patch Merging Layer.

    Args:
        input_resolution (`Tuple[int]`):
            Resolution of input feature.
        dim (`int`):
            Number of input channels.
        norm_layer (`nn.Module`, *optional*, defaults to `nn.LayerNorm`):
            Normalization layer class.
    Úinput_resolutionr’   Ú
norm_layerrV   Nc                 ó¤   •— t         ‰| �  «        || _        || _        t	        j
                  d|z  d|z  d¬«      | _         |d|z  «      | _        y )NrA   r@   F©Úbias)rg   rh   rÅ   r’   r   ÚLinearÚ	reductionrƒ   )r:   rÅ   r’   rÆ   ri   s       €r-   rh   zSwinv2PatchMerging.__init__ª  sI   ø€ Ü‰ÑÔØ 0ˆÔØˆŒÜŸ™ 1 s¡7¨A°©G¸%Ô@ˆŒÙ˜q 3™wÓ'ˆ�	r,   c                 óŠ   — |dz  dk(  xs |dz  dk(  }|r.ddd|dz  d|dz  f}t         j                  j                  ||«      }|S )Nr@   r   r   )r   r–   r½   )r:   rI   rL   rM   Ú
should_padr¾   s         r-   r¿   zSwinv2PatchMerging.maybe_pad±  sU   € Ø˜q‘j A‘oÒ:¨5°1©9¸©>ˆ
ÙØ˜Q  5¨1¡9¨a°¸!±Ð<ˆJÜŸM™M×-Ñ-¨m¸ZÓHˆMàÐr,   rI   Úinput_dimensionsc                 óº  — |\  }}|j                   \  }}}|j                  ||||«      }| j                  |||«      }|d d …dd d…dd d…d d …f   }|d d …dd d…dd d…d d …f   }	|d d …dd d…dd d…d d …f   }
|d d …dd d…dd d…d d …f   }t        j                  ||	|
|gd«      }|j                  |dd|z  «      }| j                  |«      }| j                  |«      }|S )Nr   r@   r   rC   rA   )rE   rF   r¿   r(   r˜   rË   rƒ   )r:   rI   rÎ   rL   rM   rK   r’   rN   Úinput_feature_0Úinput_feature_1Úinput_feature_2Úinput_feature_3s               r-   rl   zSwinv2PatchMerging.forward¹  s  € Ø(‰ˆ�à(5×(;Ñ(;Ñ%ˆ
�C˜à%×*Ñ*¨:°v¸uÀlÓSˆàŸ™ }°f¸eÓDˆà'ª¨1¨4¨a¨4°°°A°²qÐ(8Ñ9ˆà'ª¨1¨4¨a¨4°°°A°²qÐ(8Ñ9ˆà'ª¨1¨4¨a¨4°°°A°²qÐ(8Ñ9ˆà'ª¨1¨4¨a¨4°°°A°²qÐ(8Ñ9ˆäŸ	™	 ?°OÀ_ÐVeÐ"fÐhjÓkˆØ%×*Ñ*¨:°r¸1¸|Ñ;KÓLˆàŸ™ }Ó5ˆØŸ	™	 -Ó0ˆàÐr,   )r$   r%   r&   r'   r   r‚   r   r­   ÚModulerh   r¿   r(   r   rl   rr   rs   s   @r-   rÄ   rÄ   �  sr   ø„ ñ
ð XZ×WcÑWcñ (¨¨s©ð (¸#ð (È2Ï9É9ð (Ðhlõ (òð U§\¡\ð ÀUÈ3ÐPSÈ8Á_ð ÐY^×YeÑYe÷ r,   rÄ   c                   ó¸   ‡ — e Zd Zddgfˆ fd„	Zd„ Z	 	 	 d
dej                  deej                     deej                     dee	   de
ej                     f
d	„Zˆ xZS )ÚSwinv2SelfAttentionr   c           
      ó  •— t         ‰| �  «        ||z  dk7  rt        d|› d|› d�«      ‚|| _        t	        ||z  «      | _        | j                  | j
                  z  | _        t        |t        j                  j                  «      r|n||f| _        || _        t        j                  t        j                   dt        j"                  |ddf«      z  «      «      | _        t        j&                  t        j(                  ddd	¬
«      t        j*                  d	¬«      t        j(                  d|d¬
«      «      | _        t        j.                  | j                  d   dz
   | j                  d   t        j0                  ¬«      j3                  «       }t        j.                  | j                  d   dz
   | j                  d   t        j0                  ¬«      j3                  «       }t        j4                  t7        ||gd¬«      «      j9                  ddd«      j;                  «       j=                  d«      }|d   dkD  r;|d d …d d …d d …dfxx   |d   dz
  z  cc<   |d d …d d …d d …dfxx   |d   dz
  z  cc<   nS|dkD  rN|d d …d d …d d …dfxx   | j                  d   dz
  z  cc<   |d d …d d …d d …dfxx   | j                  d   dz
  z  cc<   |dz  }t        j>                  |«      t        j@                  t        jB                  |«      dz   «      z  tE        j@                  d«      z  }|jG                  tI        | j,                  jK                  «       «      jL                  «      }| jO                  d|d¬«       t        j.                  | j                  d   «      }	t        j.                  | j                  d   «      }
t        j4                  t7        |	|
gd¬«      «      }t        jP                  |d«      }|d d …d d …d f   |d d …d d d …f   z
  }|j9                  ddd«      j;                  «       }|d d …d d …dfxx   | j                  d   dz
  z  cc<   |d d …d d …dfxx   | j                  d   dz
  z  cc<   |d d …d d …dfxx   d| j                  d   z  dz
  z  cc<   |jS                  d«      }| jO                  d|d¬«       t        j(                  | j                  | j                  |jT                  ¬
«      | _+        t        j(                  | j                  | j                  d¬
«      | _,        t        j(                  | j                  | j                  |jT                  ¬
«      | _-        t        j\                  |j^                  «      | _0        y )Nr   zThe hidden size (z6) is not a multiple of the number of attention heads (ú)é
   r   r@   i   TrÈ   )ÚinplaceF©rY   Úij)Úindexingé   r£   Úrelative_coords_table)Ú
persistentrC   Úrelative_position_index)1rg   rh   Ú
ValueErrorÚnum_attention_headsr­   Úattention_head_sizeÚall_head_sizerµ   r¶   r·   r¸   rJ   Úpretrained_window_sizer   r|   r(   ÚlogÚonesÚlogit_scaleÚ
SequentialrÊ   ÚReLUÚcontinuous_position_bias_mlpÚarangeÚint64rp   Ústackr   rG   rH   r¥   ÚsignÚlog2ÚabsÚmathÚtoÚnextÚ
parametersrY   Úregister_bufferrÁ   ÚsumÚqkv_biasÚqueryÚkeyÚvaluer„   Úattention_probs_dropout_probr†   )r:   rˆ   r’   Ú	num_headsrJ   ræ   Úrelative_coords_hÚrelative_coords_wrß   Úcoords_hÚcoords_wÚcoordsÚcoords_flattenÚrelative_coordsrá   ri   s                  €r-   rh   zSwinv2SelfAttention.__init__Ô  sÊ  ø€ Ü‰ÑÔØ�‰?˜aÒÜØ# C 5Ð(^Ð_hÐ^iÐijÐkóð ð $-ˆÔ Ü#& s¨Y¡Ó#7ˆÔ Ø!×5Ñ5¸×8PÑ8PÑPˆÔä% k´;·?±?×3KÑ3KÔL‰KÐS^Ð`kÐRlð 	Ôð '=ˆÔ#ÜŸ<™<¬¯	©	°"´u·z±zÀ9ÈaÐQRÐBSÓ7TÑ2TÓ(UÓVˆÔä,.¯M©MÜ�I‰I�a˜ 4Ô(¬"¯'©'¸$Ô*?ÄÇÁÈ3ÐPYÐ`eÔAfó-
ˆÔ)ô
 "ŸL™L¨4×+;Ñ+;¸AÑ+>ÀÑ+BÐ)CÀT×EUÑEUÐVWÑEXÔ`e×`kÑ`kÔl×rÑrÓtÐÜ!ŸL™L¨4×+;Ñ+;¸AÑ+>ÀÑ+BÐ)CÀT×EUÑEUÐVWÑEXÔ`e×`kÑ`kÔl×rÑrÓtÐä�K‰KœÐ"3Ð5FÐ!GÐRVÔWÓXß‰W�Q˜˜1Óß‰Z‹\ß‰Y�q‹\ð	 	ð " !Ñ$ qÒ(Ø!¢!¢Qª¨1 *Ó-Ð1GÈÑ1JÈQÑ1NÑNÓ-Ø!¢!¢Qª¨1 *Ó-Ð1GÈÑ1JÈQÑ1NÑNÔ-Ø˜1Š_Ø!¢!¢Qª¨1 *Ó-°×1AÑ1AÀ!Ñ1DÀqÑ1HÑHÓ-Ø!¢!¢Qª¨1 *Ó-°×1AÑ1AÀ!Ñ1DÀqÑ1HÑHÓ-Ø Ñ"Ðä�J‰JÐ,Ó-´·
±
¼5¿9¹9ÐEZÓ;[Ð^aÑ;aÓ0bÑbÔei×enÑenÐopÓeqÑqð 	ð !6× 8Ñ 8¼¸d×>_Ñ>_×>jÑ>jÓ>lÓ9m×9sÑ9sÓ tÐØ×ÑÐ4Ð6KÐX]ÐÔ^ô —<‘< × 0Ñ 0°Ñ 3Ó4ˆÜ—<‘< × 0Ñ 0°Ñ 3Ó4ˆÜ—‘œX x°Ð&:ÀTÔJÓKˆÜŸ™ v¨qÓ1ˆØ(ªªA¨t¨Ñ4°~ÂaÈÊqÀjÑ7QÑQˆØ)×1Ñ1°!°Q¸Ó:×EÑEÓGˆØšš1˜a˜Ó  D×$4Ñ$4°QÑ$7¸!Ñ$;Ñ;Ó Øšš1˜a˜Ó  D×$4Ñ$4°QÑ$7¸!Ñ$;Ñ;Ó Øšš1˜a˜Ó  A¨×(8Ñ(8¸Ñ(;Ñ$;¸aÑ$?Ñ?Ó Ø"1×"5Ñ"5°bÓ"9ÐØ×ÑÐ6Ð8OÐ\aÐÔbä—Y‘Y˜t×1Ñ1°4×3EÑ3EÈFÏOÉOÔ\ˆŒ
Ü—9‘9˜T×/Ñ/°×1CÑ1CÈ%ÔPˆŒÜ—Y‘Y˜t×1Ñ1°4×3EÑ3EÈFÏOÉOÔ\ˆŒ
Ü—z‘z &×"EÑ"EÓFˆ�r,   c                 ó¤   — |j                  «       d d | j                  | j                  fz   }|j                  |«      }|j	                  dddd«      S )NrC   r   r@   r   r   )rŽ   rã   rä   rF   rG   )r:   ÚxÚnew_x_shapes      r-   Útranspose_for_scoresz(Swinv2SelfAttention.transpose_for_scores  sL   € Ø—f‘f“h˜s �m t×'?Ñ'?À×AYÑAYÐ&ZÑZˆØ�F‰F�;ÓˆØ�y‰y˜˜A˜q !Ó$Ð$r,   r!   Úattention_maskÚ	head_maskÚoutput_attentionsrV   c                 óz  — |j                   \  }}}| j                  |«      }| j                  | j                  |«      «      }	| j                  | j	                  |«      «      }
| j                  |«      }t
        j                  j                  |d¬«      t
        j                  j                  |	d¬«      j                  dd«      z  }t        j                  | j                  t        j                  d«      ¬«      j                  «       }||z  }| j                  | j                   «      j#                  d| j$                  «      }|| j&                  j#                  d«         j#                  | j(                  d   | j(                  d   z  | j(                  d   | j(                  d   z  d«      }|j+                  ddd«      j-                  «       }d	t        j.                  |«      z  }||j1                  d«      z   }|�“|j                   d   }|j#                  ||z  || j$                  ||«      |j1                  d«      j1                  d«      z   }||j1                  d«      j1                  d«      z   }|j#                  d| j$                  ||«      }t
        j                  j3                  |d¬«      }| j5                  |«      }|�||z  }t        j6                  ||
«      }|j+                  dddd
«      j-                  «       }|j9                  «       d d | j:                  fz   }|j#                  |«      }|r||f}|S |f}|S )NrC   r‘   éþÿÿÿg      Y@)Úmaxr   r   r@   é   r   )rE   rú   r	  rû   rü   r   r–   Ú	normalizerÂ   r(   Úclampré   ró   rç   Úexprì   rß   rF   rã   rá   rJ   rG   rH   Úsigmoidr¥   Úsoftmaxr†   ÚmatmulrŽ   rå   )r:   r!   r
  r  r  rK   r’   rN   Úmixed_query_layerÚ	key_layerÚvalue_layerÚquery_layerÚattention_scoresré   Úrelative_position_bias_tableÚrelative_position_biasÚ
mask_shapeÚattention_probsÚcontext_layerÚnew_context_layer_shapeÚoutputss                        r-   rl   zSwinv2SelfAttention.forward  s.  € ð )6×(;Ñ(;Ñ%ˆ
�C˜Ø ŸJ™J }Ó5Ðà×-Ñ-¨d¯h©h°}Ó.EÓFˆ	Ø×/Ñ/°·
±
¸=Ó0IÓJˆØ×/Ñ/Ð0AÓBˆô Ÿ=™=×2Ñ2°;ÀBÐ2ÓGÌ"Ï-É-×JaÑJaØ˜2ð Kbó K
ç
‰)�B˜Ó
ñÐô —k‘k $×"2Ñ"2¼¿¹ÀÓ8LÔM×QÑQÓSˆØ+¨kÑ9ÐØ'+×'HÑ'HÈ×IcÑIcÓ'd×'iÑ'iØ�×(Ñ(ó(
Ð$ð ">¸d×>ZÑ>Z×>_Ñ>_Ð`bÓ>cÑ!d×!iÑ!iØ×Ñ˜QÑ $×"2Ñ"2°1Ñ"5Ñ5°t×7GÑ7GÈÑ7JÈT×M]ÑM]Ð^_ÑM`Ñ7`Ðbdó"
Ðð "8×!?Ñ!?ÀÀ1ÀaÓ!H×!SÑ!SÓ!UÐØ!#¤e§m¡mÐ4JÓ&KÑ!KÐØ+Ð.D×.NÑ.NÈqÓ.QÑQÐàÐ%à'×-Ñ-¨aÑ0ˆJØ/×4Ñ4Ø˜jÑ(¨*°d×6NÑ6NÐPSÐUXó à×(Ñ(¨Ó+×5Ñ5°aÓ8ñ 9Ðð  0°.×2JÑ2JÈ1Ó2M×2WÑ2WÐXYÓ2ZÑZÐØ/×4Ñ4°R¸×9QÑ9QÐSVÐX[Ó\Ðô Ÿ-™-×/Ñ/Ð0@ÀbÐ/ÓIˆð Ÿ,™, Ó7ˆð Ð Ø-°	Ñ9ˆOäŸ™ _°kÓBˆØ%×-Ñ-¨a°°A°qÓ9×DÑDÓFˆØ"/×"4Ñ"4Ó"6°s¸Ð";¸t×?QÑ?QÐ>SÑ"SÐØ%×*Ñ*Ð+BÓCˆá6G�= /Ð2ˆàˆð O\ÐM]ˆàˆr,   ©NNF)r$   r%   r&   rh   r	  r(   r   r   r)   r¯   r   rl   rr   rs   s   @r-   rÖ   rÖ   Ó  s   ø„ ØTUÐWXÐSYõ ;Gòz%ð 7;Ø15Ø,1ñ;à—|‘|ð;ð ! ×!2Ñ!2Ñ3ð;ð ˜E×-Ñ-Ñ.ð	;ð
 $ D™>ð;ð 
ˆu�|‰|Ñ	÷;r,   rÖ   c                   ón   ‡ — e Zd Zˆ fd„Zdej
                  dej
                  dej
                  fd„Zˆ xZS )ÚSwinv2SelfOutputc                 ó    •— t         ‰| �  «        t        j                  ||«      | _        t        j
                  |j                  «      | _        y rf   )rg   rh   r   rÊ   Údenser„   rý   r†   ©r:   rˆ   r’   ri   s      €r-   rh   zSwinv2SelfOutput.__init__V  s6   ø€ Ü‰ÑÔÜ—Y‘Y˜s CÓ(ˆŒ
Ü—z‘z &×"EÑ"EÓFˆ�r,   r!   Úinput_tensorrV   c                 óJ   — | j                  |«      }| j                  |«      }|S rf   ©r'  r†   )r:   r!   r)  s      r-   rl   zSwinv2SelfOutput.forward[  s$   € ØŸ
™
 =Ó1ˆØŸ™ ]Ó3ˆàÐr,   ©r$   r%   r&   rh   r(   r   rl   rr   rs   s   @r-   r%  r%  U  s2   ø„ ôGð
 U§\¡\ð ÀÇÁð ÐRW×R^ÑR^÷ r,   r%  c                   ó²   ‡ — e Zd Zd	ˆ fd„	Zd„ Z	 	 	 d
dej                  deej                     deej                     dee	   de
ej                     f
d„Zˆ xZS )ÚSwinv2Attentionc           
      óÜ   •— t         ‰| �  «        t        ||||t        |t        j
                  j                  «      r|n||f¬«      | _        t        ||«      | _	        t        «       | _        y )N©rˆ   r’   rþ   rJ   ræ   )rg   rh   rÖ   rµ   r¶   r·   r¸   r:   r%  ra   ÚsetÚpruned_heads)r:   rˆ   r’   rþ   rJ   ræ   ri   s         €r-   rh   zSwinv2Attention.__init__c  sc   ø€ Ü‰ÑÔÜ'ØØØØ#äÐ0´+·/±/×2JÑ2JÔKñ $:à(Ð*@ÐAô
ˆŒ	ô ' v¨sÓ3ˆŒÜ›EˆÕr,   c                 ó>  — t        |«      dk(  ry t        || j                  j                  | j                  j                  | j
                  «      \  }}t        | j                  j                  |«      | j                  _        t        | j                  j                  |«      | j                  _        t        | j                  j                  |«      | j                  _	        t        | j                  j                  |d¬«      | j                  _        | j                  j                  t        |«      z
  | j                  _        | j                  j                  | j                  j                  z  | j                  _        | j
                  j                  |«      | _        y )Nr   r   r‘   )Úlenr   r:   rã   rä   r2  r   rú   rû   rü   ra   r'  rå   Úunion)r:   ÚheadsÚindexs      r-   Úprune_headszSwinv2Attention.prune_headsq  s  € Üˆu‹:˜Š?ØÜ7Ø�4—9‘9×0Ñ0°$·)±)×2OÑ2OÐQU×QbÑQbó
‰ˆˆuô
 -¨T¯Y©Y¯_©_¸eÓDˆ�	‰	ŒÜ*¨4¯9©9¯=©=¸%Ó@ˆ�	‰	ŒÜ,¨T¯Y©Y¯_©_¸eÓDˆ�	‰	ŒÜ.¨t¯{©{×/@Ñ/@À%ÈQÔOˆ�‰Ôð )-¯	©	×(EÑ(EÌÈEË
Ñ(Rˆ�	‰	Ô%Ø"&§)¡)×"?Ñ"?À$Ç)Á)×B_ÑB_Ñ"_ˆ�	‰	ÔØ ×-Ñ-×3Ñ3°EÓ:ˆÕr,   r!   r
  r  r  rV   c                 ój   — | j                  ||||«      }| j                  |d   |«      }|f|dd  z   }|S )Nr   r   )r:   ra   )r:   r!   r
  r  r  Úself_outputsÚattention_outputr"  s           r-   rl   zSwinv2Attention.forwardƒ  sG   € ð —y‘y °À	ÐK\Ó]ˆØŸ;™; |°A¡¸ÓFÐØ#Ð%¨°Q°RÐ(8Ñ8ˆØˆr,   ©r   r#  )r$   r%   r&   rh   r8  r(   r   r   r)   r¯   r   rl   rr   rs   s   @r-   r.  r.  b  st   ø„ õ"ò;ð* 7;Ø15Ø,1ñ
à—|‘|ð
ð ! ×!2Ñ!2Ñ3ð
ð ˜E×-Ñ-Ñ.ð	
ð
 $ D™>ð
ð 
ˆu�|‰|Ñ	÷
r,   r.  c                   óV   ‡ — e Zd Zˆ fd„Zdej
                  dej
                  fd„Zˆ xZS )ÚSwinv2Intermediatec                 ó  •— t         ‰| �  «        t        j                  |t	        |j
                  |z  «      «      | _        t        |j                  t        «      rt        |j                     | _        y |j                  | _        y rf   )rg   rh   r   rÊ   r­   Ú	mlp_ratior'  rµ   Ú
hidden_actrq   r   Úintermediate_act_fnr(  s      €r-   rh   zSwinv2Intermediate.__init__’  sa   ø€ Ü‰ÑÔÜ—Y‘Y˜s¤C¨×(8Ñ(8¸3Ñ(>Ó$?Ó@ˆŒ
Ü�f×'Ñ'¬Ô-Ü'-¨f×.?Ñ.?Ñ'@ˆDÕ$à'-×'8Ñ'8ˆDÕ$r,   r!   rV   c                 óJ   — | j                  |«      }| j                  |«      }|S rf   )r'  rB  rk   s     r-   rl   zSwinv2Intermediate.forwardš  s&   € ØŸ
™
 =Ó1ˆØ×0Ñ0°Ó?ˆØÐr,   r,  rs   s   @r-   r>  r>  ‘  s#   ø„ ô9ð U§\¡\ð °e·l±l÷ r,   r>  c                   óV   ‡ — e Zd Zˆ fd„Zdej
                  dej
                  fd„Zˆ xZS )ÚSwinv2Outputc                 óÌ   •— t         ‰| �  «        t        j                  t	        |j
                  |z  «      |«      | _        t        j                  |j                  «      | _	        y rf   )
rg   rh   r   rÊ   r­   r@  r'  r„   r…   r†   r(  s      €r-   rh   zSwinv2Output.__init__¢  sF   ø€ Ü‰ÑÔÜ—Y‘Yœs 6×#3Ñ#3°cÑ#9Ó:¸CÓ@ˆŒ
Ü—z‘z &×"<Ñ"<Ó=ˆ�r,   r!   rV   c                 óJ   — | j                  |«      }| j                  |«      }|S rf   r+  rk   s     r-   rl   zSwinv2Output.forward§  s$   € ØŸ
™
 =Ó1ˆØŸ™ ]Ó3ˆØÐr,   r,  rs   s   @r-   rE  rE  ¡  s#   ø„ ô>ð
 U§\¡\ð °e·l±l÷ r,   rE  c                   óê   ‡ — e Zd Z	 dˆ fd„	Zdeeeef   eeef   f   fd„Zd„ Zd„ Z	 	 dde	j                  deeef   dee	j                     d	ee   dee	j                  e	j                  f   f
d
„Zˆ xZS )ÚSwinv2Layerc           
      ón  •— t         ‰	| �  «        || _        | j                  |j                  |j                  f||f«      \  }}|d   | _        |d   | _        t        |||| j                  t        |t        j                  j                  «      r|n||f¬«      | _        t        j                  ||j                  ¬«      | _        |dkD  rt!        |«      nt        j"                  «       | _        t'        ||«      | _        t+        ||«      | _        t        j                  ||j                  ¬«      | _        y )Nr   r0  ©ÚepsrX   )rg   rh   rÅ   Ú_compute_window_shiftrJ   Ú
shift_sizer.  rµ   r¶   r·   r¸   Ú	attentionr   r‚   Úlayer_norm_epsÚlayernorm_beforerd   ÚIdentityrb   r>  ÚintermediaterE  ra   Úlayernorm_after)
r:   rˆ   r’   rÅ   rþ   Údrop_path_raterN  ræ   rJ   ri   s
            €r-   rh   zSwinv2Layer.__init__®  s  ø€ ô 	‰ÑÔØ 0ˆÔØ"&×"<Ñ"<Ø×Ñ ×!3Ñ!3Ð4°zÀ:Ð6Nó#
Ñˆ�Zð ' q™>ˆÔØ$ Q™-ˆŒÜ(ØØØØ×(Ñ(äÐ0´+·/±/×2JÑ2JÔKñ $:à(Ð*@ÐAô
ˆŒô !#§¡¨S°f×6KÑ6KÔ LˆÔØ;IÈCÒ;Oœ¨Ô7ÔUW×U`ÑU`ÓUbˆŒÜ.¨v°sÓ;ˆÔÜ" 6¨3Ó/ˆŒÜ!Ÿ|™|¨C°V×5JÑ5JÔKˆÕr,   rV   c                 óè   — t        | j                  |«      D ��cg c]  \  }}||k  r|n|‘Œ }}}t        | j                  ||«      D ���cg c]  \  }}}||k  rdn|‘Œ }}}}||fS c c}}w c c}}}w ©Nr   )ÚziprÅ   )r:   Útarget_window_sizeÚtarget_shift_sizeÚrÚwrJ   ÚsrN  s           r-   rM  z!Swinv2Layer._compute_window_shiftÇ  s~   € Ü69¸$×:OÑ:OÐQcÓ6d×e©d¨a°˜A šF‘q¨Ñ)ÐeˆÑeÜ8;¸D×<QÑ<QÐS^Ð`qÓ8r×sÐs©W¨Q°°1˜1 š6‘a qÑ(Ðsˆ
ÒsØ˜JÐ&Ð&ùó fùÜss   šA'ÁA-c           	      ó  — | j                   dkD  �ryt        j                  d||df|¬«      }t        d| j                   «      t        | j                   | j                    «      t        | j                    d «      f}t        d| j                   «      t        | j                   | j                    «      t        | j                    d «      f}d}|D ]  }|D ]  }	||d d …||	d d …f<   |dz  }Œ Œ t        || j                  «      }
|
j                  d| j                  | j                  z  «      }
|
j                  d«      |
j                  d«      z
  }|j                  |dk7  t        d«      «      j                  |dk(  t        d«      «      }|S d }|S )Nr   r   rÛ   rC   r@   g      YÀrX   )
rN  r(   r}   ÚslicerJ   rP   rF   r¥   Úmasked_fillrp   )r:   rL   rM   rY   Úimg_maskÚheight_slicesÚwidth_slicesÚcountÚheight_sliceÚwidth_sliceÚmask_windowsÚ	attn_masks               r-   Úget_attn_maskzSwinv2Layer.get_attn_maskÌ  s•  € Ø�?‰?˜QÓä—{‘{ A v¨u°aÐ#8ÀÔFˆHä�a˜$×*Ñ*Ð*Ó+Ü�t×'Ñ'Ð'¨$¯/©/Ð)9Ó:Ü�t—‘Ð&¨Ó-ðˆMô �a˜$×*Ñ*Ð*Ó+Ü�t×'Ñ'Ð'¨$¯/©/Ð)9Ó:Ü�t—‘Ð&¨Ó-ðˆLð
 ˆEØ -ò �Ø#/ò �KØ@E�HšQ ¨kº1Ð<Ñ=Ø˜Q‘J‘Eñðô
 ,¨H°d×6FÑ6FÓGˆLØ'×,Ñ,¨R°×1AÑ1AÀD×DTÑDTÑ1TÓUˆLØ$×.Ñ.¨qÓ1°L×4JÑ4JÈ1Ó4MÑMˆIØ!×-Ñ-¨i¸1©n¼eÀF»mÓL×XÑXÐYbÐfgÑYgÔinÐorÓisÓtˆIð Ðð ˆIØÐr,   c                 óþ   — | j                   || j                   z  z
  | j                   z  }| j                   || j                   z  z
  | j                   z  }ddd|d|f}t        j                  j                  ||«      }||fS rW  )rJ   r   r–   r½   )r:   r!   rL   rM   Ú	pad_rightÚ
pad_bottomr¾   s          r-   r¿   zSwinv2Layer.maybe_padè  s�   € Ø×%Ñ%¨°×0@Ñ0@Ñ(@Ñ@ÀD×DTÑDTÑTˆ	Ø×&Ñ&¨°$×2BÑ2BÑ)BÑBÀd×FVÑFVÑVˆ
Ø˜˜A˜y¨!¨ZÐ8ˆ
ÜŸ™×)Ñ)¨-¸ÓDˆØ˜jÐ(Ð(r,   r!   rÎ   r  r  c                 óâ  — |\  }}|j                  «       \  }}}	|}
|j                  ||||	«      }| j                  |||«      \  }}|j                  \  }}}}| j                  dkD  r1t        j                  || j                   | j                   fd¬«      }n|}t        || j                  «      }|j                  d| j                  | j                  z  |	«      }| j                  |||j                  ¬«      }|�|j                  |j                  «      }| j                  ||||¬«      }|d   }|j                  d| j                  | j                  |	«      }t        || j                  ||«      }| j                  dkD  r/t        j                  || j                  | j                  fd¬«      }n|}|d   dkD  xs |d   dkD  }|r|d d …d |…d |…d d …f   j                  «       }|j                  |||z  |	«      }| j!                  |«      }|
| j#                  |«      z   }| j%                  |«      }| j'                  |«      }|| j#                  | j)                  |«      «      z   }|r	||d	   f}|S |f}|S )
Nr   )r   r@   )ÚshiftsÚdimsrC   rÛ   )r  r   rB   r   )rŽ   rF   r¿   rE   rN  r(   ÚrollrP   rJ   ri  rY   rô   rZ   rO  rR   rH   rQ  rb   rS  ra   rT  )r:   r!   rÎ   r  r  rL   rM   rK   r§   ÚchannelsÚshortcutr¾   Ú
height_padÚ	width_padÚshifted_hidden_statesÚhidden_states_windowsrh  Úattention_outputsr;  Úattention_windowsÚshifted_windowsÚ
was_paddedÚlayer_outputÚlayer_outputss                           r-   rl   zSwinv2Layer.forwardï  s¢  € ð )‰ˆ�Ø"/×"4Ñ"4Ó"6Ñˆ
�A�xØ ˆð &×*Ñ*¨:°v¸uÀhÓOˆØ$(§N¡N°=À&È%Ó$PÑ!ˆ�zØ&3×&9Ñ&9Ñ#ˆˆ:�y !à�?‰?˜QÒÜ$)§J¡J¨}ÀtÇÁÐFVÐY]×YhÑYhÐXhÐEiÐpvÔ$wÑ!à$1Ð!ô !1Ð1FÈ×HXÑHXÓ YÐØ 5× :Ñ :¸2¸t×?OÑ?OÐRV×RbÑRbÑ?bÐdlÓ mÐØ×&Ñ& z°9ÀM×DWÑDWÐ&ÓXˆ	ØÐ Ø!Ÿ™Ð%:×%AÑ%AÓBˆIà ŸN™NØ! 9¨iÐK\ð +ó 
Ðð -¨QÑ/Ðà,×1Ñ1°"°d×6FÑ6FÈ×HXÑHXÐZbÓcÐÜ(Ð):¸D×<LÑ<LÈjÐZcÓdˆð �?‰?˜QÒÜ %§
¡
¨?ÀDÇOÁOÐUY×UdÑUdÐCeÐlrÔ sÑà /Ðà ‘] QÑ&Ò;¨*°Q©-¸!Ñ*;ˆ
ÙØ 1²!°W°f°W¸f¸u¸fÂaÐ2GÑ H× SÑ SÓ UÐà-×2Ñ2°:¸vÈ¹~ÈxÓXÐØ×-Ñ-Ð.?Ó@ˆØ  4§>¡>°-Ó#@Ñ@ˆà×(Ñ(¨Ó7ˆØ—{‘{ <Ó0ˆØ$ t§~¡~°d×6JÑ6JÈ<Ó6XÓ'YÑYˆá@Q˜Ð'8¸Ñ';Ð<ˆØÐð YeÐWfˆØÐr,   )rX   r   r   r¬   )r$   r%   r&   rh   r   r­   rM  ri  r¿   r(   r   r   r)   r¯   rl   rr   rs   s   @r-   rI  rI  ­  s°   ø„ àqrõLð2'ÈeÐTYÐZ]Ð_bÐZbÑTcÐejÐknÐpsÐksÑetÐTtÑNuó 'ò
ò8)ð 26Ø,1ñ8à—|‘|ð8ð    S ™/ð8ð ˜E×-Ñ-Ñ.ð	8ð
 $ D™>ð8ð 
ˆu�|‰|˜UŸ\™\Ð)Ñ	*÷8r,   rI  c                   óœ   ‡ — e Zd Z	 dˆ fd„	Z	 	 d	dej
                  deeef   deej                     dee
   deej
                     f
d„Zˆ xZS )
ÚSwinv2Stagec	           
      ó|  •— t         ‰| �  «        || _        || _        g }	t	        |«      D ]?  }
t        ||||||
   |
dz  dk(  rdn|j                  dz  |¬«      }|	j                  |«       ŒA t        j                  |	«      | _
        |�& |||t        j                  ¬«      | _        d| _        y d | _        d| _        y )Nr@   r   )rˆ   r’   rÅ   rþ   rU  rN  ræ   )r’   rÆ   F)rg   rh   rˆ   r’   ÚrangerI  rJ   Úappendr   Ú
ModuleListÚblocksr‚   Ú
downsampleÚpointing)r:   rˆ   r’   rÅ   Údepthrþ   rb   r„  ræ   rƒ  ÚiÚblockri   s               €r-   rh   zSwinv2Stage.__init__+  sÆ   ø€ ô 	‰ÑÔØˆŒØˆŒØˆÜ�u“ò 
	!ˆAÜØØØ!1Ø#Ø(¨™|Ø!" Q¡¨!¢™1°&×2DÑ2DÈÑ2IØ'=ôˆEð �M‰M˜%Õ ð
	!ô —m‘m FÓ+ˆŒð Ð!Ù(Ð)9¸sÌrÏ|É|Ô\ˆDŒOð ˆ�ð #ˆDŒOàˆ�r,   r!   rÎ   r  r  rV   c                 ó  — |\  }}t        | j                  «      D ]  \  }}|�||   nd }	 ||||	|«      }
|
d   }Œ  |}| j                  �)|dz   dz  |dz   dz  }}||||f}| j                  ||«      }n||||f}|||f}|r|
dd  z  }|S )Nr   r   r@   )Ú	enumeraterƒ  r„  )r:   r!   rÎ   r  r  rL   rM   r‡  Úlayer_moduleÚlayer_head_maskr|  Ú!hidden_states_before_downsamplingÚheight_downsampledÚwidth_downsampledr¨   Ústage_outputss                   r-   rl   zSwinv2Stage.forwardG  sæ   € ð )‰ˆ�Ü(¨¯©Ó5ò 
	-‰OˆAˆ|Ø.7Ð.C˜i¨šlÈˆOá(ØØ ØØ!ó	ˆMð *¨!Ñ,‰Mð
	-ð -:Ð)Ø�?‰?Ð&Ø5;¸a±ZÀAÑ4EÈÐPQÉ	ÐVWÑGWÐ 1ÐØ!'¨Ð0BÐDUÐ VÐØ ŸO™OÐ,MÐO_Ó`‰Mà!'¨°¸Ð >Ðà&Ð(IÐK\Ð]ˆáØ˜]¨1¨2Ð.Ñ.ˆMØÐr,   r<  r¬   )r$   r%   r&   rh   r(   r   r   r­   r   r)   r¯   rl   rr   rs   s   @r-   r~  r~  *  sm   ø„ àmnõð@ 26Ø,1ñ à—|‘|ð ð    S ™/ð ð ˜E×-Ñ-Ñ.ð	 ð
 $ D™>ð ð 
ˆu�|‰|Ñ	÷ r,   r~  c                   ó®   ‡ — e Zd Zdˆ fd„	Z	 	 	 	 	 ddej
                  deeef   deej                     dee
   dee
   dee
   dee
   d	eeef   fd
„Zˆ xZS )ÚSwinv2Encoderc                 ó:  •— t         ‰	| �  «        t        |j                  «      | _        || _        | j
                  j                  �|j                  }t        j                  d|j                  t        |j                  «      «      D �cg c]  }|j                  «       ‘Œ }}g }t        | j                  «      D ]º  }t        |t        |j                  d|z  z  «      |d   d|z  z  |d   d|z  z  f|j                  |   |j                   |   |t        |j                  d | «      t        |j                  d |dz    «       || j                  dz
  k  rt"        nd ||   ¬«      }|j%                  |«       Œ¼ t'        j(                  |«      | _        d| _        y c c}w )Nr   r@   r   )rˆ   r’   rÅ   r†  rþ   rb   r„  ræ   F)rg   rh   r4  ÚdepthsÚ
num_layersrˆ   Úpretrained_window_sizesr(   ÚlinspacerU  rø   Úitemr€  r~  r­   r~   rþ   rÄ   r�  r   r‚  ÚlayersÚgradient_checkpointing)
r:   rˆ   rz   r–  r  Údprr™  Úi_layerÚstageri   s
            €r-   rh   zSwinv2Encoder.__init__k  st  ø€ Ü‰ÑÔÜ˜fŸm™mÓ,ˆŒØˆŒØ�;‰;×.Ñ.Ð:Ø&,×&DÑ&DÐ#Ü!&§¡°°6×3HÑ3HÌ#ÈfÏmÉmÓJ\Ó!]Ö^˜Aˆq�v‰v�xÐ^ˆÐ^àˆÜ˜TŸ_™_Ó-ò 	!ˆGÜØÜ˜×(Ñ(¨1¨g©:Ñ5Ó6Ø"+¨A¡,°1°g±:Ñ">À	È!ÁÐQRÐT[ÑQ[Ñ@\Ð!]Ø—m‘m GÑ,Ø ×*Ñ*¨7Ñ3Øœc &§-¡-°°Ð"9Ó:¼SÀÇÁÈ}ÐQXÐ[\ÑQ\ÐA]Ó=^Ð_Ø29¸D¿O¹OÈaÑ<OÒ2OÕ-ÐVZØ'>¸wÑ'Gô	ˆEð �M‰M˜%Õ ð	!ô —m‘m FÓ+ˆŒà&+ˆÕ#ùò# _s   Â	Fr!   rÎ   r  r  Úoutput_hidden_statesÚ(output_hidden_states_before_downsamplingÚreturn_dictrV   c                 óV  — |rdnd }|rdnd }	|rdnd }
|rE|j                   \  }}} |j                  |g|¢|‘­Ž }|j                  dddd«      }||fz  }|	|fz  }	t        | j                  «      D �]  \  }}|�||   nd }| j
                  r+| j                  r| j                  |j                  |||«      }n |||||«      }|d   }|d   }|d   }|d   |d   f}|rP|rN|j                   \  }}} |j                  |g|d   |d   f¢|‘­Ž }|j                  dddd«      }||fz  }|	|fz  }	nI|rG|sE|j                   \  }}} |j                  |g|¢|‘­Ž }|j                  dddd«      }||fz  }|	|fz  }	|s�Œ
|
|dd  z  }
�Œ |st        d„ |||
|	fD «       «      S t        |||
|	¬	«      S )
Nr+   r   r   r   r@   r  rC   c              3   ó$   K  — | ]  }|�|–— Œ
 y ­wrf   r+   )Ú.0Úvs     r-   ú	<genexpr>z(Swinv2Encoder.forward.<locals>.<genexpr>Å  s   è ø€ ò àØ�=ô ñùs   ‚)r    r!   r"   r#   )rE   rF   rG   rŠ  r™  rš  rU   Ú_gradient_checkpointing_funcÚ__call__Útupler   )r:   r!   rÎ   r  r  rž  rŸ  r   Úall_hidden_statesÚall_reshaped_hidden_statesÚall_self_attentionsrK   r§   r»   Úreshaped_hidden_stater‡  r‹  rŒ  r|  r�  r¨   s                        r-   rl   zSwinv2Encoder.forward„  s†  € ñ #7™B¸DÐÙ+?¡RÀTÐ"Ù$5™b¸4ÐáØ)6×)<Ñ)<Ñ&ˆJ˜˜;à$6 M×$6Ñ$6°zÐ$bÐDTÐ$bÐVaÒ$bÐ!Ø$9×$AÑ$AÀ!ÀQÈÈ1Ó$MÐ!Ø -Ð!1Ñ1ÐØ&Ð+@Ð*BÑBÐ&ä(¨¯©Ó5ó (	9‰OˆAˆ|Ø.7Ð.C˜i¨šlÈˆOà×*Ò*¨t¯}ª}Ø $× AÑ AØ ×)Ñ)¨=Ð:JÈOó!‘ñ !-Ø!Ø$Ø#Ø%ó	!�ð *¨!Ñ,ˆMØ0=¸aÑ0@Ð-Ø -¨aÑ 0Ðà 1°"Ñ 5Ð7HÈÑ7LÐMÐá#Ñ(PØ-N×-TÑ-TÑ*�
˜A˜{ð )OÐ(I×(NÑ(NØð)Ø"3°AÑ"6Ð8IÈ!Ñ8LÐ!Mð)ØOZò)Ð%ð )>×(EÑ(EÀaÈÈAÈqÓ(QÐ%Ø!Ð&GÐ%IÑIÐ!Ø*Ð/DÐ.FÑFÑ*Ù%Ñ.VØ-:×-@Ñ-@Ñ*�
˜A˜{à(:¨×(:Ñ(:¸:Ð(fÐHXÐ(fÐZeÒ(fÐ%Ø(=×(EÑ(EÀaÈÈAÈqÓ(QÐ%Ø! mÐ%5Ñ5Ð!Ø*Ð/DÐ.FÑFÐ*ã Ø# }°Q°RÐ'8Ñ8Ò#ðQ(	9ñT Üñ à'Ð):Ð<OÐQkÐlôó ð ô #Ø+Ø+Ø*Ø#=ô	
ð 	
r,   ))r   r   r   r   )NFFFT)r$   r%   r&   rh   r(   r   r   r­   r   r)   r¯   r   r   rl   rr   rs   s   @r-   r’  r’  j  s§   ø„ õ,ð: 26Ø,1Ø/4ØCHØ&*ñL
à—|‘|ðL
ð    S ™/ðL
ð ˜E×-Ñ-Ñ.ð	L
ð
 $ D™>ðL
ð ' t™nðL
ð 3;¸4±.ðL
ð ˜d‘^ðL
ð 
ˆuÐ)Ð)Ñ	*÷L
r,   r’  c                   ó,   — e Zd ZdZeZdZdZdZdgZ	d„ Z
y)ÚSwinv2PreTrainedModelz†
    An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained
    models.
    Úswinv2r    Tr~  c                 óp  — t        |t        j                  t        j                  f«      rm|j                  j
                  j                  d| j                  j                  ¬«       |j                  �%|j                  j
                  j                  «        yyt        |t        j                  «      rJ|j                  j
                  j                  «        |j                  j
                  j                  d«       yt        |t        «      rb|j                  �$|j                  j
                  j                  «        |j                  �%|j                  j
                  j                  «        yyt        |t         «      r9|j"                  j
                  j                  t%        j&                  d«      «       yy)zInitialize the weightsrX   )ÚmeanÚstdNr£   rÙ   )rµ   r   rÊ   r¹   ÚweightÚdataÚnormal_rˆ   Úinitializer_rangerÉ   Úzero_r‚   Úfill_ru   r   r�   rÖ   ré   ró   rç   )r:   Úmodules     r-   Ú_init_weightsz#Swinv2PreTrainedModel._init_weightsß  s-  € ä�fœrŸy™y¬"¯)©)Ð4Ô5ð �M‰M×Ñ×&Ñ&¨C°T·[±[×5RÑ5RÐ&ÔSØ�{‰{Ð&Ø—‘× Ñ ×&Ñ&Õ(ð 'ä˜¤§¡Ô-Ø�K‰K×Ñ×"Ñ"Ô$Ø�M‰M×Ñ×$Ñ$ SÕ)Ü˜Ô 0Ô1Ø× Ñ Ð,Ø×!Ñ!×&Ñ&×,Ñ,Ô.Ø×)Ñ)Ð5Ø×*Ñ*×/Ñ/×5Ñ5Õ7ð 6ä˜Ô 3Ô4Ø×Ñ×#Ñ#×)Ñ)¬$¯(©(°2«,Õ7ð 5r,   N)r$   r%   r&   r'   r   Úconfig_classÚbase_model_prefixÚmain_input_nameÚsupports_gradient_checkpointingÚ_no_split_modulesrº  r+   r,   r-   r®  r®  Ó  s,   „ ñð
  €LØ ÐØ$€OØ&*Ð#Ø&˜Ðó8r,   r®  aI  
    This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class. Use
    it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and
    behavior.

    Parameters:
        config ([`Swinv2Config`]): Model configuration class with all the parameters of the model.
            Initializing with a config file does not load the weights associated with the model, only the
            configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
aÙ  
    Args:
        pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`):
            Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See [`ViTImageProcessor.__call__`]
            for details.
        head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*):
            Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`:

            - 1 indicates the head is **not masked**,
            - 0 indicates the head is **masked**.

        output_attentions (`bool`, *optional*):
            Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
            tensors for more detail.
        output_hidden_states (`bool`, *optional*):
            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
            more detail.
        interpolate_pos_encoding (`bool`, *optional*, default `False`):
            Whether to interpolate the pre-trained position encodings.
        return_dict (`bool`, *optional*):
            Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
z`The bare Swinv2 Model transformer outputting raw hidden-states without any specific head on top.c                   ó  ‡ — e Zd Zdˆ fd„	Zd„ Zd„ Z ee«       ee	e
ede¬«      	 	 	 	 	 	 	 ddeej                     deej                      deej                     d	ee   d
ee   dedee   deee
f   fd„«       «       Zˆ xZS )ÚSwinv2Modelc                 óê  •— t         ‰| �  |«       || _        t        |j                  «      | _        t        |j                  d| j
                  dz
  z  z  «      | _        t        ||¬«      | _
        t        || j                  j                  «      | _        t        j                  | j                  |j                   ¬«      | _        |rt        j$                  d«      nd | _        | j)                  «        y )Nr@   r   )r‰   rK  )rg   rh   rˆ   r4  r”  r•  r­   r~   Únum_featuresru   rŠ   r’  r{   Úencoderr   r‚   rP  Ú	layernormÚAdaptiveAvgPool1dÚpoolerÚ	post_init)r:   rˆ   Úadd_pooling_layerr‰   ri   s       €r-   rh   zSwinv2Model.__init__  s³   ø€ Ü‰Ñ˜Ô ØˆŒÜ˜fŸm™mÓ,ˆŒÜ × 0Ñ 0°1¸¿¹È1Ñ9LÑ3MÑ MÓNˆÔä*¨6À.ÔQˆŒÜ$ V¨T¯_©_×-GÑ-GÓHˆŒäŸ™ d×&7Ñ&7¸V×=RÑ=RÔSˆŒÙ1B”b×*Ñ*¨1Ô-ÈˆŒð 	�‰Õr,   c                 ó.   — | j                   j                  S rf   ©rŠ   rx   r9   s    r-   Úget_input_embeddingsz Swinv2Model.get_input_embeddings+  ó   € Ø�‰×/Ñ/Ð/r,   c                 ó˜   — |j                  «       D ]7  \  }}| j                  j                  |   j                  j	                  |«       Œ9 y)z�
        Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base
        class PreTrainedModel
        N)ÚitemsrÄ  ÚlayerrO  r8  )r:   Úheads_to_prunerÐ  r6  s       r-   Ú_prune_headszSwinv2Model._prune_heads.  sE   € ð
 +×0Ñ0Ó2ò 	C‰LˆE�5Ø�L‰L×Ñ˜uÑ%×/Ñ/×;Ñ;¸EÕBñ	Cr,   Úvision)Ú
checkpointÚoutput_typer»  ÚmodalityÚexpected_outputr    r¡   r  r  rž  rŸ   r   rV   c                 ó   — |�|n| j                   j                  }|�|n| j                   j                  }|�|n| j                   j                  }|€t	        d«      ‚| j                  |t        | j                   j                  «      «      }| j                  |||¬«      \  }}	| j                  ||	||||¬«      }
|
d   }| j                  |«      }d}| j                  �7| j                  |j                  dd«      «      }t        j                  |d«      }|s||f|
dd z   }|S t        |||
j                   |
j"                  |
j$                  ¬«      S )	zË
        bool_masked_pos (`torch.BoolTensor` of shape `(batch_size, num_patches)`, *optional*):
            Boolean masked positions. Indicates which patches are masked (1) and which aren't (0).
        Nz You have to specify pixel_values)r¡   rŸ   )r  r  rž  r   r   r   r@   )r    r0   r!   r"   r#   )rˆ   r  rž  Úuse_return_dictrâ   Úget_head_maskr4  r”  rŠ   rÄ  rÅ  rÇ  rÂ   r(   rÁ   r/   r!   r"   r#   )r:   r    r¡   r  r  rž  rŸ   r   Úembedding_outputrÎ   Úencoder_outputsÚsequence_outputÚpooled_outputra   s                 r-   rl   zSwinv2Model.forward6  sp  € ð, 2CÐ1NÑ-ÐTX×T_ÑT_×TqÑTqÐà$8Ð$DÑ È$Ï+É+×JjÑJjð 	ð &1Ð%<‘kÀ$Ç+Á+×B]ÑB]ˆàÐÜÐ?Ó@Ð@ð ×&Ñ& y´#°d·k±k×6HÑ6HÓ2IÓJˆ	à-1¯_©_Ø¨/ÐTlð .=ó .
Ñ*ÐÐ*ð Ÿ,™,ØØØØ/Ø!5Ø#ð 'ó 
ˆð *¨!Ñ,ˆØŸ.™.¨Ó9ˆàˆØ�;‰;Ð"Ø ŸK™K¨×(AÑ(AÀ!ÀQÓ(GÓHˆMÜ!ŸM™M¨-¸Ó;ˆMáØ% }Ð5¸ÈÈÐ8KÑKˆFàˆMä Ø-Ø'Ø)×7Ñ7Ø&×1Ñ1Ø#2×#IÑ#Iô
ð 	
r,   )TF©NNNNNFN)r$   r%   r&   rh   rÌ  rÒ  r   ÚSWINV2_INPUTS_DOCSTRINGr   Ú_CHECKPOINT_FOR_DOCr/   Ú_CONFIG_FOR_DOCÚ_EXPECTED_OUTPUT_SHAPEr   r(   r)   r®   r¯   r   r   rl   rr   rs   s   @r-   rÁ  rÁ    sç   ø„ õò0òCñ +Ð+BÓCÙØ&Ø%Ø$ØØ.ôð 59Ø6:Ø15Ø,0Ø/3Ø).Ø&*ñ>
à˜u×0Ñ0Ñ1ð>
ð " %×"2Ñ"2Ñ3ð>
ð ˜E×-Ñ-Ñ.ð	>
ð
 $ D™>ð>
ð ' t™nð>
ð #'ð>
ð ˜d‘^ð>
ð 
ˆuÐ'Ð'Ñ	(ò>
óó Dô>
r,   rÁ  aY  Swinv2 Model with a decoder on top for masked image modeling, as proposed in
[SimMIM](https://arxiv.org/abs/2111.09886).

    <Tip>

    Note that we provide a script to pre-train this model on custom data in our [examples
    directory](https://github.com/huggingface/transformers/tree/main/examples/pytorch/image-pretraining).

    </Tip>
    c                   óð   ‡ — e Zd Zˆ fd„Z ee«       eee¬«      	 	 	 	 	 	 	 dde	e
j                     de	e
j                     de	e
j                     de	e   de	e   ded	e	e   d
eeef   fd„«       «       Zˆ xZS )ÚSwinv2ForMaskedImageModelingc                 óŠ  •— t         ‰| �  |«       t        |dd¬«      | _        t	        |j
                  d|j                  dz
  z  z  «      }t        j                  t        j                  ||j                  dz  |j                  z  d¬«      t        j                  |j                  «      «      | _        | j                  «        y )NFT)rÉ  r‰   r@   r   )Úin_channelsÚout_channelsr²   )rg   rh   rÁ  r¯  r­   r~   r•  r   rê   r¹   Úencoder_striderN   ÚPixelShuffleÚdecoderrÈ  )r:   rˆ   rÃ  ri   s      €r-   rh   z%Swinv2ForMaskedImageModeling.__init__Ž  s¡   ø€ Ü‰Ñ˜Ô ä! &¸EÐRVÔWˆŒä˜6×+Ñ+¨a°F×4EÑ4EÈÑ4IÑ.JÑJÓKˆÜ—}‘}Ü�I‰IØ(°v×7LÑ7LÈaÑ7OÐRX×ReÑReÑ7eÐstôô �O‰O˜F×1Ñ1Ó2ó	
ˆŒð 	�‰Õr,   ©rÕ  r»  r    r¡   r  r  rž  rŸ   r   rV   c           	      óÐ  — |�|n| j                   j                  }| j                  |||||||¬«      }|d   }	|	j                  dd«      }	|	j                  \  }
}}t        j                  |dz  «      x}}|	j                  |
|||«      }	| j                  |	«      }d}|��| j                   j                  | j                   j                  z  }|j                  d||«      }|j                  | j                   j                  d«      j                  | j                   j                  d«      j                  d«      j                  «       }t        j                  j!                  ||d¬	«      }||z  j#                  «       |j#                  «       d
z   z  | j                   j$                  z  }|s|f|dd z   }|�|f|z   S |S t'        |||j(                  |j*                  |j,                  ¬«      S )aQ  
        bool_masked_pos (`torch.BoolTensor` of shape `(batch_size, num_patches)`):
            Boolean masked positions. Indicates which patches are masked (1) and which aren't (0).

        Returns:

        Examples:
        ```python
        >>> from transformers import AutoImageProcessor, Swinv2ForMaskedImageModeling
        >>> import torch
        >>> from PIL import Image
        >>> import requests

        >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
        >>> image = Image.open(requests.get(url, stream=True).raw)

        >>> image_processor = AutoImageProcessor.from_pretrained("microsoft/swinv2-tiny-patch4-window8-256")
        >>> model = Swinv2ForMaskedImageModeling.from_pretrained("microsoft/swinv2-tiny-patch4-window8-256")

        >>> num_patches = (model.config.image_size // model.config.patch_size) ** 2
        >>> pixel_values = image_processor(images=image, return_tensors="pt").pixel_values
        >>> # create random boolean mask of shape (batch_size, num_patches)
        >>> bool_masked_pos = torch.randint(low=0, high=2, size=(1, num_patches)).bool()

        >>> outputs = model(pixel_values, bool_masked_pos=bool_masked_pos)
        >>> loss, reconstructed_pixel_values = outputs.loss, outputs.reconstruction
        >>> list(reconstructed_pixel_values.shape)
        [1, 3, 256, 256]
        ```N)r¡   r  r  rž  rŸ   r   r   r   r@   rŒ   rC   Únone)rË   gñhãˆµøä>)r3   r4   r!   r"   r#   )rˆ   rÙ  r¯  rÂ   rE   ró   Úfloorr•   rë  r´   r‡   Úrepeat_interleaver¥   rH   r   r–   Úl1_lossrø   rN   r2   r!   r"   r#   )r:   r    r¡   r  r  rž  rŸ   r   r"  rÝ  rK   rN   Úsequence_lengthrL   rM   Úreconstructed_pixel_valuesÚmasked_im_lossrŽ   r«   Úreconstruction_lossra   s                        r-   rl   z$Swinv2ForMaskedImageModeling.forwardž  sè  € ðR &1Ð%<‘kÀ$Ç+Á+×B]ÑB]ˆà—+‘+ØØ+ØØ/Ø!5Ø%=Ø#ð ó 
ˆð " !™*ˆà)×3Ñ3°A°qÓ9ˆØ4C×4IÑ4IÑ1ˆ
�L /ÜŸ™ O°SÑ$8Ó9Ð9ˆ�Ø)×1Ñ1°*¸lÈFÐTYÓZˆð &*§\¡\°/Ó%BÐ"àˆØÑ&Ø—;‘;×)Ñ)¨T¯[©[×-CÑ-CÑCˆDØ-×5Ñ5°b¸$ÀÓEˆOà×1Ñ1°$·+±+×2HÑ2HÈ!ÓLß"Ñ" 4§;¡;×#9Ñ#9¸1Ó=ß‘˜1“ß‘“ð	 ô #%§-¡-×"7Ñ"7¸ÐF`ÐlrÐ"7Ó"sÐØ1°DÑ8×=Ñ=Ó?À4Ç8Á8Ã:ÐPTÑCTÑUÐX\×XcÑXc×XpÑXpÑpˆNáØ0Ð2°W¸Q¸R°[Ñ@ˆFØ3AÐ3M�^Ð%¨Ñ.ÐYÐSYÐYä.ØØ5Ø!×/Ñ/Ø×)Ñ)Ø#*×#AÑ#Aô
ð 	
r,   rß  )r$   r%   r&   rh   r   rà  r   r2   râ  r   r(   r)   r®   r¯   r   r   rl   rr   rs   s   @r-   rå  rå    sÜ   ø„ ôñ  +Ð+BÓCÙÐ+JÐYhÔið 59Ø6:Ø15Ø,0Ø/3Ø).Ø&*ñT
à˜u×0Ñ0Ñ1ðT
ð " %×"2Ñ"2Ñ3ðT
ð ˜E×-Ñ-Ñ.ð	T
ð
 $ D™>ðT
ð ' t™nðT
ð #'ðT
ð ˜d‘^ðT
ð 
ˆuÐ5Ð5Ñ	6òT
ó jó DôT
r,   rå  aæ  
    Swinv2 Model transformer with an image classification head on top (a linear layer on top of the final hidden state
    of the [CLS] token) e.g. for ImageNet.

    <Tip>

        Note that it's possible to fine-tune SwinV2 on higher resolution images than the ones it has been trained on, by
        setting `interpolate_pos_encoding` to `True` in the forward of the model. This will interpolate the pre-trained
        position embeddings to the higher resolution.

    </Tip>
    c                   óô   ‡ — e Zd Zˆ fd„Z ee«       eeee	e
¬«      	 	 	 	 	 	 	 ddeej                     deej                     deej                     dee   dee   ded	ee   d
eeef   fd„«       «       Zˆ xZS )ÚSwinv2ForImageClassificationc                 ó>  •— t         ‰| �  |«       |j                  | _        t        |«      | _        |j                  dkD  r4t        j                  | j                  j                  |j                  «      nt        j                  «       | _	        | j                  «        y rW  )rg   rh   Ú
num_labelsrÁ  r¯  r   rÊ   rÃ  rR  Ú
classifierrÈ  )r:   rˆ   ri   s     €r-   rh   z%Swinv2ForImageClassification.__init__  sx   ø€ Ü‰Ñ˜Ô à ×+Ñ+ˆŒÜ! &Ó)ˆŒð GM×FWÑFWÐZ[ÒF[ŒB�I‰I�d—k‘k×.Ñ.°×0AÑ0AÔBÔac×alÑalÓanð 	Œð
 	�‰Õr,   )rÔ  rÕ  r»  r×  r    r  Úlabelsr  rž  rŸ   r   rV   c                 ó.  — |�|n| j                   j                  }| j                  ||||||¬«      }|d   }	| j                  |	«      }
d}|��‡| j                   j                  €�| j
                  dk(  rd| j                   _        nl| j
                  dkD  rL|j                  t        j                  k(  s|j                  t        j                  k(  rd| j                   _        nd| j                   _        | j                   j                  dk(  rIt        «       }| j
                  dk(  r& ||
j                  «       |j                  «       «      }nŒ ||
|«      }n‚| j                   j                  dk(  r=t        «       } ||
j                  d| j
                  «      |j                  d«      «      }n,| j                   j                  dk(  rt        «       } ||
|«      }|s|
f|dd z   }|�|f|z   S |S t        ||
|j                   |j"                  |j$                  ¬	«      S )
aŠ  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the image classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        N)r  r  rž  rŸ   r   r   Ú
regressionÚsingle_label_classificationÚmulti_label_classificationrC   r@   )r3   r;   r!   r"   r#   )rˆ   rÙ  r¯  rú  Úproblem_typerù  rY   r(   Úlongr­   r   Úsqueezer
   rF   r	   r>   r!   r"   r#   )r:   r    r  rû  r  rž  rŸ   r   r"  rÞ  r;   r3   Úloss_fctra   s                 r-   rl   z$Swinv2ForImageClassification.forward  sÚ  € ð. &1Ð%<‘kÀ$Ç+Á+×B]ÑB]ˆà—+‘+ØØØ/Ø!5Ø%=Ø#ð ó 
ˆð   ™
ˆà—‘ Ó/ˆàˆØÑØ�{‰{×'Ñ'Ð/Ø—?‘? aÒ'Ø/;�D—K‘KÕ,Ø—_‘_ qÒ(¨f¯l©l¼e¿j¹jÒ.HÈFÏLÉLÔ\a×\eÑ\eÒLeØ/L�D—K‘KÕ,à/K�D—K‘KÔ,à�{‰{×'Ñ'¨<Ò7Ü"›9�Ø—?‘? aÒ'Ù# F§N¡NÓ$4°f·n±nÓ6FÓG‘Dá# F¨FÓ3‘DØ—‘×)Ñ)Ð-JÒJÜ+Ó-�Ù §¡¨B°·±Ó @À&Ç+Á+ÈbÃ/ÓR‘Ø—‘×)Ñ)Ð-IÒIÜ,Ó.�Ù ¨Ó/�áØ�Y ¨¨ Ñ,ˆFØ)-Ð)9�T�G˜fÑ$ÐE¸vÐEä*ØØØ!×/Ñ/Ø×)Ñ)Ø#*×#AÑ#Aô
ð 	
r,   rß  )r$   r%   r&   rh   r   rà  r   Ú_IMAGE_CLASS_CHECKPOINTr>   râ  Ú_IMAGE_CLASS_EXPECTED_OUTPUTr   r(   r)   Ú
LongTensorr¯   r   r   rl   rr   rs   s   @r-   r÷  r÷  ÷  sã   ø„ ô"ñ +Ð+BÓCÙØ*Ø/Ø$Ø4ô	ð 59Ø15Ø-1Ø,0Ø/3Ø).Ø&*ñ@
à˜u×0Ñ0Ñ1ð@
ð ˜E×-Ñ-Ñ.ð@
ð ˜×)Ñ)Ñ*ð	@
ð
 $ D™>ð@
ð ' t™nð@
ð #'ð@
ð ˜d‘^ð@
ð 
ˆuÐ1Ð1Ñ	2ò@
óó Dô@
r,   r÷  zO
    Swinv2 backbone, to be used with frameworks like DETR and MaskFormer.
    c                   óŠ   ‡ — e Zd Zˆ fd„Zd„ Z ee«       eee	¬«      	 	 	 d
de
dee   dee   dee   def
d	„«       «       Zˆ xZS )ÚSwinv2Backbonec           	      ó†  •— t         ‰| �  |«       t         ‰| �	  |«       |j                  gt	        t        |j                  «      «      D �cg c]  }t        |j                  d|z  z  «      ‘Œ c}z   | _        t        |«      | _
        t        || j                  j                  «      | _        | j                  «        y c c}w )Nr@   )rg   rh   Ú_init_backboner~   r€  r4  r”  r­   rÃ  ru   rŠ   r’  r{   rÄ  rÈ  )r:   rˆ   r‡  ri   s      €r-   rh   zSwinv2Backbone.__init__g  sœ   ø€ Ü‰Ñ˜Ô Ü‰Ñ˜vÔ&à#×-Ñ-Ð.ÔX]Ô^aÐbh×boÑboÓ^pÓXqÖ1rÐST´#°f×6FÑ6FÈÈAÉÑ6MÕ2NÒ1rÑrˆÔÜ*¨6Ó2ˆŒÜ$ V¨T¯_©_×-GÑ-GÓHˆŒð 	�‰Õùò 2ss   Á"B>c                 ó.   — | j                   j                  S rf   rË  r9   s    r-   rÌ  z#Swinv2Backbone.get_input_embeddingsr  rÍ  r,   rì  r    r  rž  r   rV   c           	      ó  — |�|n| j                   j                  }|�|n| j                   j                  }|�|n| j                   j                  }| j	                  |«      \  }}| j                  ||d|dd|¬«      }|r|j                  n|d   }d}	t        | j                  |«      D ]  \  }
}|
| j                  v sŒ|	|fz  }	Œ |s|	f}|r	||d   fz  }|r	||d   fz  }|S t        |	|r|j                  nd|j                  ¬«      S )	a]  
        Returns:

        Examples:

        ```python
        >>> from transformers import AutoImageProcessor, AutoBackbone
        >>> import torch
        >>> from PIL import Image
        >>> import requests

        >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
        >>> image = Image.open(requests.get(url, stream=True).raw)

        >>> processor = AutoImageProcessor.from_pretrained("microsoft/swinv2-tiny-patch4-window8-256")
        >>> model = AutoBackbone.from_pretrained(
        ...     "microsoft/swinv2-tiny-patch4-window8-256", out_features=["stage1", "stage2", "stage3", "stage4"]
        ... )

        >>> inputs = processor(image, return_tensors="pt")

        >>> outputs = model(**inputs)
        >>> feature_maps = outputs.feature_maps
        >>> list(feature_maps[-1].shape)
        [1, 2048, 7, 7]
        ```NT)r  r  rž  rŸ  r   rC   r+   r   r@   )Úfeature_mapsr!   r"   )rˆ   rÙ  rž  r  rŠ   rÄ  r#   rX  Ústage_namesÚout_featuresr   r!   r"   )r:   r    r  rž  r   rÛ  rÎ   r"  r!   r  r�  Úhidden_statera   s                r-   rl   zSwinv2Backbone.forwardu  sD  € ðF &1Ð%<‘kÀ$Ç+Á+×B]ÑB]ˆà$8Ð$DÑ È$Ï+É+×JjÑJjð 	ð 2CÐ1NÑ-ÐTX×T_ÑT_×TqÑTqÐà-1¯_©_¸\Ó-JÑ*ÐÐ*à—,‘,ØØØØ/Ø!%Ø59Ø#ð ó 
ˆñ ;F˜×6Ò6È7ÐSUÉ;ˆàˆÜ#& t×'7Ñ'7¸Ó#Gò 	0ÑˆE�<Ø˜×)Ñ)Ò)Ø  Ñ/‘ð	0ñ Ø"�_ˆFÙ#Ø˜7 1™:˜-Ñ'�Ù Ø˜7 1™:˜-Ñ'�ØˆMäØ%Ù3G˜'×/Ò/ÈTØ×)Ñ)ô
ð 	
r,   )NNN)r$   r%   r&   rh   rÌ  r   rà  r   r   râ  r   r   r¯   rl   rr   rs   s   @r-   r  r  `  s†   ø„ ô	ò0ñ +Ð+BÓCÙ¨>ÈÔXð -1Ø/3Ø&*ñF
àðF
ð $ D™>ðF
ð ' t™nð	F
ð
 ˜d‘^ðF
ð 
òF
ó Yó DôF
r,   r  )r÷  rå  rÁ  r®  r  )rX   F)Or'   Úcollections.abcr¶   ró   r6   Údataclassesr   Útypingr   r   r   r(   Útorch.utils.checkpointr   r   Útorch.nnr	   r
   r   Úactivationsr   Úmodeling_outputsr   Úmodeling_utilsr   Úpytorch_utilsr   r   r   Úutilsr   r   r   r   r   r   r   Úutils.backbone_utilsr   Úconfiguration_swinv2r   Ú
get_loggerr$   Úloggerrâ  rá  rã  r  r  r   r/   r2   r>   rP   rR   rp   r¯   rb   rÔ   rd   ru   rw   rÄ   rÖ   r%  r.  r>  rE  rI  r~  r’  r®  ÚSWINV2_START_DOCSTRINGrà  rÁ  rå  r÷  r  Ú__all__r+   r,   r-   ú<module>r!     sç  ðñ (ã Û Û Ý !ß )Ñ )ã Û ß ß AÑ Aå !Ý .Ý -ß [Ñ [÷÷ ñ õ 2Ý .ð 
ˆ×	Ñ	˜HÓ	%€ð !€ð AÐ Ú%Ð ð EÐ Ø-Ð ð ôK˜+ó Kó ðKð@ ô K˜ó  Kó ð KðF ô)# kó )#ó ð)#ðX ô K +ó  Kó ð KòH	òñ�U—\‘\ð ¨eð ÀTð ÐV[×VbÑVbó ô*-�R—Y‘Yô -ôY-�r—y‘yô Y-ôz(-˜BŸI™Iô (-ôV3˜Ÿ™ô 3ôl~˜"Ÿ)™)ô ~ôD
�r—y‘yô 
ô+�b—i‘iô +ô^˜Ÿ™ô ô 	�2—9‘9ô 	ôz�"—)‘)ô zôz=�"—)‘)ô =ô@f
�B—I‘Iô f
ôR8˜Oô 8ð@	Ð ðÐ ñ0 ØfØóô
a
Ð'ó a
óð
a
ñH ð	ð óôg
Ð#8ó g
óðg
ñT ðð óô V
Ð#8ó V
ó!ð V
ñr ðð ó	ôW
Ð*¨Mó W
óðW
òt�r,   