Ë
    S^(h›< ã                   ó”  — d Z ddlZddlmZ ddlmZmZmZmZm	Z	 ddl
Z
ddlmZ ddlZ
ddlmZ ddlmZmZmZmZmZ ddlmZ dd	lmZmZmZ dd
lmZmZmZm Z m!Z! ddl"m#Z#m$Z$m%Z%  ejL                  e'«      Z(dZ)dZ*dZ+dZ,dZ-dZ.de
j^                  de
j^                  fd„Z0de
j^                  de
j^                  fd„Z1e G d„ de«      «       Z2 G d„ dejf                  «      Z4 G d„ dejf                  «      Z5 G d„ dejf                  «      Z6d e5iZ7 G d!„ d"ejf                  «      Z8 G d#„ d$ejf                  «      Z9 G d%„ d&ejf                  «      Z: G d'„ d(ejf                  «      Z; G d)„ d*ejf                  «      Z< G d+„ d,ejf                  «      Z= G d-„ d.ejf                  «      Z> G d/„ d0ejf                  «      Z? G d1„ d2ejf                  «      Z@ G d3„ d4ejf                  «      ZA G d5„ d6ejf                  «      ZB G d7„ d8e«      ZC G d9„ d:ejf                  «      ZD G d;„ d<eC«      ZE G d=„ d>eC«      ZF G d?„ d@eC«      ZG G dA„ dBeC«      ZHdEdC„ZIg dD¢ZJy)FzPyTorch AltCLIP model.é    N)Ú	dataclass)ÚAnyÚListÚOptionalÚTupleÚUnioné   )ÚACT2FN)ÚBaseModelOutputÚ)BaseModelOutputWithPastAndCrossAttentionsÚBaseModelOutputWithPoolingÚ,BaseModelOutputWithPoolingAndCrossAttentionsÚ'BaseModelOutputWithPoolingAndProjection)ÚPreTrainedModel)Úapply_chunking_to_forwardÚ find_pruneable_heads_and_indicesÚprune_linear_layer)ÚModelOutputÚ%add_start_docstrings_to_model_forwardÚloggingÚreplace_return_docstringsÚ	torch_inté   )ÚAltCLIPConfigÚAltCLIPTextConfigÚAltCLIPVisionConfigzBAAI/AltCLIPr   a=  
    This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
    library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
    etc.)

    This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
    Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
    and behavior.

    Parameters:
        config ([`CLIPConfig`]): Model configuration class with all the parameters of the model.
            Initializing with a config file does not load the weights associated with the model, only the
            configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
aƒ  
    Args:
        input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
            Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide
            it.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are input IDs?](../glossary#input-ids)
        attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
            Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:

            - 1 for tokens that are **not masked**,
            - 0 for tokens that are **masked**.

            [What are attention masks?](../glossary#attention-mask)
        position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
            config.max_position_embeddings - 1]`.

            [What are position IDs?](../glossary#position-ids)
        output_attentions (`bool`, *optional*):
            Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
            tensors for more detail.
        output_hidden_states (`bool`, *optional*):
            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
            more detail.
        return_dict (`bool`, *optional*):
            Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
aÕ  
    Args:
        pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`):
            Pixel values. Padding will be ignored by default should you provide it. Pixel values can be obtained using
            [`AutoImageProcessor`]. See [`CLIPImageProcessor.__call__`] for details.
        output_attentions (`bool`, *optional*):
            Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
            tensors for more detail.
        output_hidden_states (`bool`, *optional*):
            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
            more detail.
        interpolate_pos_encoding (`bool`, *optional*, defaults `False`):
            Whether to interpolate the pre-trained position encodings.
        return_dict (`bool`, *optional*):
            Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
a¥  
    Args:
        input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
            Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide
            it.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are input IDs?](../glossary#input-ids)
        attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
            Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:

            - 1 for tokens that are **not masked**,
            - 0 for tokens that are **masked**.

            [What are attention masks?](../glossary#attention-mask)
        position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
            config.max_position_embeddings - 1]`.

            [What are position IDs?](../glossary#position-ids)
        pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`):
            Pixel values. Padding will be ignored by default should you provide it. Pixel values can be obtained using
            [`AutoImageProcessor`]. See [`CLIPImageProcessor.__call__`] for details.
        return_loss (`bool`, *optional*):
            Whether or not to return the contrastive loss.
        output_attentions (`bool`, *optional*):
            Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
            tensors for more detail.
        output_hidden_states (`bool`, *optional*):
            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
            more detail.
        interpolate_pos_encoding (`bool`, *optional*, defaults `False`):
            Whether to interpolate the pre-trained position encodings.
        return_dict (`bool`, *optional*):
            Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
ÚlogitsÚreturnc                 ó’   — t         j                  j                  | t        j                  t        | «      | j                  ¬«      «      S )N©Údevice)ÚnnÚ
functionalÚcross_entropyÚtorchÚarangeÚlenr!   )r   s    új/var/www/skyplay_api_hub/venv/lib/python3.12/site-packages/transformers/models/altclip/modeling_altclip.pyÚcontrastive_lossr)   —   s/   € Ü�=‰=×&Ñ& v¬u¯|©|¼CÀ»KÐPV×P]ÑP]Ô/^Ó_Ð_ó    Ú
similarityc                 óZ   — t        | «      }t        | j                  «       «      }||z   dz  S )Ng       @)r)   Út)r+   Úcaption_lossÚ
image_losss      r(   Ú	clip_lossr0   ›   s,   € Ü# JÓ/€LÜ! *§,¡,£.Ó1€JØ˜:Ñ%¨Ñ,Ð,r*   c                   ó  — e Zd ZU dZdZeej                     ed<   dZ	eej                     ed<   dZ
eej                     ed<   dZeej                     ed<   dZeej                     ed<   dZeed<   dZeed	<   d
ee   fd„Zy)ÚAltCLIPOutputaÿ  
    Args:
        loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `return_loss` is `True`):
            Contrastive loss for image-text similarity.
        logits_per_image (`torch.FloatTensor` of shape `(image_batch_size, text_batch_size)`):
            The scaled dot product scores between `image_embeds` and `text_embeds`. This represents the image-text
            similarity scores.
        logits_per_text (`torch.FloatTensor` of shape `(text_batch_size, image_batch_size)`):
            The scaled dot product scores between `text_embeds` and `image_embeds`. This represents the text-image
            similarity scores.
        text_embeds (`torch.FloatTensor` of shape `(batch_size, output_dim`):
            The text embeddings obtained by applying the projection layer to the pooled output of [`AltCLIPTextModel`].
        image_embeds (`torch.FloatTensor` of shape `(batch_size, output_dim`):
            The image embeddings obtained by applying the projection layer to the pooled output of [`AltCLIPVisionModel`].
        text_model_output (`BaseModelOutputWithPooling`):
            The output of the [`AltCLIPTextModel`].
        vision_model_output (`BaseModelOutputWithPooling`):
            The output of the [`AltCLIPVisionModel`].
    NÚlossÚlogits_per_imageÚlogits_per_textÚtext_embedsÚimage_embedsÚtext_model_outputÚvision_model_outputr   c                 óH   ‡ — t        ˆ fd„‰ j                  «       D «       «      S )Nc              3   ód   •K  — | ]'  }|d vr‰|   nt        ‰|«      j                  «       –— Œ) y­w))r8   r9   N)ÚgetattrÚto_tuple)Ú.0ÚkÚselfs     €r(   ú	<genexpr>z)AltCLIPOutput.to_tuple.<locals>.<genexpr>Á   s=   øè ø€ ò 
àð Ð LÑLˆD�ŠGÔRYÐZ^Ð`aÓRb×RkÑRkÓRmÓmñ
ùs   ƒ-0)ÚtupleÚkeys©r@   s   `r(   r=   zAltCLIPOutput.to_tupleÀ   s#   ø€ Üó 
à—Y‘Y“[ô
ó 
ð 	
r*   )Ú__name__Ú
__module__Ú__qualname__Ú__doc__r3   r   r%   ÚFloatTensorÚ__annotations__r4   r5   r6   r7   r8   r   r9   r   r   r=   © r*   r(   r2   r2   ¡   s›   … ñð( )-€Dˆ(�5×$Ñ$Ñ
%Ó,Ø48Ð�h˜u×0Ñ0Ñ1Ó8Ø37€O�X˜e×/Ñ/Ñ0Ó7Ø/3€K�˜%×+Ñ+Ñ,Ó3Ø04€L�(˜5×,Ñ,Ñ-Ó4Ø48ÐÐ1Ó8Ø6:ÐÐ3Ó:ð
˜% ™*ô 
r*   r2   c                   ó2   ‡ — e Zd ZdZˆ fd„Z	 dd„Zd„ Zˆ xZS )ÚAltRobertaEmbeddingszV
    Same as BertEmbeddings with a tiny tweak for positional embeddings indexing.
    c                 óÖ  •— t         ‰| �  «        t        j                  |j                  |j
                  |j                  ¬«      | _        t        j                  |j                  |j
                  «      | _	        t        j                  |j                  |j
                  «      | _        t        j                  |j
                  |j                  ¬«      | _        t        j                  |j                  «      | _        t#        |dd«      | _        | j'                  dt)        j*                  |j                  «      j-                  d«      d¬«       | j'                  d	t)        j.                  | j0                  j3                  «       t(        j4                  ¬
«      d¬«       |j                  | _        t        j                  |j                  |j
                  | j6                  ¬«      | _	        y )N)Úpadding_idx©ÚepsÚposition_embedding_typeÚabsoluteÚposition_ids©r   éÿÿÿÿF©Ú
persistentÚtoken_type_ids©Údtype)ÚsuperÚ__init__r"   Ú	EmbeddingÚ
vocab_sizeÚhidden_sizeÚpad_token_idÚword_embeddingsÚmax_position_embeddingsÚposition_embeddingsÚtype_vocab_sizeÚtoken_type_embeddingsÚ	LayerNormÚlayer_norm_epsÚDropoutÚhidden_dropout_probÚdropoutr<   rR   Úregister_bufferr%   r&   ÚexpandÚzerosrT   ÚsizeÚlongrO   ©r@   ÚconfigÚ	__class__s     €r(   r]   zAltRobertaEmbeddings.__init__Î   si  ø€ Ü‰ÑÔÜ!Ÿ|™|¨F×,=Ñ,=¸v×?QÑ?QÐ_e×_rÑ_rÔsˆÔÜ#%§<¡<°×0NÑ0NÐPV×PbÑPbÓ#cˆÔ Ü%'§\¡\°&×2HÑ2HÈ&×J\ÑJ\Ó%]ˆÔ"ô Ÿ™ f×&8Ñ&8¸f×>SÑ>SÔTˆŒÜ—z‘z &×"<Ñ"<Ó=ˆŒä'.¨vÐ7PÐR\Ó']ˆÔ$Ø×ÑØœEŸL™L¨×)GÑ)GÓH×OÑOÐPWÓXÐejð 	ô 	
ð 	×ÑØœeŸk™k¨$×*;Ñ*;×*@Ñ*@Ó*BÌ%Ï*É*ÔUÐbgð 	ô 	
ð
 "×.Ñ.ˆÔÜ#%§<¡<Ø×*Ñ*¨F×,>Ñ,>ÈD×L\ÑL\ô$
ˆÕ r*   c                 ó€  — |€+|�t        || j                  |«      }n| j                  |«      }|�|j                  «       }n|j                  «       d d }|d   }|€st	        | d«      r-| j
                  d d …d |…f   }|j                  |d   |«      }	|	}n:t        j                  |t        j                  | j                  j                  ¬«      }|€| j                  |«      }| j                  |«      }
||
z   }| j                  dk(  r| j                  |«      }||z  }| j!                  |«      }| j#                  |«      }|S )NrV   r   rY   r   ©r[   r!   rS   )Ú"create_position_ids_from_input_idsrO   Ú&create_position_ids_from_inputs_embedsro   ÚhasattrrY   rm   r%   rn   rp   rT   r!   rb   rf   rR   rd   rg   rk   )r@   Ú	input_idsrY   rT   Úinputs_embedsÚpast_key_values_lengthÚinput_shapeÚ
seq_lengthÚbuffered_token_type_idsÚ buffered_token_type_ids_expandedrf   Ú
embeddingsrd   s                r(   ÚforwardzAltRobertaEmbeddings.forwardç   sR  € ð ÐØÐ$äAÀ)ÈT×M]ÑM]Ð_uÓv‘à#×JÑJÈ=ÓY�àÐ Ø#Ÿ.™.Ó*‰Kà'×,Ñ,Ó.¨s°Ð3ˆKà  ‘^ˆ
ð
 Ð!Ü�tÐ-Ô.Ø*.×*=Ñ*=ºaÀÀ*À¸nÑ*MÐ'Ø3J×3QÑ3QÐR]Ð^_ÑR`ÐblÓ3mÐ0Ø!A‘ä!&§¡¨[ÄÇ
Á
ÐSW×SdÑSd×SkÑSkÔ!l�àÐ Ø ×0Ñ0°Ó;ˆMØ $× :Ñ :¸>Ó JÐà"Ð%:Ñ:ˆ
Ø×'Ñ'¨:Ò5Ø"&×":Ñ":¸<Ó"HÐØÐ-Ñ-ˆJØ—^‘^ JÓ/ˆ
Ø—\‘\ *Ó-ˆ
ØÐr*   c                 ó  — |j                  «       dd }|d   }t        j                  | j                  dz   || j                  z   dz   t        j                  |j
                  ¬«      }|j                  d«      j                  |«      S )z×
        We are provided embeddings directly. We cannot infer which are padded so just generate sequential position ids.

        Args:
            inputs_embeds: torch.Tensor

        Returns: torch.Tensor
        NrV   r   ru   r   )ro   r%   r&   rO   rp   r!   Ú	unsqueezerm   )r@   rz   r|   Úsequence_lengthrT   s        r(   rw   z;AltRobertaEmbeddings.create_position_ids_from_inputs_embeds  s€   € ð $×(Ñ(Ó*¨3¨BÐ/ˆØ% a™.ˆä—|‘|Ø×Ñ˜qÑ  /°D×4DÑ4DÑ"DÀqÑ"HÔPU×PZÑPZÐcp×cwÑcwô
ˆð ×%Ñ% aÓ(×/Ñ/°Ó<Ð<r*   )NNNNr   )rE   rF   rG   rH   r]   r�   rw   Ú__classcell__©rs   s   @r(   rM   rM   È   s   ø„ ñô

ð4 rsó&öP=r*   rM   c                   óP  ‡ — e Zd Zdˆ fd„	Zdej
                  dej
                  fd„Z	 	 	 	 	 	 ddej
                  deej                     deej                     deej                     d	eej                     d
ee	e	ej                           dee
   de	ej
                     fd„Zˆ xZS )ÚAltRobertaSelfAttentionc                 óâ  •— t         ‰| �  «        |j                  |j                  z  dk7  r2t	        |d«      s&t        d|j                  › d|j                  › d�«      ‚|j                  | _        t        |j                  |j                  z  «      | _        | j                  | j                  z  | _        t        j                  |j                  | j                  «      | _        t        j                  |j                  | j                  «      | _        t        j                  |j                  | j                  «      | _        t        j                  |j                  «      | _        |xs t#        |dd«      | _        | j$                  dk(  s| j$                  d	k(  rF|j&                  | _        t        j(                  d
|j&                  z  dz
  | j                  «      | _        |j,                  | _        y )Nr   Úembedding_sizezThe hidden size (z6) is not a multiple of the number of attention heads (ú)rR   rS   Úrelative_keyÚrelative_key_queryé   r   )r\   r]   r`   Únum_attention_headsrx   Ú
ValueErrorÚintÚattention_head_sizeÚall_head_sizer"   ÚLinearÚqueryÚkeyÚvalueri   Úattention_probs_dropout_probrk   r<   rR   rc   r^   Údistance_embeddingÚ
is_decoder©r@   rr   rR   rs   s      €r(   r]   z AltRobertaSelfAttention.__init__#  s�  ø€ Ü‰ÑÔØ×Ñ × :Ñ :Ñ:¸aÒ?ÌÐPVÐXhÔHiÜØ# F×$6Ñ$6Ð#7ð 8Ø ×4Ñ4Ð5°Qð8óð ð
 $*×#=Ñ#=ˆÔ Ü#& v×'9Ñ'9¸F×<VÑ<VÑ'VÓ#WˆÔ Ø!×5Ñ5¸×8PÑ8PÑPˆÔä—Y‘Y˜v×1Ñ1°4×3EÑ3EÓFˆŒ
Ü—9‘9˜V×/Ñ/°×1CÑ1CÓDˆŒÜ—Y‘Y˜v×1Ñ1°4×3EÑ3EÓFˆŒ
ä—z‘z &×"EÑ"EÓFˆŒØ'>ò (
Ä'ØÐ-¨zóC
ˆÔ$ð ×'Ñ'¨>Ò9¸T×=YÑ=YÐ]qÒ=qØ+1×+IÑ+IˆDÔ(Ü&(§l¡l°1°v×7UÑ7UÑ3UÐXYÑ3YÐ[_×[sÑ[sÓ&tˆDÔ#à ×+Ñ+ˆ�r*   Úxr   c                 ó¤   — |j                  «       d d | j                  | j                  fz   }|j                  |«      }|j	                  dddd«      S )NrV   r   rŽ   r   r	   )ro   r�   r’   ÚviewÚpermute)r@   rœ   Únew_x_shapes      r(   Útranspose_for_scoresz,AltRobertaSelfAttention.transpose_for_scores=  sL   € Ø—f‘f“h˜s �m t×'?Ñ'?À×AYÑAYÐ&ZÑZˆØ�F‰F�;ÓˆØ�y‰y˜˜A˜q !Ó$Ð$r*   Úhidden_statesÚattention_maskÚ	head_maskÚencoder_hidden_statesÚencoder_attention_maskÚpast_key_valueÚoutput_attentionsc                 ó$  — | j                  |«      }|d u}	|	r|�|d   }
|d   }|}�n |	rC| j                  | j                  |«      «      }
| j                  | j                  |«      «      }|}n»|�y| j                  | j                  |«      «      }
| j                  | j                  |«      «      }t	        j
                  |d   |
gd¬«      }
t	        j
                  |d   |gd¬«      }n@| j                  | j                  |«      «      }
| j                  | j                  |«      «      }| j                  |«      }|d u}| j                  r|
|f}t	        j                  ||
j                  dd«      «      }| j                  dk(  s| j                  dk(  �r—|j                  d   |
j                  d   }}|rDt	        j                  |dz
  t        j                  |j                  ¬	«      j                  dd«      }n@t	        j                  |t        j                  |j                  ¬	«      j                  dd«      }t	        j                  |t        j                  |j                  ¬	«      j                  dd«      }||z
  }| j!                  || j"                  z   dz
  «      }|j%                  |j&                  ¬
«      }| j                  dk(  rt	        j(                  d||«      }||z   }nE| j                  dk(  r6t	        j(                  d||«      }t	        j(                  d|
|«      }||z   |z   }|t+        j,                  | j.                  «      z  }|�||z   }t0        j2                  j5                  |d¬«      }| j7                  |«      }|�||z  }t	        j                  ||«      }|j9                  dddd«      j;                  «       }|j=                  «       d d | j>                  fz   }|j                  |«      }|r||fn|f}| j                  r||fz   }|S )Nr   r   rŽ   ©ÚdimrV   éþÿÿÿrŒ   r�   ru   rZ   zbhld,lrd->bhlrzbhrd,lrd->bhlrr	   ) r•   r¡   r–   r—   r%   Úcatrš   ÚmatmulÚ	transposerR   ÚshapeÚtensorrp   r!   rž   r&   r™   rc   Útor[   ÚeinsumÚmathÚsqrtr’   r"   r#   Úsoftmaxrk   rŸ   Ú
contiguousro   r“   )r@   r¢   r£   r¤   r¥   r¦   r§   r¨   Úmixed_query_layerÚis_cross_attentionÚ	key_layerÚvalue_layerÚquery_layerÚ	use_cacheÚattention_scoresÚquery_lengthÚ
key_lengthÚposition_ids_lÚposition_ids_rÚdistanceÚpositional_embeddingÚrelative_position_scoresÚrelative_position_scores_queryÚrelative_position_scores_keyÚattention_probsÚcontext_layerÚnew_context_layer_shapeÚoutputss                               r(   r�   zAltRobertaSelfAttention.forwardB  sç  € ð !ŸJ™J }Ó5Ðð
 3¸$Ð>Ðá .Ð"<à& qÑ)ˆIØ(¨Ñ+ˆKØ3ŠNÙØ×1Ñ1°$·(±(Ð;PÓ2QÓRˆIØ×3Ñ3°D·J±JÐ?TÓ4UÓVˆKØ3‰NØÐ'Ø×1Ñ1°$·(±(¸=Ó2IÓJˆIØ×3Ñ3°D·J±J¸}Ó4MÓNˆKÜŸ	™	 >°!Ñ#4°iÐ"@ÀaÔHˆIÜŸ)™) ^°AÑ%6¸Ð$DÈ!ÔL‰Kà×1Ñ1°$·(±(¸=Ó2IÓJˆIØ×3Ñ3°D·J±J¸}Ó4MÓNˆKà×/Ñ/Ð0AÓBˆà"¨$Ð.ˆ	Ø�?Š?ð (¨Ð5ˆNô !Ÿ<™<¨°Y×5HÑ5HÈÈRÓ5PÓQÐà×'Ñ'¨>Ò9¸T×=YÑ=YÐ]qÓ=qØ'2×'8Ñ'8¸Ñ';¸Y¿_¹_ÈQÑ=O˜*ˆLÙÜ!&§¡¨j¸1©nÄEÇJÁJÐWd×WkÑWkÔ!l×!qÑ!qØ˜ó"‘ô "'§¡¨lÄ%Ç*Á*ÐUb×UiÑUiÔ!j×!oÑ!oÐprÐtuÓ!v�Ü"Ÿ\™\¨*¼E¿J¹JÈ}×OcÑOcÔd×iÑiÐjkÐmoÓpˆNØ%¨Ñ6ˆHà#'×#:Ñ#:¸8Àd×FbÑFbÑ;bÐefÑ;fÓ#gÐ Ø#7×#:Ñ#:À×ARÑARÐ#:Ó#SÐ à×+Ñ+¨~Ò=Ü+0¯<©<Ð8HÈ+ÐWkÓ+lÐ(Ø#3Ð6NÑ#NÑ Ø×-Ñ-Ð1EÒEÜ16·±Ð>NÐP[Ð]qÓ1rÐ.Ü/4¯|©|Ð<LÈiÐYmÓ/nÐ,Ø#3Ð6TÑ#TÐWsÑ#sÐ à+¬d¯i©i¸×8PÑ8PÓ.QÑQÐØÐ%à/°.Ñ@Ðô Ÿ-™-×/Ñ/Ð0@ÀbÐ/ÓIˆð Ÿ,™, Ó7ˆð Ð Ø-°	Ñ9ˆOäŸ™ _°kÓBˆà%×-Ñ-¨a°°A°qÓ9×DÑDÓFˆØ"/×"4Ñ"4Ó"6°s¸Ð";¸t×?QÑ?QÐ>SÑ"SÐØ%×*Ñ*Ð+BÓCˆá6G�= /Ñ2ÈmÐM]ˆà�?Š?Ø Ð 1Ñ1ˆGØˆr*   ©N©NNNNNF)rE   rF   rG   r]   r%   ÚTensorr¡   r   rI   r   Úboolr�   r…   r†   s   @r(   rˆ   rˆ   "  så   ø„ õ,ð4% e§l¡lð %°u·|±|ó %ð 7;Ø15Ø=AØ>BØDHØ,1ñcà—|‘|ðcð ! ×!2Ñ!2Ñ3ðcð ˜E×-Ñ-Ñ.ð	cð
  (¨×(9Ñ(9Ñ:ðcð !)¨×):Ñ):Ñ ;ðcð !  u¨U×->Ñ->Ñ'?Ñ!@ÑAðcð $ D™>ðcð 
ˆu�|‰|Ñ	÷cr*   rˆ   c                   ón   ‡ — e Zd Zˆ fd„Zdej
                  dej
                  dej
                  fd„Zˆ xZS )ÚAltRobertaSelfOutputc                 ó(  •— t         ‰| �  «        t        j                  |j                  |j                  «      | _        t        j                  |j                  |j                  ¬«      | _        t        j                  |j                  «      | _
        y ©NrP   )r\   r]   r"   r”   r`   Údenserg   rh   ri   rj   rk   rq   s     €r(   r]   zAltRobertaSelfOutput.__init__ª  s`   ø€ Ü‰ÑÔÜ—Y‘Y˜v×1Ñ1°6×3EÑ3EÓFˆŒ
ÜŸ™ f×&8Ñ&8¸f×>SÑ>SÔTˆŒÜ—z‘z &×"<Ñ"<Ó=ˆ�r*   r¢   Úinput_tensorr   c                 ór   — | j                  |«      }| j                  |«      }| j                  ||z   «      }|S rÌ   ©rÔ   rk   rg   ©r@   r¢   rÕ   s      r(   r�   zAltRobertaSelfOutput.forward°  ó7   € ØŸ
™
 =Ó1ˆØŸ™ ]Ó3ˆØŸ™ }°|Ñ'CÓDˆØÐr*   ©rE   rF   rG   r]   r%   rÎ   r�   r…   r†   s   @r(   rÑ   rÑ   ©  ó1   ø„ ô>ð U§\¡\ð ÀÇÁð ÐRW×R^ÑR^÷ r*   rÑ   Úeagerc                   ó  ‡ — e Zd Zdˆ fd„	Zd„ Z	 	 	 	 	 	 ddej                  deej                     deej                     deej                     deej                     dee	e	ej                           d	ee
   d
e	ej                     fd„Zˆ xZS )ÚAltRobertaAttentionc                 óž   •— t         ‰| �  «        t        |j                     ||¬«      | _        t        |«      | _        t        «       | _        y )N©rR   )	r\   r]   Ú"ALT_ROBERTA_SELF_ATTENTION_CLASSESÚ_attn_implementationr@   rÑ   ÚoutputÚsetÚpruned_headsr›   s      €r(   r]   zAltRobertaAttention.__init__¾  sC   ø€ Ü‰ÑÔÜ6°v×7RÑ7RÑSØÐ,Cô
ˆŒ	ô +¨6Ó2ˆŒÜ›EˆÕr*   c                 ó>  — t        |«      dk(  ry t        || j                  j                  | j                  j                  | j
                  «      \  }}t        | j                  j                  |«      | j                  _        t        | j                  j                  |«      | j                  _        t        | j                  j                  |«      | j                  _	        t        | j                  j                  |d¬«      | j                  _        | j                  j                  t        |«      z
  | j                  _        | j                  j                  | j                  j                  z  | j                  _        | j
                  j                  |«      | _        y )Nr   r   rª   )r'   r   r@   r�   r’   rå   r   r•   r–   r—   rã   rÔ   r“   Úunion)r@   ÚheadsÚindexs      r(   Úprune_headszAltRobertaAttention.prune_headsÆ  s  € Üˆu‹:˜Š?ØÜ7Ø�4—9‘9×0Ñ0°$·)±)×2OÑ2OÐQU×QbÑQbó
‰ˆˆuô
 -¨T¯Y©Y¯_©_¸eÓDˆ�	‰	ŒÜ*¨4¯9©9¯=©=¸%Ó@ˆ�	‰	ŒÜ,¨T¯Y©Y¯_©_¸eÓDˆ�	‰	ŒÜ.¨t¯{©{×/@Ñ/@À%ÈQÔOˆ�‰Ôð )-¯	©	×(EÑ(EÌÈEË
Ñ(Rˆ�	‰	Ô%Ø"&§)¡)×"?Ñ"?À$Ç)Á)×B_ÑB_Ñ"_ˆ�	‰	ÔØ ×-Ñ-×3Ñ3°EÓ:ˆÕr*   r¢   r£   r¤   r¥   r¦   r§   r¨   r   c           	      óp   — | j                  |||||||«      }| j                  |d   |«      }	|	f|dd  z   }
|
S )Nr   r   )r@   rã   )r@   r¢   r£   r¤   r¥   r¦   r§   r¨   Úself_outputsÚattention_outputrË   s              r(   r�   zAltRobertaAttention.forwardØ  sW   € ð —y‘yØØØØ!Ø"ØØó
ˆð  Ÿ;™; |°A¡¸ÓFÐØ#Ð%¨°Q°RÐ(8Ñ8ˆØˆr*   rÌ   rÍ   )rE   rF   rG   r]   rê   r%   rÎ   r   rI   r   rÏ   r�   r…   r†   s   @r(   rÞ   rÞ   ½  sÆ   ø„ õ"ò;ð* 7;Ø15Ø=AØ>BØDHØ,1ñà—|‘|ðð ! ×!2Ñ!2Ñ3ðð ˜E×-Ñ-Ñ.ð	ð
  (¨×(9Ñ(9Ñ:ðð !)¨×):Ñ):Ñ ;ðð !  u¨U×->Ñ->Ñ'?Ñ!@ÑAðð $ D™>ðð 
ˆu�|‰|Ñ	÷r*   rÞ   c                   óV   ‡ — e Zd Zˆ fd„Zdej
                  dej
                  fd„Zˆ xZS )ÚAltRobertaIntermediatec                 ó  •— t         ‰| �  «        t        j                  |j                  |j
                  «      | _        t        |j                  t        «      rt        |j                     | _        y |j                  | _        y rÌ   )r\   r]   r"   r”   r`   Úintermediate_sizerÔ   Ú
isinstanceÚ
hidden_actÚstrr
   Úintermediate_act_fnrq   s     €r(   r]   zAltRobertaIntermediate.__init__ò  s]   ø€ Ü‰ÑÔÜ—Y‘Y˜v×1Ñ1°6×3KÑ3KÓLˆŒ
Ü�f×'Ñ'¬Ô-Ü'-¨f×.?Ñ.?Ñ'@ˆDÕ$à'-×'8Ñ'8ˆDÕ$r*   r¢   r   c                 óJ   — | j                  |«      }| j                  |«      }|S rÌ   )rÔ   rõ   ©r@   r¢   s     r(   r�   zAltRobertaIntermediate.forwardú  s&   € ØŸ
™
 =Ó1ˆØ×0Ñ0°Ó?ˆØÐr*   rÚ   r†   s   @r(   rï   rï   ñ  s#   ø„ ô9ð U§\¡\ð °e·l±l÷ r*   rï   c                   ón   ‡ — e Zd Zˆ fd„Zdej
                  dej
                  dej
                  fd„Zˆ xZS )ÚAltRobertaOutputc                 ó(  •— t         ‰| �  «        t        j                  |j                  |j
                  «      | _        t        j                  |j
                  |j                  ¬«      | _        t        j                  |j                  «      | _        y rÓ   )r\   r]   r"   r”   rñ   r`   rÔ   rg   rh   ri   rj   rk   rq   s     €r(   r]   zAltRobertaOutput.__init__  s`   ø€ Ü‰ÑÔÜ—Y‘Y˜v×7Ñ7¸×9KÑ9KÓLˆŒ
ÜŸ™ f×&8Ñ&8¸f×>SÑ>SÔTˆŒÜ—z‘z &×"<Ñ"<Ó=ˆ�r*   r¢   rÕ   r   c                 ór   — | j                  |«      }| j                  |«      }| j                  ||z   «      }|S rÌ   r×   rØ   s      r(   r�   zAltRobertaOutput.forward  rÙ   r*   rÚ   r†   s   @r(   rù   rù     rÛ   r*   rù   c                   ó  ‡ — e Zd Zˆ fd„Z	 	 	 	 	 	 ddej
                  deej                     deej                     deej                     deej                     deeeej                           dee	   d	eej
                     fd
„Z
d„ Zˆ xZS )ÚAltRobertaLayerc                 óf  •— t         ‰| �  «        |j                  | _        d| _        t	        |«      | _        |j                  | _        |j                  | _        | j                  r,| j                  st        | › d�«      ‚t	        |d¬«      | _	        t        |«      | _        t        |«      | _        y )Nr   z> should be used as a decoder model if cross attention is addedrS   rà   )r\   r]   Úchunk_size_feed_forwardÚseq_len_dimrÞ   Ú	attentionrš   Úadd_cross_attentionr�   Úcrossattentionrï   Úintermediaterù   rã   rq   s     €r(   r]   zAltRobertaLayer.__init__  s—   ø€ Ü‰ÑÔØ'-×'EÑ'EˆÔ$ØˆÔÜ,¨VÓ4ˆŒØ ×+Ñ+ˆŒØ#)×#=Ñ#=ˆÔ Ø×#Ò#Ø—?’?Ü  D 6Ð)gÐ!hÓiÐiÜ"5°fÐV`Ô"aˆDÔÜ2°6Ó:ˆÔÜ& vÓ.ˆ�r*   r¢   r£   r¤   r¥   r¦   r§   r¨   r   c           	      óÒ  — |�|d d nd }| j                  |||||¬«      }	|	d   }
| j                  r|	dd }|	d   }n|	dd  }d }| j                  rT|�Rt        | d«      st        d| › d�«      ‚|�|d	d  nd }| j	                  |
||||||«      }|d   }
||dd z   }|d   }|z   }t        | j                  | j                  | j                  |
«      }|f|z   }| j                  r|fz   }|S )
NrŽ   )r¨   r§   r   r   rV   r  z'If `encoder_hidden_states` are passed, z` has to be instantiated with cross-attention layers by setting `config.add_cross_attention=True`r¬   )	r  rš   rx   r�   r  r   Úfeed_forward_chunkrÿ   r   )r@   r¢   r£   r¤   r¥   r¦   r§   r¨   Úself_attn_past_key_valueÚself_attention_outputsrí   rË   Úpresent_key_valueÚcross_attn_present_key_valueÚcross_attn_past_key_valueÚcross_attention_outputsÚlayer_outputs                    r(   r�   zAltRobertaLayer.forward  s}  € ð :HÐ9S >°"°1Ñ#5ÐY]Ð Ø!%§¡ØØØØ/Ø3ð "0ó "
Ðð 2°!Ñ4Ðð �?Š?Ø,¨Q¨rÐ2ˆGØ 6°rÑ :Ñà,¨Q¨RÐ0ˆGà'+Ð$Ø�?Š?Ð4Ð@Ü˜4Ð!1Ô2Ü Ø=¸d¸Vð DDð Dóð ð @NÐ?Y¨°r°sÑ(;Ð_cÐ%Ø&*×&9Ñ&9Ø ØØØ%Ø&Ø)Ø!ó'Ð#ð  7°qÑ9ÐØÐ 7¸¸"Ð =Ñ=ˆGð ,CÀ2Ñ+FÐ(Ø 1Ð4PÑ PÐä0Ø×#Ñ# T×%AÑ%AÀ4×CSÑCSÐUeó
ˆð  �/ GÑ+ˆð �?Š?ØÐ!2Ð 4Ñ4ˆGàˆr*   c                 óL   — | j                  |«      }| j                  ||«      }|S rÌ   )r  rã   )r@   rí   Úintermediate_outputr  s       r(   r  z"AltRobertaLayer.feed_forward_chunk`  s,   € Ø"×/Ñ/Ð0@ÓAÐØ—{‘{Ð#6Ð8HÓIˆØÐr*   rÍ   )rE   rF   rG   r]   r%   rÎ   r   rI   r   rÏ   r�   r  r…   r†   s   @r(   rý   rý     sÇ   ø„ ô/ð" 7;Ø15Ø=AØ>BØDHØ,1ñ?à—|‘|ð?ð ! ×!2Ñ!2Ñ3ð?ð ˜E×-Ñ-Ñ.ð	?ð
  (¨×(9Ñ(9Ñ:ð?ð !)¨×):Ñ):Ñ ;ð?ð !  u¨U×->Ñ->Ñ'?Ñ!@ÑAð?ð $ D™>ð?ð 
ˆu�|‰|Ñ	ó?öBr*   rý   c                   óD  ‡ — e Zd Zˆ fd„Z	 	 	 	 	 	 	 	 	 ddej
                  deej                     deej                     deej                     deej                     deeeej                           dee	   d	ee	   d
ee	   dee	   de
eej
                     ef   fd„Zˆ xZS )ÚAltRobertaEncoderc                 óÐ   •— t         ‰| �  «        || _        t        j                  t        |j                  «      D �cg c]  }t        |«      ‘Œ c}«      | _        d| _	        y c c}w ©NF)
r\   r]   rr   r"   Ú
ModuleListÚrangeÚnum_hidden_layersrý   ÚlayerÚgradient_checkpointing©r@   rr   Ú_rs   s      €r(   r]   zAltRobertaEncoder.__init__h  sN   ø€ Ü‰ÑÔØˆŒÜ—]‘]ÄUÈ6×KcÑKcÓEdÖ#eÀ¤O°FÕ$;Ò#eÓfˆŒ
Ø&+ˆÕ#ùò $fó   ½A#r¢   r£   r¤   r¥   r¦   Úpast_key_valuesr½   r¨   Úoutput_hidden_statesÚreturn_dictr   c                 óš  — |	rdnd }|rdnd }|r| j                   j                  rdnd }| j                  r%| j                  r|rt        j                  d«       d}|rdnd }t        | j                  «      D ]¤  \  }}|	r||fz   }|�||   nd }|�||   nd }| j                  r/| j                  r#| j                  |j                  |||||||«      }n ||||||||«      }|d   }|r	||d   fz  }|sŒ|||d   fz   }| j                   j                  sŒœ||d   fz   }Œ¦ |	r||fz   }|
st        d„ |||||fD «       «      S t        |||||¬	«      S )
NrK   zZ`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...Fr   rV   r   rŽ   c              3   ó$   K  — | ]  }|�|–— Œ
 y ­wrÌ   rK   ©r>   Úvs     r(   rA   z,AltRobertaEncoder.forward.<locals>.<genexpr>°  s   è ø€ ò 
àð �=ô ñ
ùs   ‚)Úlast_hidden_stater  r¢   Ú
attentionsÚcross_attentions)rr   r  r  ÚtrainingÚloggerÚwarning_onceÚ	enumerater  Ú_gradient_checkpointing_funcÚ__call__rB   r   )r@   r¢   r£   r¤   r¥   r¦   r  r½   r¨   r  r  Úall_hidden_statesÚall_self_attentionsÚall_cross_attentionsÚnext_decoder_cacheÚiÚlayer_moduleÚlayer_head_maskr§   Úlayer_outputss                       r(   r�   zAltRobertaEncoder.forwardn  sÎ  € ñ #7™B¸DÐÙ$5™b¸4ÐÙ%6¸4¿;¹;×;ZÒ;Z™rÐ`dÐà×&Ò&¨4¯=ª=ÙÜ×#Ñ#Øpôð "�	á#,™R°$ÐÜ(¨¯©Ó4ò #	V‰OˆAˆ|Ù#Ø$5¸Ð8HÑ$HÐ!à.7Ð.C˜i¨šlÈˆOØ3BÐ3N˜_¨QÒ/ÐTXˆNà×*Ò*¨t¯}ª}Ø $× AÑ AØ ×)Ñ)Ø!Ø"Ø#Ø)Ø*Ø"Ø%ó	!‘ñ !-Ø!Ø"Ø#Ø)Ø*Ø"Ø%ó!�ð *¨!Ñ,ˆMÙØ" }°RÑ'8Ð&:Ñ:Ð"Ú Ø&9¸]È1Ñ=MÐ<OÑ&OÐ#Ø—;‘;×2Ó2Ø+?À=ÐQRÑCSÐBUÑ+UÑ(ðG#	VñJ  Ø 1°]Ð4DÑ DÐáÜñ 
ð "Ø&Ø%Ø'Ø(ðô
ó 
ð 
ô 9Ø+Ø.Ø+Ø*Ø1ô
ð 	
r*   )	NNNNNNFFT)rE   rF   rG   r]   r%   rÎ   r   rI   r   rÏ   r   r   r�   r…   r†   s   @r(   r  r  g  s  ø„ ô,ð 7;Ø15Ø=AØ>BØEIØ$(Ø,1Ø/4Ø&*ñS
à—|‘|ðS
ð ! ×!2Ñ!2Ñ3ðS
ð ˜E×-Ñ-Ñ.ð	S
ð
  (¨×(9Ñ(9Ñ:ðS
ð !)¨×):Ñ):Ñ ;ðS
ð " %¨¨e×.?Ñ.?Ñ(@Ñ"AÑBðS
ð ˜D‘>ðS
ð $ D™>ðS
ð ' t™nðS
ð ˜d‘^ðS
ð 
ˆu�U—\‘\Ñ"Ð$MÐMÑ	N÷S
r*   r  c                   óV   ‡ — e Zd Zˆ fd„Zdej
                  dej
                  fd„Zˆ xZS )ÚAltRobertaPoolerc                 ó²   •— t         ‰| �  «        t        j                  |j                  |j                  «      | _        t        j                  «       | _        y rÌ   )r\   r]   r"   r”   r`   rÔ   ÚTanhÚ
activationrq   s     €r(   r]   zAltRobertaPooler.__init__Æ  s9   ø€ Ü‰ÑÔÜ—Y‘Y˜v×1Ñ1°6×3EÑ3EÓFˆŒ
ÜŸ'™'›)ˆ�r*   r¢   r   c                 ó\   — |d d …df   }| j                  |«      }| j                  |«      }|S )Nr   )rÔ   r8  )r@   r¢   Úfirst_token_tensorÚpooled_outputs       r(   r�   zAltRobertaPooler.forwardË  s6   € ð +ª1¨a¨4Ñ0ÐØŸ
™
Ð#5Ó6ˆØŸ™¨Ó6ˆØÐr*   rÚ   r†   s   @r(   r5  r5  Å  s#   ø„ ô$ð
 U§\¡\ð °e·l±l÷ r*   r5  c                   óô   ‡ — e Zd ZdZˆ fd„Zdej                  dedefd„Z	 	 	 ddej                  de	ej                     d	e	ej                     d
e	e
   deej                  e	ej                     f   f
d„Zˆ xZS )ÚAltCLIPAttentionz=Multi-headed attention from 'Attention Is All You Need' paperc                 ó
  •— t         ‰| �  «        || _        |j                  | _        |j
                  | _        | j                  | j                  z  | _        | j                  | j                  z  | j                  k7  r&t        d| j                  › d| j                  › d�«      ‚| j                  dz  | _	        |j                  | _        t        j                  | j                  | j                  «      | _        t        j                  | j                  | j                  «      | _        t        j                  | j                  | j                  «      | _        t        j                  | j                  | j                  «      | _        y )Nz;embed_dim must be divisible by num_heads (got `embed_dim`: z and `num_heads`: ú).ç      à¿)r\   r]   rr   r`   Ú	embed_dimr�   Ú	num_headsÚhead_dimr�   ÚscaleÚattention_dropoutrk   r"   r”   Úk_projÚv_projÚq_projÚout_projrq   s     €r(   r]   zAltCLIPAttention.__init__Ø  s  ø€ Ü‰ÑÔØˆŒØ×+Ñ+ˆŒØ×3Ñ3ˆŒØŸ™¨$¯.©.Ñ8ˆŒØ�=‰=˜4Ÿ>™>Ñ)¨T¯^©^Ò;ÜØMÈdÏnÉnÐM]ð ^Ø—N‘NÐ# 2ð'óð ð —]‘] DÑ(ˆŒ
Ø×/Ñ/ˆŒä—i‘i §¡°·±Ó?ˆŒÜ—i‘i §¡°·±Ó?ˆŒÜ—i‘i §¡°·±Ó?ˆŒÜŸ	™	 $§.¡.°$·.±.ÓAˆ�r*   r±   Úseq_lenÚbszc                 óŽ   — |j                  ||| j                  | j                  «      j                  dd«      j	                  «       S )Nr   rŽ   )rž   rB  rC  r¯   r·   )r@   r±   rJ  rK  s       r(   Ú_shapezAltCLIPAttention._shapeë  s7   € Ø�{‰{˜3 ¨¯©¸¿¹ÓG×QÑQÐRSÐUVÓW×bÑbÓdÐdr*   r¢   r£   Úcausal_attention_maskr¨   r   c                 ó”  — |j                  «       \  }}}| j                  |«      | j                  z  }| j                  | j	                  |«      d|«      }	| j                  | j                  |«      d|«      }
|| j                  z  d| j                  f} | j                  |||«      j                  |Ž } |	j                  |Ž }	 |
j                  |Ž }
|	j                  d«      }t        j                  ||	j                  dd«      «      }|j                  «       || j                  z  ||fk7  r/t        d|| j                  z  ||f› d|j                  «       › �«      ‚|�{|j                  «       |d||fk7  r#t        d|d||f› d|j                  «       › �«      ‚|j                  || j                  ||«      |z   }|j                  || j                  z  ||«      }|�{|j                  «       |d||fk7  r#t        d|d||f› d|j                  «       › �«      ‚|j                  || j                  ||«      |z   }|j                  || j                  z  ||«      }t        j                  j                  |d¬«      }|r?|j                  || j                  ||«      }|j                  || j                  z  ||«      }nd}t        j                  j!                  || j                   | j"                  ¬	«      }t        j                  ||
«      }|j                  «       || j                  z  || j                  fk7  r7t        d
|| j                  || j                  f› d|j                  «       › �«      ‚|j                  || j                  || j                  «      }|j                  dd«      }|j%                  |||«      }| j'                  |«      }||fS )z#Input shape: Batch x Time x ChannelrV   r   rŽ   z$Attention weights should be of size z	, but is Nz!Attention mask should be of size rª   )Úpr&  z `attn_output` should be of size )ro   rH  rD  rM  rF  rG  rB  rC  rž   r%   Úbmmr¯   r�   r"   r#   r¶   rk   r&  ÚreshaperI  )r@   r¢   r£   rN  r¨   rK  Útgt_lenrA  Úquery_statesÚ
key_statesÚvalue_statesÚ
proj_shapeÚsrc_lenÚattn_weightsÚattn_weights_reshapedÚ
attn_probsÚattn_outputs                    r(   r�   zAltCLIPAttention.forwardî  sÕ  € ð #0×"4Ñ"4Ó"6ÑˆˆW�ið —{‘{ =Ó1°D·J±JÑ>ˆØ—[‘[ §¡¨]Ó!;¸RÀÓEˆ
Ø—{‘{ 4§;¡;¨}Ó#=¸rÀ3ÓGˆà˜DŸN™NÑ*¨B°·±Ð>ˆ
ØC�t—{‘{ <°¸#Ó>×CÑCÀZÐPˆØ$�Z—_‘_ jÐ1ˆ
Ø(�|×(Ñ(¨*Ð5ˆà—/‘/ !Ó$ˆÜ—y‘y ¨z×/CÑ/CÀAÀqÓ/IÓJˆà×ÑÓ 3¨¯©Ñ#7¸À'Ð"JÒJÜØ6¸¸d¿n¹nÑ8LÈgÐW^Ð7_Ð6`ð aØ ×%Ñ%Ó'Ð(ð*óð ð !Ð,Ø$×)Ñ)Ó+°°Q¸ÀÐ/IÒIÜ Ø7¸¸aÀÈ'Ð8RÐ7Sð TØ-×2Ñ2Ó4Ð5ð7óð ð (×,Ñ,¨S°$·.±.À'È7ÓSÐVkÑkˆLØ'×,Ñ,¨S°4·>±>Ñ-AÀ7ÈGÓTˆLàÐ%Ø×"Ñ"Ó$¨¨a°¸'Ð(BÒBÜ Ø7¸¸aÀÈ'Ð8RÐ7SÐS\Ð]k×]pÑ]pÓ]rÐ\sÐtóð ð (×,Ñ,¨S°$·.±.À'È7ÓSÐVdÑdˆLØ'×,Ñ,¨S°4·>±>Ñ-AÀ7ÈGÓTˆLä—}‘}×,Ñ,¨\¸rÐ,ÓBˆáð
 %1×$5Ñ$5°c¸4¿>¹>È7ÐT[Ó$\Ð!Ø0×5Ñ5°c¸D¿N¹NÑ6JÈGÐU\Ó]‰Là$(Ð!ä—]‘]×*Ñ*¨<¸4¿<¹<ÐRV×R_ÑR_Ð*Ó`ˆ
ä—i‘i 
¨LÓ9ˆà×ÑÓ #¨¯©Ñ"6¸ÀÇÁÐ!OÒOÜØ2°C¸¿¹ÈÐRV×R_ÑR_Ð3`Ð2að bØ×$Ñ$Ó&Ð'ð)óð ð
 "×&Ñ& s¨D¯N©N¸GÀTÇ]Á]ÓSˆØ!×+Ñ+¨A¨qÓ1ˆØ!×)Ñ)¨#¨w¸	ÓBˆà—m‘m KÓ0ˆàÐ1Ð1Ð1r*   )NNF)rE   rF   rG   rH   r]   r%   rÎ   r‘   rM  r   rÏ   r   r�   r…   r†   s   @r(   r=  r=  Õ  s¥   ø„ ÙGôBð&e˜UŸ\™\ð e°Cð e¸có eð 26Ø8<Ø,1ñL2à—|‘|ðL2ð ! §¡Ñ.ðL2ð  (¨¯©Ñ5ð	L2ð
 $ D™>ðL2ð 
ˆu�|‰|˜X e§l¡lÑ3Ð3Ñ	4÷L2r*   r=  c                   óV   ‡ — e Zd Zˆ fd„Zdej
                  dej
                  fd„Zˆ xZS )Ú
AltCLIPMLPc                 ó  •— t         ‰| �  «        || _        t        |j                     | _        t        j                  |j                  |j                  «      | _
        t        j                  |j                  |j                  «      | _        y rÌ   )r\   r]   rr   r
   ró   Úactivation_fnr"   r”   r`   rñ   Úfc1Úfc2rq   s     €r(   r]   zAltCLIPMLP.__init__?  sd   ø€ Ü‰ÑÔØˆŒÜ# F×$5Ñ$5Ñ6ˆÔÜ—9‘9˜V×/Ñ/°×1IÑ1IÓJˆŒÜ—9‘9˜V×5Ñ5°v×7IÑ7IÓJˆ�r*   r¢   r   c                 ól   — | j                  |«      }| j                  |«      }| j                  |«      }|S rÌ   )ra  r`  rb  r÷   s     r(   r�   zAltCLIPMLP.forwardF  s4   € ØŸ™ Ó/ˆØ×*Ñ*¨=Ó9ˆØŸ™ Ó/ˆØÐr*   rÚ   r†   s   @r(   r^  r^  >  s$   ø„ ôKð U§\¡\ð °e·l±l÷ r*   r^  c                   ó    ‡ — e Zd Zdefˆ fd„Z	 d	dej                  dej                  dej                  dee   de	ej                     f
d„Zˆ xZS )
ÚAltCLIPEncoderLayerrr   c                 óD  •— t         ‰| �  «        |j                  | _        t	        |«      | _        t        j                  | j                  |j                  ¬«      | _	        t        |«      | _        t        j                  | j                  |j                  ¬«      | _        y rÓ   )r\   r]   r`   rA  r=  Ú	self_attnr"   rg   rh   Úlayer_norm1r^  ÚmlpÚlayer_norm2rq   s     €r(   r]   zAltCLIPEncoderLayer.__init__N  sm   ø€ Ü‰ÑÔØ×+Ñ+ˆŒÜ)¨&Ó1ˆŒÜŸ<™<¨¯©¸F×<QÑ<QÔRˆÔÜ˜fÓ%ˆŒÜŸ<™<¨¯©¸F×<QÑ<QÔRˆÕr*   r¢   r£   rN  r¨   r   c                 óÎ   — |}| j                  |«      }| j                  ||||¬«      \  }}||z   }|}| j                  |«      }| j                  |«      }||z   }|f}|r||fz  }|S )aI  
        Args:
            hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)`
            attention_mask (`torch.FloatTensor`): attention mask of size
                `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values.
                `(config.encoder_attention_heads,)`.
            output_attentions (`bool`, *optional*):
                Whether or not to return the attentions tensors of all attention layers. See `attentions` under
                returned tensors for more detail.
        )r¢   r£   rN  r¨   )rh  rg  rj  ri  )r@   r¢   r£   rN  r¨   ÚresidualrY  rË   s           r(   r�   zAltCLIPEncoderLayer.forwardV  s’   € ð" !ˆà×(Ñ(¨Ó7ˆØ&*§n¡nØ'Ø)Ø"7Ø/ð	 '5ó '
Ñ#ˆ�|ð ! =Ñ0ˆà ˆØ×(Ñ(¨Ó7ˆØŸ™ Ó/ˆØ  =Ñ0ˆà Ð"ˆáØ˜�Ñ&ˆGàˆr*   ©F)rE   rF   rG   r   r]   r%   rÎ   r   rÏ   r   rI   r�   r…   r†   s   @r(   re  re  M  sf   ø„ ðS˜}õ Sð -2ñ&à—|‘|ð&ð Ÿ™ð&ð  %Ÿ|™|ð	&ð
 $ D™>ð&ð 
ˆu× Ñ Ñ	!÷&r*   re  c                   ó¤   ‡ — e Zd ZdZdefˆ fd„Z	 	 	 	 	 ddeej                     deej                     dee	   dee	   dee	   d	e
eef   fd
„Zˆ xZS )ÚAltCLIPEncoderz³
    Transformer encoder consisting of `config.num_hidden_layers` self attention layers. Each layer is a
    [`AltCLIPEncoderLayer`].

    Args:
        config: AltCLIPConfig
    rr   c                 óÐ   •— t         ‰| �  «        || _        t        j                  t        |j                  «      D �cg c]  }t        |«      ‘Œ c}«      | _        d| _	        y c c}w r  )
r\   r]   rr   r"   r  r  r  re  Úlayersr  r  s      €r(   r]   zAltCLIPEncoder.__init__ˆ  sP   ø€ Ü‰ÑÔØˆŒÜ—m‘mÌ%ÐPV×PhÑPhÓJiÖ$jÀQÔ%8¸Õ%@Ò$jÓkˆŒØ&+ˆÕ#ùò %kr  r£   rN  r¨   r  r  r   c                 ó  — |�|n| j                   j                  }|�|n| j                   j                  }|�|n| j                   j                  }|rdnd}|rdnd}|}	t	        | j
                  «      D ]b  \  }
}|r||	fz   }| j                  r,| j                  r | j                  |j                  |	|||«      }n ||	|||¬«      }|d   }	|sŒZ||d   fz   }Œd |r||	fz   }|st        d„ |	||fD «       «      S t        |	||¬«      S )aÕ  
        Args:
            inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
                Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
                This is useful if you want more control over how to convert `input_ids` indices into associated vectors
                than the model's internal embedding lookup matrix.
            attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
                Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:

                - 1 for tokens that are **not masked**,
                - 0 for tokens that are **masked**.

                [What are attention masks?](../glossary#attention-mask)
            causal_attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
                Causal mask for the text model. Mask values selected in `[0, 1]`:

                - 1 for tokens that are **not masked**,
                - 0 for tokens that are **masked**.

                [What are attention masks?](../glossary#attention-mask)
            output_attentions (`bool`, *optional*):
                Whether or not to return the attentions tensors of all attention layers. See `attentions` under
                returned tensors for more detail.
            output_hidden_states (`bool`, *optional*):
                Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors
                for more detail.
            return_dict (`bool`, *optional*):
                Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
        NrK   )r¨   r   r   c              3   ó&   K  — | ]	  }|€Œ|–— Œ y ­wrÌ   rK   r!  s     r(   rA   z)AltCLIPEncoder.forward.<locals>.<genexpr>Ú  s   è ø€ Òe˜qÐWXÑWdœÑeùs   ‚Š)r#  r¢   r$  )rr   r¨   r  Úuse_return_dictr)  rq  r  r&  r*  r+  rB   r   )r@   rz   r£   rN  r¨   r  r  Úencoder_statesÚall_attentionsr¢   ÚidxÚencoder_layerr3  s                r(   r�   zAltCLIPEncoder.forwardŽ  sH  € ðL 2CÐ1NÑ-ÐTX×T_ÑT_×TqÑTqÐà$8Ð$DÑ È$Ï+É+×JjÑJjð 	ð &1Ð%<‘kÀ$Ç+Á+×B]ÑB]ˆá3™¸ˆÙ0™°dˆà%ˆÜ"+¨D¯K©KÓ"8ò 	FÑˆC�Ù#Ø!/°=Ð2BÑ!B�Ø×*Ò*¨t¯}ª}Ø $× AÑ AØ!×*Ñ*Ø!Ø"Ø)Ø%ó!‘ñ !.Ø!Ø"Ø)Ø&7ô	!�ð *¨!Ñ,ˆMâ Ø!/°=ÀÑ3CÐ2EÑ!E‘ð-	Fñ0  Ø+¨}Ð.>Ñ>ˆNáÜÑe ]°NÀNÐ$SÔeÓeÐeÜØ+¸>ÐVdô
ð 	
r*   )NNNNN)rE   rF   rG   rH   r   r]   r   r%   rÎ   rÏ   r   r   r   r�   r…   r†   s   @r(   ro  ro    s•   ø„ ñð,˜}õ ,ð 26Ø8<Ø,0Ø/3Ø&*ñO
ð ! §¡Ñ.ðO
ð  (¨¯©Ñ5ð	O
ð
 $ D™>ðO
ð ' t™nðO
ð ˜d‘^ðO
ð 
ˆu�oÐ%Ñ	&÷O
r*   ro  c                   óž   ‡ — e Zd Zdefˆ fd„Zdej                  dededej                  fd„Zd
dej                  dej                  fd	„Z
ˆ xZS )ÚAltCLIPVisionEmbeddingsrr   c                 óÚ  •— t         ‰| �  «        || _        |j                  | _        |j
                  | _        |j                  | _        t        j                  t        j                  | j                  «      «      | _        t        j                  |j                  | j                  | j                  | j                  d¬«      | _        | j
                  | j                  z  dz  | _        | j                  dz   | _        t        j"                  | j                   | j                  «      | _        | j'                  dt        j(                  | j                   «      j+                  d«      d¬«       y )NF)Úin_channelsÚout_channelsÚkernel_sizeÚstrideÚbiasrŽ   r   rT   rU   rW   )r\   r]   rr   r`   rA  Ú
image_sizeÚ
patch_sizer"   Ú	Parameterr%   ÚrandnÚclass_embeddingÚConv2dÚnum_channelsÚpatch_embeddingÚnum_patchesÚnum_positionsr^   Úposition_embeddingrl   r&   rm   rq   s     €r(   r]   z AltCLIPVisionEmbeddings.__init__â  s	  ø€ Ü‰ÑÔØˆŒØ×+Ñ+ˆŒØ ×+Ñ+ˆŒØ ×+Ñ+ˆŒä!Ÿ|™|¬E¯K©K¸¿¹Ó,GÓHˆÔä!Ÿy™yØ×+Ñ+ØŸ™ØŸ™Ø—?‘?Øô 
ˆÔð !ŸO™O¨t¯©Ñ>À1ÑDˆÔØ!×-Ñ-°Ñ1ˆÔÜ"$§,¡,¨t×/AÑ/AÀ4Ç>Á>Ó"RˆÔØ×Ñ˜^¬U¯\©\¸$×:LÑ:LÓ-M×-TÑ-TÐU\Ó-]ÐjoÐÕpr*   r€   ÚheightÚwidthr   c                 óÒ  — |j                   d   dz
  }| j                  j                  j                  d«      }|j                   d   dz
  }t        j
                  j                  «       s%||k(  r ||k(  r| j                  | j                  «      S |dd…dd…f   }|dd…dd…f   }|j                   d   }	|| j                  z  }
|| j                  z  }t        |dz  «      }|j                  d|||	«      }|j                  dddd«      }t        j                  j                  ||
|fdd	¬
«      }|j                  dddd«      j                  dd|	«      }t	        j                   ||fd¬«      S )a   
        This method allows to interpolate the pre-trained position encodings, to be able to use the model on higher resolution
        images. This method is also adapted to support torch.jit tracing.

        Adapted from:
        - https://github.com/facebookresearch/dino/blob/de9ee3df6cf39fac952ab558447af1fa1365362a/vision_transformer.py#L174-L194, and
        - https://github.com/facebookresearch/dinov2/blob/e1277af2ba9496fbadf7aec6eba56e8d882d1e35/dinov2/models/vision_transformer.py#L179-L211
        r   r   NrV   g      à?r	   rŽ   ÚbicubicF)ro   ÚmodeÚalign_cornersrª   )r°   r‹  Úweightrƒ   r%   ÚjitÚ
is_tracingrT   r‚  r   rR  rŸ   r"   r#   Úinterpolaterž   r­   )r@   r€   rŒ  r�  r‰  r‹  rŠ  Úclass_pos_embedÚpatch_pos_embedr«   Ú
new_heightÚ	new_widthÚsqrt_num_positionss                r(   Úinterpolate_pos_encodingz0AltCLIPVisionEmbeddings.interpolate_pos_encodingø  sv  € ð !×&Ñ& qÑ)¨AÑ-ˆØ!×4Ñ4×;Ñ;×EÑEÀaÓHÐØ*×0Ñ0°Ñ3°aÑ7ˆô �y‰y×#Ñ#Ô%¨+¸Ò*FÈ6ÐUZÊ?Ø×*Ñ*¨4×+<Ñ+<Ó=Ð=à,ªQ°°°¨UÑ3ˆØ,ªQ°±¨UÑ3ˆà×Ñ˜rÑ"ˆà˜tŸ™Ñ.ˆ
Ø˜TŸ_™_Ñ,ˆ	ä& }°cÑ'9Ó:ÐØ)×1Ñ1°!Ð5GÐI[Ð]`ÓaˆØ)×1Ñ1°!°Q¸¸1Ó=ˆäŸ-™-×3Ñ3ØØ˜iÐ(ØØð	 4ó 
ˆð *×1Ñ1°!°Q¸¸1Ó=×BÑBÀ1ÀbÈ#ÓNˆä�y‰y˜/¨?Ð;ÀÔCÐCr*   Úpixel_valuesc                 ó`  — |j                   \  }}}}|sJ|| j                  k7  s|| j                  k7  r,t        d|› d|› d| j                  › d| j                  › d�	«      ‚| j                  j                  j
                  }| j                  |j                  |¬«      «      }|j                  d«      j                  dd«      }| j                  j                  |dd«      }	t        j                  |	|gd¬	«      }
|r|
| j                  |
||«      z   }
|
S |
| j                  | j                  «      z   }
|
S )
NzInput image size (Ú*z) doesn't match model (r?  rZ   rŽ   r   rV   rª   )r°   r�  r�   rˆ  r’  r[   r²   Úflattenr¯   r…  rm   r%   r­   r›  r‹  rT   )r@   rœ  r›  Ú
batch_sizer  rŒ  r�  Útarget_dtypeÚpatch_embedsÚclass_embedsr€   s              r(   r�   zAltCLIPVisionEmbeddings.forward!  s6  € Ø'3×'9Ñ'9Ñ$ˆ
�A�v˜uÙ'¨V°t·±Ò-FÈ%ÐSW×SbÑSbÒJbÜØ$ V H¨A¨e¨WÐ4KÈDÏOÉOÐK\Ð\]Ð^b×^mÑ^mÐ]nÐnpÐqóð ð ×+Ñ+×2Ñ2×8Ñ8ˆØ×+Ñ+¨L¯O©OÀ,¨OÓ,OÓPˆØ#×+Ñ+¨AÓ.×8Ñ8¸¸AÓ>ˆà×+Ñ+×2Ñ2°:¸qÀ"ÓEˆÜ—Y‘Y ¨lÐ;ÀÔCˆ
Ù#Ø# d×&CÑ&CÀJÐPVÐX]Ó&^Ñ^ˆJð Ðð $ d×&=Ñ&=¸d×>OÑ>OÓ&PÑPˆJØÐr*   rm  )rE   rF   rG   r   r]   r%   rÎ   r‘   r›  rI   r�   r…   r†   s   @r(   rz  rz  á  se   ø„ ðqÐ2õ qð,'D°5·<±<ð 'DÈð 'DÐUXð 'DÐ]b×]iÑ]ió 'DñR E×$5Ñ$5ð ÐZ_×ZfÑZf÷ r*   rz  c                   ó&   — e Zd ZdZeZdZdZg Zd„ Z	y)ÚAltCLIPPreTrainedModelz†
    An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained
    models.
    ÚaltclipTc                 ó:  — | j                   j                  }t        |t        «      rë| j                   j                  }t        j
                  j                  |j                  d|j                  dz  |z  ¬«       t        j
                  j                  |j                  j                  |j                   j                  |z  ¬«       t        j
                  j                  |j                  j                  |j                   j                  |z  ¬«       yt        |t        «      �r,| j                   j                  }|j                  dz  d|j                   j                  z  dz  z  |z  }|j                  dz  |z  }t        j
                  j                  |j                  j                  |¬«       t        j
                  j                  |j                   j                  |¬«       t        j
                  j                  |j"                  j                  |¬«       t        j
                  j                  |j$                  j                  |¬«       yt        |t&        «      rÙ| j                   j                  }|j                   j(                  dz  d|j                   j                  z  dz  z  |z  }d|j                   j(                  z  dz  |z  }t        j
                  j                  |j*                  j                  |¬«       t        j
                  j                  |j,                  j                  |¬«       yt        |t.        «      rÕt        j
                  j                  |j0                  j                  |j2                  dz  | j                   j                  z  ¬«       d|j0                  _        t        j
                  j                  |j6                  j                  |j8                  dz  | j                   j                  z  ¬«       d|j6                  _        yt        |t        j:                  «      rJ|j<                  j>                  jA                  «        |j                  j>                  jC                  d«       yt        |t        jD                  «      rm|j                  j>                  j                  d| j                   j                  ¬«       |j<                  �%|j<                  j>                  jA                  «        yyt        |t        jF                  «      rz|j                  j>                  j                  d| j                   j                  ¬«       |jH                  �2|j                  j>                  |jH                     jA                  «        yyy)	zInitialize the weightsg        r@  )ÚmeanÚstd)r©  rŽ   Tg      ð?N)%rr   Úinitializer_factorrò   rz  r"   ÚinitÚnormal_r…  rA  rˆ  r’  Úinitializer_ranger‹  r=  r  rH  rF  rG  rI  r^  r`   ra  rb  ÚAltCLIPModelÚtext_projectionÚtext_embed_dimÚ_is_hf_initializedÚvisual_projectionÚvision_embed_dimrg   r€  ÚdataÚzero_Úfill_r”   r^   rO   )r@   ÚmoduleÚfactorÚin_proj_stdÚout_proj_stdÚfc_stds         r(   Ú_init_weightsz$AltCLIPPreTrainedModel._init_weights?  så  € à—‘×/Ñ/ˆÜ�fÔ5Ô6Ø—[‘[×3Ñ3ˆFÜ�G‰G�O‰O˜F×2Ñ2¸À&×BRÑBRÐTXÑBXÐ[aÑBaˆOÔbÜ�G‰G�O‰O˜F×2Ñ2×9Ñ9¸v¿}¹}×?^Ñ?^ÐagÑ?gˆOÔhÜ�G‰G�O‰O˜F×5Ñ5×<Ñ<À&Ç-Á-×BaÑBaÐdjÑBjˆOÕkÜ˜Ô 0Õ1Ø—[‘[×3Ñ3ˆFØ!×+Ñ+¨TÑ1°q¸6¿=¹=×;ZÑ;ZÑ7ZÐ_cÑ6cÑdÐgmÑmˆKØ"×,Ñ,¨dÑ2°fÑ<ˆLÜ�G‰G�O‰O˜FŸM™M×0Ñ0°kˆOÔBÜ�G‰G�O‰O˜FŸM™M×0Ñ0°kˆOÔBÜ�G‰G�O‰O˜FŸM™M×0Ñ0°kˆOÔBÜ�G‰G�O‰O˜FŸO™O×2Ñ2¸ˆOÕEÜ˜¤
Ô+Ø—[‘[×3Ñ3ˆFØ!Ÿ=™=×4Ñ4°dÑ:ÀÀFÇMÁM×DcÑDcÑ@cÐhlÑ?lÑmÐpvÑvˆKØ˜&Ÿ-™-×3Ñ3Ñ3¸Ñ<¸vÑEˆFÜ�G‰G�O‰O˜FŸJ™J×-Ñ-°6ˆOÔ:Ü�G‰G�O‰O˜FŸJ™J×-Ñ-°;ˆOÕ?Ü˜¤Ô-Ü�G‰G�O‰OØ×&Ñ&×-Ñ-Ø×)Ñ)¨4Ñ/°$·+±+×2PÑ2PÑPð ô ð 9=ˆF×"Ñ"Ô5Ü�G‰G�O‰OØ×(Ñ(×/Ñ/Ø×+Ñ+¨TÑ1°D·K±K×4RÑ4RÑRð ô ð ;?ˆF×$Ñ$Õ7Ü˜¤§¡Ô-Ø�K‰K×Ñ×"Ñ"Ô$Ø�M‰M×Ñ×$Ñ$ SÕ)Ü˜¤§	¡	Ô*Ø�M‰M×Ñ×&Ñ&¨C°T·[±[×5SÑ5SÐ&ÔTØ�{‰{Ð&Ø—‘× Ñ ×&Ñ&Õ(ð 'ä˜¤§¡Ô-Ø�M‰M×Ñ×&Ñ&¨C°T·[±[×5SÑ5SÐ&ÔTØ×!Ñ!Ð-Ø—‘×"Ñ" 6×#5Ñ#5Ñ6×<Ñ<Õ>ð .ð .r*   N)
rE   rF   rG   rH   r   Úconfig_classÚbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_moduler¼  rK   r*   r(   r¥  r¥  4  s%   „ ñð
 !€LØ!ÐØ&*Ð#ØÐó+?r*   r¥  c                   ó¼   ‡ — e Zd Zdefˆ fd„Z ee«       eee¬«      	 	 	 	 	 dde	e
j                     de	e   de	e   de	e   de	e   d	eeef   fd
„«       «       Zˆ xZS )ÚAltCLIPVisionTransformerrr   c                 ó   •— t         ‰| �  «        || _        |j                  }t	        |«      | _        t        j                  ||j                  ¬«      | _	        t        |«      | _        t        j                  ||j                  ¬«      | _        y rÓ   )r\   r]   rr   r`   rz  r€   r"   rg   rh   Úpre_layrnormro  ÚencoderÚpost_layernorm)r@   rr   rA  rs   s      €r(   r]   z!AltCLIPVisionTransformer.__init__n  sj   ø€ Ü‰ÑÔØˆŒØ×&Ñ&ˆ	ä1°&Ó9ˆŒÜŸL™L¨¸×8MÑ8MÔNˆÔÜ% fÓ-ˆŒÜ Ÿl™l¨9¸&×:OÑ:OÔPˆÕr*   ©Úoutput_typer½  rœ  r¨   r  r  r›  r   c                 óÌ  — |�|n| j                   j                  }|�|n| j                   j                  }|�|n| j                   j                  }|€t	        d«      ‚| j                  ||¬«      }| j                  |«      }| j                  ||||¬«      }|d   }|dd…ddd…f   }	| j                  |	«      }	|s
||	f|dd z   S t        ||	|j                  |j                  ¬«      S )z
        Returns:

        Nz You have to specify pixel_values)r›  )rz   r¨   r  r  r   r   ©r#  Úpooler_outputr¢   r$  )rr   r¨   r  rt  r�   r€   rÄ  rÅ  rÆ  r   r¢   r$  )
r@   rœ  r¨   r  r  r›  r¢   Úencoder_outputsr#  r;  s
             r(   r�   z AltCLIPVisionTransformer.forwardx  s  € ð 2CÐ1NÑ-ÐTX×T_ÑT_×TqÑTqÐà$8Ð$DÑ È$Ï+É+×JjÑJjð 	ð &1Ð%<‘kÀ$Ç+Á+×B]ÑB]ˆàÐÜÐ?Ó@Ð@àŸ™¨ÐOg˜ÓhˆØ×)Ñ)¨-Ó8ˆàŸ,™,Ø'Ø/Ø!5Ø#ð	 'ó 
ˆð ,¨AÑ.ÐØ)ª!¨Q²¨'Ñ2ˆØ×+Ñ+¨MÓ:ˆáØ% }Ð5¸ÈÈÐ8KÑKÐKä)Ø/Ø'Ø)×7Ñ7Ø&×1Ñ1ô	
ð 	
r*   )NNNNF)rE   rF   rG   r   r]   r   ÚALTCLIP_VISION_INPUTS_DOCSTRINGr   r   r   r%   rI   rÏ   r   r   r�   r…   r†   s   @r(   rÂ  rÂ  m  s°   ø„ ðQÐ2õ Qñ +Ð+JÓKÙÐ+EÐTgÔhð 59Ø,0Ø/3Ø&*Ø38ñ+
à˜u×0Ñ0Ñ1ð+
ð $ D™>ð+
ð ' t™nð	+
ð
 ˜d‘^ð+
ð #+¨4¡.ð+
ð 
ˆuÐ0Ð0Ñ	1ò+
ó ió Lô+
r*   rÂ  c                   óÞ   ‡ — e Zd ZeZdZdefˆ fd„Zdej                  fd„Z	 e
e«       eee¬«      	 	 	 	 	 ddeej                      dee   dee   d	ed
ee   deeef   fd„«       «       Zˆ xZS )ÚAltCLIPVisionModelrœ  rr   c                 ód   •— t         ‰| �  |«       t        |«      | _        | j	                  «        y rÌ   )r\   r]   rÂ  Úvision_modelÚ	post_initrq   s     €r(   r]   zAltCLIPVisionModel.__init__¬  s'   ø€ Ü‰Ñ˜Ô Ü4°VÓ<ˆÔà�‰Õr*   r   c                 óB   — | j                   j                  j                  S rÌ   )rÑ  r€   rˆ  rD   s    r(   Úget_input_embeddingsz'AltCLIPVisionModel.get_input_embeddings²  s   € Ø× Ñ ×+Ñ+×;Ñ;Ð;r*   rÇ  r¨   r  r›  r  c                 ób   — |�|n| j                   j                  }| j                  |||||¬«      S )aÐ  
        Returns:

        Examples:

        ```python
        >>> from PIL import Image
        >>> import requests
        >>> from transformers import AutoProcessor, AltCLIPVisionModel

        >>> model = AltCLIPVisionModel.from_pretrained("BAAI/AltCLIP")
        >>> processor = AutoProcessor.from_pretrained("BAAI/AltCLIP")

        >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
        >>> image = Image.open(requests.get(url, stream=True).raw)

        >>> inputs = processor(images=image, return_tensors="pt")

        >>> outputs = model(**inputs)
        >>> last_hidden_state = outputs.last_hidden_state
        >>> pooled_output = outputs.pooler_output  # pooled CLS states
        ```©rœ  r¨   r  r›  r  )rr   rt  rÑ  )r@   rœ  r¨   r  r›  r  s         r(   r�   zAltCLIPVisionModel.forwardµ  sB   € ð@ &1Ð%<‘kÀ$Ç+Á+×B]ÑB]ˆà× Ñ Ø%Ø/Ø!5Ø%=Ø#ð !ó 
ð 	
r*   ©NNNFN)rE   rF   rG   r   r½  Úmain_input_namer]   r"   ÚModulerÔ  r   rÍ  r   r   r   r%   rI   rÏ   r   r   r�   r…   r†   s   @r(   rÏ  rÏ  ¨  sÄ   ø„ Ø&€LØ$€OðÐ2õ ð< b§i¡ió <ñ +Ð+JÓKÙÐ+EÐTgÔhð 59Ø,0Ø/3Ø).Ø&*ñ&
à˜u×0Ñ0Ñ1ð&
ð $ D™>ð&
ð ' t™nð	&
ð
 #'ð&
ð ˜d‘^ð&
ð 
ˆuÐ0Ð0Ñ	1ò&
ó ió Lô&
r*   rÏ  c                   óÂ  ‡ — e Zd ZdZeZdˆ fd„	Zd„ Zd„ Zd„ Z		 	 	 	 	 	 	 	 	 	 	 	 	 dde
ej                     de
ej                     de
ej                     d	e
ej                     d
e
ej                     de
ej                     de
ej                     de
ej                     de
eej                        de
e   de
e   de
e   de
e   deeej                     ef   fd„Zˆ xZS )ÚAltRobertaModela*  

    The model can behave as an encoder (with only self-attention) as well as a decoder, in which case a layer of
    cross-attention is added between the self-attention layers, following the architecture described in *Attention is
    all you need*_ by Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit, Llion Jones, Aidan N. Gomez, Lukasz
    Kaiser and Illia Polosukhin.

    To behave as an decoder the model needs to be initialized with the `is_decoder` argument of the configuration set
    to `True`. To be used in a Seq2Seq model, the model needs to initialized with both `is_decoder` argument and
    `add_cross_attention` set to `True`; an `encoder_hidden_states` is then expected as an input to the forward pass.

    .. _*Attention is all you need*: https://arxiv.org/abs/1706.03762

    c                 óº   •— t         ‰| �  |«       || _        t        |«      | _        t        |«      | _        |rt        |«      nd | _        | j                  «        y rÌ   )
r\   r]   rr   rM   r€   r  rÅ  r5  ÚpoolerrÒ  )r@   rr   Úadd_pooling_layerrs   s      €r(   r]   zAltRobertaModel.__init__ó  sL   ø€ Ü‰Ñ˜Ô ØˆŒä.¨vÓ6ˆŒÜ(¨Ó0ˆŒá2CÔ& vÔ.ÈˆŒð 	�‰Õr*   c                 ó.   — | j                   j                  S rÌ   ©r€   rb   rD   s    r(   rÔ  z$AltRobertaModel.get_input_embeddingsÿ  s   € Ø�‰×.Ñ.Ð.r*   c                 ó&   — || j                   _        y rÌ   rà  ©r@   r—   s     r(   Úset_input_embeddingsz$AltRobertaModel.set_input_embeddings  s   € Ø*/ˆ�‰Õ'r*   c                 ó˜   — |j                  «       D ]7  \  }}| j                  j                  |   j                  j	                  |«       Œ9 y)z�
        Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base
        class PreTrainedModel
        N)ÚitemsrÅ  r  r  rê   )r@   Úheads_to_pruner  rè   s       r(   Ú_prune_headszAltRobertaModel._prune_heads  sE   € ð
 +×0Ñ0Ó2ò 	C‰LˆE�5Ø�L‰L×Ñ˜uÑ%×/Ñ/×;Ñ;¸EÕBñ	Cr*   ry   r£   rY   rT   r¤   rz   r¥   r¦   r  r½   r¨   r  r  r   c                 óœ  — |�|n| j                   j                  }|�|n| j                   j                  }|�|n| j                   j                  }| j                   j                  r|
�|
n| j                   j
                  }
nd}
|�|�t        d«      ‚|�#| j                  ||«       |j                  «       }n!|�|j                  «       dd }nt        d«      ‚|\  }}|�|j                  n|j                  }|	�|	d   d   j                  d   nd}|€t        j                  |||z   f|¬«      }|€pt        | j                  d	«      r4| j                  j                  dd…d|…f   }|j!                  ||«      }|}n&t        j"                  |t        j$                  |¬
«      }| j'                  ||«      }| j                   j                  rE|�C|j                  «       \  }}}||f}|€t        j                  ||¬«      }| j)                  |«      }nd}| j+                  || j                   j,                  «      }| j                  |||||¬«      }| j/                  ||||||	|
|||¬«
      }|d   }| j0                  �| j1                  |«      nd}|s
||f|dd z   S t3        |||j4                  |j6                  |j8                  |j:                  ¬«      S )a  
        encoder_hidden_states  (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
            Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if
            the model is configured as a decoder.
        encoder_attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in
            the cross-attention if the model is configured as a decoder. Mask values selected in `[0, 1]`:

            - 1 for tokens that are **not masked**,
            - 0 for tokens that are **masked**.
        past_key_values (`tuple(tuple(torch.FloatTensor))` of length `config.n_layers` with each tuple having 4 tensors of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`):
            Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding.

            If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that
            don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all
            `decoder_input_ids` of shape `(batch_size, sequence_length)`.
        use_cache (`bool`, *optional*):
            If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see
            `past_key_values`).
        NFzDYou cannot specify both input_ids and inputs_embeds at the same timerV   z5You have to specify either input_ids or inputs_embedsr   rŽ   r    rY   ru   )ry   rT   rY   rz   r{   )	r£   r¤   r¥   r¦   r  r½   r¨   r  r  r   )r#  rË  r  r¢   r$  r%  )rr   r¨   r  rt  rš   r½   r�   Ú%warn_if_padding_and_no_attention_maskro   r!   r°   r%   Úonesrx   r€   rY   rm   rn   rp   Úget_extended_attention_maskÚinvert_attention_maskÚget_head_maskr  rÅ  rÝ  r   r  r¢   r$  r%  )r@   ry   r£   rY   rT   r¤   rz   r¥   r¦   r  r½   r¨   r  r  r|   r   r}   r!   r{   r~   r   Úextended_attention_maskÚencoder_batch_sizeÚencoder_sequence_lengthr  Úencoder_hidden_shapeÚencoder_extended_attention_maskÚembedding_outputrÌ  Úsequence_outputr;  s                                  r(   r�   zAltRobertaModel.forward  s  € ðH 2CÐ1NÑ-ÐTX×T_ÑT_×TqÑTqÐà$8Ð$DÑ È$Ï+É+×JjÑJjð 	ð &1Ð%<‘kÀ$Ç+Á+×B]ÑB]ˆà�;‰;×!Ò!Ø%.Ð%:™	ÀÇÁ×@UÑ@U‰IàˆIàÐ  ]Ð%>ÜÐcÓdÐdØÐ"Ø×6Ñ6°yÀ.ÔQØ#Ÿ.™.Ó*‰KØÐ&Ø'×,Ñ,Ó.¨s°Ð3‰KäÐTÓUÐUà!,Ñˆ
�JØ%.Ð%:�×!Ò!À×@TÑ@Tˆð DSÐC^ °Ñ!3°AÑ!6×!<Ñ!<¸QÒ!?ÐdeÐàÐ!Ü"ŸZ™Z¨*°jÐCYÑ6YÐ)ZÐdjÔkˆNàÐ!Ü�t—‘Ð(8Ô9Ø*.¯/©/×*HÑ*HÊÈKÈZÈKÈÑ*XÐ'Ø3J×3QÑ3QÐR\Ð^hÓ3iÐ0Ø!A‘ä!&§¡¨[ÄÇ
Á
ÐSYÔ!Z�ð 15×0PÑ0PÐQ_ÐalÓ0mÐð �;‰;×!Ò!Ð&;Ð&GØ=R×=WÑ=WÓ=YÑ:ÐÐ 7¸Ø$6Ð8OÐ#PÐ Ø%Ð-Ü).¯©Ð4HÐQWÔ)XÐ&Ø.2×.HÑ.HÐI_Ó.`Ñ+à.2Ð+ð ×&Ñ& y°$·+±+×2OÑ2OÓPˆ	àŸ?™?ØØ%Ø)Ø'Ø#9ð +ó 
Ðð Ÿ,™,ØØ2ØØ"7Ø#BØ+ØØ/Ø!5Ø#ð 'ó 
ˆð *¨!Ñ,ˆØ8<¿¹Ð8O˜Ÿ™ OÔ4ÐUYˆáØ# ]Ð3°oÀaÀbÐ6IÑIÐIä;Ø-Ø'Ø+×;Ñ;Ø)×7Ñ7Ø&×1Ñ1Ø,×=Ñ=ô
ð 	
r*   )T)NNNNNNNNNNNNN)rE   rF   rG   rH   r   r½  r]   rÔ  rã  rç  r   r%   rÎ   r   rI   rÏ   r   r   r   r�   r…   r†   s   @r(   rÛ  rÛ  à  sg  ø„ ñð %€Lõ
ò/ò0òCð -1Ø15Ø15Ø/3Ø,0Ø04Ø8<Ø9=Ø=AØ$(Ø,0Ø/3Ø&*ñ@
à˜EŸL™LÑ)ð@
ð ! §¡Ñ.ð@
ð ! §¡Ñ.ð	@
ð
 ˜uŸ|™|Ñ,ð@
ð ˜EŸL™LÑ)ð@
ð   §¡Ñ-ð@
ð  (¨¯©Ñ5ð@
ð !)¨¯©Ñ 6ð@
ð " $ u×'8Ñ'8Ñ"9Ñ:ð@
ð ˜D‘>ð@
ð $ D™>ð@
ð ' t™nð@
ð ˜d‘^ð@
ð 
ˆu�U—\‘\Ñ"Ð$PÐPÑ	Q÷@
r*   rÛ  c                   ó  ‡ — e Zd ZeZˆ fd„Zdej                  fd„Zdej                  ddfd„Z
ddee   dej                  fˆ fd„Z ee«       eee¬	«      	 	 	 	 	 	 	 	 	 	 	 dd
eej&                     deej&                     deej&                     deej&                     deej&                     deej&                     deej&                     deej&                     dee   dee   dee   deeef   fd„«       «       Zˆ xZS )ÚAltCLIPTextModelc                 ó&  •— t         ‰| �  |«       t        |d¬«      | _        t	        j
                  |j                  |j                  «      | _        t	        j                  |j                  |j                  ¬«      | _        | j                  «        y )NF)rÞ  rP   )r\   r]   rÛ  Úrobertar"   r”   r`   Úproject_dimÚtransformationrg   rh   Úpre_LNrÒ  rq   s     €r(   r]   zAltCLIPTextModel.__init__”  se   ø€ Ü‰Ñ˜Ô Ü& vÀÔGˆŒÜ Ÿi™i¨×(:Ñ(:¸F×<NÑ<NÓOˆÔÜ—l‘l 6×#5Ñ#5¸6×;PÑ;PÔQˆŒØ�‰Õr*   r   c                 óB   — | j                   j                  j                  S rÌ   ©rø  r€   rb   rD   s    r(   rÔ  z%AltCLIPTextModel.get_input_embeddings›  s   € Ø�|‰|×&Ñ&×6Ñ6Ð6r*   r—   Nc                 ó:   — || j                   j                  _        y rÌ   rý  râ  s     r(   rã  z%AltCLIPTextModel.set_input_embeddingsž  s   € Ø27ˆ�‰×ÑÕ/r*   Únew_num_tokensc                 ó"   •— t         ‰| �  |«      S rÌ   )r\   Úresize_token_embeddings)r@   rÿ  rs   s     €r(   r  z(AltCLIPTextModel.resize_token_embeddings¡  s   ø€ Ü‰wÑ.¨~Ó>Ð>r*   rÇ  ry   r£   rY   rT   r¤   rz   r¥   r¦   r¨   r  r  c                 ó,  — |
�|
n| j                   j                  }
| j                  |||||||||	||
¬«      }|d   }| j                  |«      }| j	                  |«      }|dd…df   }|
s
||f|dd z   S t        |||j                  |j                  ¬«      S )a=  
        Returns:

        Examples:

        ```python
        >>> from transformers import AutoProcessor, AltCLIPTextModel

        >>> model = AltCLIPTextModel.from_pretrained("BAAI/AltCLIP")
        >>> processor = AutoProcessor.from_pretrained("BAAI/AltCLIP")

        >>> texts = ["it's a cat", "it's a dog"]

        >>> inputs = processor(text=texts, padding=True, return_tensors="pt")

        >>> outputs = model(**inputs)
        >>> last_hidden_state = outputs.last_hidden_state
        >>> pooled_output = outputs.pooler_output  # pooled CLS states
        ```N)ry   r£   rY   rT   r¤   rz   r¥   r¦   r¨   r  r  r   rŽ   é   rÊ  )rr   rt  rø  rû  rú  r   r¢   r$  )r@   ry   r£   rY   rT   r¤   rz   r¥   r¦   r¨   r  r  rË   rô  Úprojection_staterË  s                   r(   r�   zAltCLIPTextModel.forward¤  sÉ   € ðH &1Ð%<‘kÀ$Ç+Á+×B]ÑB]ˆà—,‘,ØØ)Ø)Ø%ØØ'Ø"7Ø#9Ø/Ø!5Ø#ð ó 
ˆð " !™*ˆð Ÿ+™+ oÓ6ˆð  ×.Ñ.¨Ó?ÐØ(ª¨A¨Ñ.ˆáØ$ mÐ4°w¸qÀ°|ÑCÐCä6Ø.Ø'Ø!×/Ñ/Ø×)Ñ)ô	
ð 	
r*   rÌ   )NNNNNNNNNNN)rE   rF   rG   r   r½  r]   r"   rÙ  rÔ  r^   rã  r   r‘   r  r   ÚALTCLIP_TEXT_INPUTS_DOCSTRINGr   r   r%   rÎ   rÏ   r   r   r�   r…   r†   s   @r(   rö  rö  ‘  s‚  ø„ Ø$€Lôð7 b§i¡ió 7ð8¨"¯,©,ð 8¸4ó 8ñ?°h¸s±mð ?ÈrÏ|É|õ ?ñ +Ð+HÓIÙÐ+RÐarÔsð -1Ø15Ø15Ø/3Ø,0Ø04Ø8<Ø9=Ø,0Ø&*Ø/3ñD
à˜EŸL™LÑ)ðD
ð ! §¡Ñ.ðD
ð ! §¡Ñ.ð	D
ð
 ˜uŸ|™|Ñ,ðD
ð ˜EŸL™LÑ)ðD
ð   §¡Ñ-ðD
ð  (¨¯©Ñ5ðD
ð !)¨¯©Ñ 6ðD
ð $ D™>ðD
ð ˜d‘^ðD
ð ' t™nðD
ð 
ˆuÐ=Ð=Ñ	>òD
ó tó JôD
r*   rö  c                   ó†  ‡ — e Zd ZeZdefˆ fd„Z ee«      	 	 	 	 	 	 	 ddee	j                     dee	j                     dee	j                     dee   dee   dee   d	e	j                  fd
„«       Z ee«      	 	 	 	 	 ddee	j                     dee   dee   dedee   d	e	j                  fd„«       Z ee«       eee¬«      	 	 	 	 	 	 	 	 	 	 ddee	j&                     dee	j                     dee	j                     dee	j&                     dee	j                     dee   dee   dee   dedee   d	eeef   fd„«       «       Zˆ xZS )r®  rr   c                 óP  •— t         ‰| �  |«       t        |j                  t        «      s"t        dt        |j                  «      › d�«      ‚t        |j                  t        «      s"t        dt        |j                  «      › d�«      ‚|j                  }|j                  }|j                  | _	        |j                  | _        |j                  | _        t        |«      | _        t!        |«      | _        t%        j&                  | j                  | j                  d¬«      | _        t%        j&                  | j                  | j                  d¬«      | _        t%        j,                  t/        j0                  | j2                  j4                  «      «      | _        | j9                  «        y )NzRconfig.vision_config is expected to be of type AltCLIPVisionConfig but is of type ú.zNconfig.text_config is expected to be of type AltCLIPTextConfig but is of type F)r€  )r\   r]   rò   Úvision_configr   Ú	TypeErrorÚtypeÚtext_configr   Úprojection_dimrù  r°  r`   r³  rö  Ú
text_modelrÂ  rÑ  r"   r”   r²  r¯  rƒ  r%   r±   rr   Úlogit_scale_init_valueÚlogit_scalerÒ  )r@   rr   r  r	  rs   s       €r(   r]   zAltCLIPModel.__init__ð  sW  ø€ Ü‰Ñ˜Ô ä˜&×.Ñ.Ô0CÔDÜðÜ˜×-Ñ-Ó.Ð/¨qð2óð ô ˜&×,Ñ,Ô.?Ô@ÜðÜ˜×+Ñ+Ó,Ð-¨Qð0óð ð
 ×(Ñ(ˆØ×,Ñ,ˆà$×3Ñ3ˆÔØ)×5Ñ5ˆÔØ -× 9Ñ 9ˆÔä*¨;Ó7ˆŒÜ4°]ÓCˆÔä!#§¡¨4×+@Ñ+@À$×BUÑBUÐ\aÔ!bˆÔÜ!Ÿy™y¨×)<Ñ)<¸d×>QÑ>QÐX]Ô^ˆÔÜŸ<™<¬¯©°T·[±[×5WÑ5WÓ(XÓYˆÔð 	�‰Õr*   ry   r£   rT   r¨   r  r  r   c           	      óþ   — |�|n| j                   j                  }|�|n| j                   j                  }|�|n| j                   j                  }| j	                  |||||||¬«      }|d   }	| j                  |	«      }
|
S )a‡  
        Returns:
            text_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The text embeddings obtained by
            applying the projection layer to the pooled output of [`AltCLIPTextModel`].

        Examples:

        ```python
        >>> from transformers import AutoProcessor, AltCLIPModel

        >>> model = AltCLIPModel.from_pretrained("BAAI/AltCLIP")
        >>> processor = AutoProcessor.from_pretrained("BAAI/AltCLIP")
        >>> inputs = processor(text=["a photo of a cat", "a photo of a dog"], padding=True, return_tensors="pt")
        >>> text_features = model.get_text_features(**inputs)
        ```)ry   r£   rT   rY   r¨   r  r  r   )rr   r¨   r  rt  r  r¯  )r@   ry   r£   rT   rY   r¨   r  r  Útext_outputsr;  Útext_featuress              r(   Úget_text_featureszAltCLIPModel.get_text_features  s›   € ð6 2CÐ1NÑ-ÐTX×T_ÑT_×TqÑTqÐà$8Ð$DÑ È$Ï+É+×JjÑJjð 	ð &1Ð%<‘kÀ$Ç+Á+×B]ÑB]ˆà—‘ØØ)Ø%Ø)Ø/Ø!5Ø#ð 'ó 
ˆð % Q™ˆØ×,Ñ,¨]Ó;ˆàÐr*   rœ  r›  c                 óú   — |�|n| j                   j                  }|�|n| j                   j                  }|�|n| j                   j                  }| j	                  |||||¬«      }|d   }| j                  |«      }|S )a*  
        Returns:
            image_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The image embeddings obtained by
            applying the projection layer to the pooled output of [`AltCLIPVisionModel`].

        Examples:

        ```python
        >>> from PIL import Image
        >>> import requests
        >>> from transformers import AutoProcessor, AltCLIPModel

        >>> model = AltCLIPModel.from_pretrained("BAAI/AltCLIP")
        >>> processor = AutoProcessor.from_pretrained("BAAI/AltCLIP")
        >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
        >>> image = Image.open(requests.get(url, stream=True).raw)
        >>> inputs = processor(images=image, return_tensors="pt")
        >>> image_features = model.get_image_features(**inputs)
        ```rÖ  r   )rr   r¨   r  rt  rÑ  r²  )	r@   rœ  r¨   r  r›  r  Úvision_outputsr;  Úimage_featuress	            r(   Úget_image_featureszAltCLIPModel.get_image_features>  s˜   € ð: 2CÐ1NÑ-ÐTX×T_ÑT_×TqÑTqÐà$8Ð$DÑ È$Ï+É+×JjÑJjð 	ð &1Ð%<‘kÀ$Ç+Á+×B]ÑB]ˆà×*Ñ*Ø%Ø/Ø!5Ø%=Ø#ð +ó 
ˆð ' qÑ)ˆØ×/Ñ/°Ó>ˆàÐr*   rÇ  rY   Úreturn_lossc           	      ó²  — |�|n| j                   j                  }|�|n| j                   j                  }|
�|
n| j                   j                  }
| j	                  |||||||
¬«      }| j                  ||||	|
¬«      }|d   }| j                  |«      }|d   }| j                  |«      }||j                  ddd¬«      z  }||j                  ddd¬«      z  }| j                  j                  «       }t        j                  ||j                  «       «      |z  }|j                  }d}|rt        |«      }|
s||||||f}|�|f|z   S |S t!        |||||||¬	«      S )
al  
        Returns:

        Examples:

        ```python
        >>> from PIL import Image
        >>> import requests
        >>> from transformers import AutoProcessor, AltCLIPModel

        >>> model = AltCLIPModel.from_pretrained("BAAI/AltCLIP")
        >>> processor = AutoProcessor.from_pretrained("BAAI/AltCLIP")
        >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
        >>> image = Image.open(requests.get(url, stream=True).raw)
        >>> inputs = processor(
        ...     text=["a photo of a cat", "a photo of a dog"], images=image, return_tensors="pt", padding=True
        ... )
        >>> outputs = model(**inputs)
        >>> logits_per_image = outputs.logits_per_image  # this is the image-text similarity score
        >>> probs = logits_per_image.softmax(dim=1)  # we can take the softmax to get the label probabilities
        ```N)ry   r£   rY   rT   r¨   r  r  rÖ  r   rŽ   rV   T)rP  r«   Úkeepdim)r3   r4   r5   r6   r7   r8   r9   )rr   r¨   r  rt  r  rÑ  r²  r¯  Únormr  Úexpr%   r®   r-   ÚTr0   r2   )r@   ry   rœ  r£   rT   rY   r  r¨   r  r›  r  r  r  r7   r6   r  r5   r4   r3   rã   s                       r(   r�   zAltCLIPModel.forwardn  s°  € ðJ 2CÐ1NÑ-ÐTX×T_ÑT_×TqÑTqÐà$8Ð$DÑ È$Ï+É+×JjÑJjð 	ð &1Ð%<‘kÀ$Ç+Á+×B]ÑB]ˆà—‘ØØ)Ø)Ø%Ø/Ø!5Ø#ð 'ó 
ˆð ×*Ñ*Ø%Ø/Ø!5Ø%=Ø#ð +ó 
ˆð & aÑ(ˆØ×-Ñ-¨lÓ;ˆà" 1‘oˆØ×*Ñ*¨;Ó7ˆð $ l×&7Ñ&7¸!ÀÈTÐ&7Ó&RÑRˆØ! K×$4Ñ$4°q¸bÈ$Ð$4Ó$OÑOˆð ×&Ñ&×*Ñ*Ó,ˆÜŸ,™, {°L·N±NÓ4DÓEÈÑSˆØ*×,Ñ,ÐàˆÙÜ˜_Ó-ˆDáØ&¨¸ÀlÐT`ÐbpÐqˆFØ)-Ð)9�T�G˜fÑ$ÐE¸vÐEäØØ-Ø+Ø#Ø%Ø*Ø .ô
ð 	
r*   )NNNNNNNr×  )
NNNNNNNNFN)rE   rF   rG   r   r½  r]   r   r  r   r%   rÎ   rÏ   rI   r  rÍ  r  ÚALTCLIP_INPUTS_DOCSTRINGr   r2   Ú
LongTensorr   r   r�   r…   r†   s   @r(   r®  r®  í  sM  ø„ Ø €Lð˜}õ ñ> +Ð+HÓIð -1Ø15Ø/3ØØ,0Ø/3Ø&*ñ,à˜EŸL™LÑ)ð,ð ! §¡Ñ.ð,ð ˜uŸ|™|Ñ,ð	,ð $ D™>ð,ð ' t™nð,ð ˜d‘^ð,ð 
×	Ñ	ò,ó Jð,ñ\ +Ð+JÓKð 59Ø,0Ø/3Ø).Ø&*ñ-à˜u×0Ñ0Ñ1ð-ð $ D™>ð-ð ' t™nð	-ð
 #'ð-ð ˜d‘^ð-ð 
×	Ñ	ò-ó Lð-ñ^ +Ð+CÓDÙ¨=À}ÔUð 15Ø48Ø15Ø37Ø15Ø&*Ø,0Ø/3Ø).Ø&*ñZ
à˜E×,Ñ,Ñ-ðZ
ð ˜u×0Ñ0Ñ1ðZ
ð ! §¡Ñ.ð	Z
ð
 ˜u×/Ñ/Ñ0ðZ
ð ! §¡Ñ.ðZ
ð ˜d‘^ðZ
ð $ D™>ðZ
ð ' t™nðZ
ð #'ðZ
ð ˜d‘^ðZ
ð 
ˆu�mÐ#Ñ	$òZ
ó Vó EôZ
r*   r®  c                 ó¾   — | j                  |«      j                  «       }t        j                  |d¬«      j	                  |«      |z   |z  }|j                  «       |z   S )a  
    Replace non-padding symbols with their position numbers. Position numbers begin at padding_idx+1. Padding symbols
    are ignored. This is modified from fairseq's `utils.make_positions`.

    Args:
        x: torch.Tensor x:

    Returns: torch.Tensor
    r   rª   )Úner‘   r%   ÚcumsumÚtype_asrp   )ry   rO   r{   ÚmaskÚincremental_indicess        r(   rv   rv   Î  sW   € ð �<‰<˜Ó$×(Ñ(Ó*€DÜ Ÿ<™<¨°!Ô4×<Ñ<¸TÓBÐE[Ñ[Ð_cÑcÐØ×#Ñ#Ó%¨Ñ3Ð3r*   )r¥  rÏ  rö  r®  )r   )KrH   r´   Údataclassesr   Útypingr   r   r   r   r   r%   Útorch.nnr"   Útorch.utils.checkpointÚactivationsr
   Úmodeling_outputsr   r   r   r   r   Úmodeling_utilsr   Úpytorch_utilsr   r   r   Úutilsr   r   r   r   r   Úconfiguration_altclipr   r   r   Ú
get_loggerrE   r'  Ú_CHECKPOINT_FOR_DOCÚ_CONFIG_FOR_DOCÚALTCLIP_START_DOCSTRINGr  rÍ  r  rÎ   r)   r0   r2   rÙ  rM   rˆ   rÑ   rá   rÞ   rï   rù   rý   r  r5  r=  r^  re  ro  rz  r¥  rÂ  rÏ  rÛ  rö  r®  rv   Ú__all__rK   r*   r(   ú<module>r6     sI  ðñ ã Ý !ß 4Õ 4ã Ý Û å !÷õ õ .ß lÑ lß vÕ vß XÑ Xð 
ˆ×	Ñ	˜HÓ	%€à$Ð Ø!€ðÐ ð!Ð ð@#Ð ð"%Ð ðT`˜UŸ\™\ð `¨e¯l©ló `ð-˜%Ÿ,™,ð -¨5¯<©<ó -ð ô!
�Kó !
ó ð!
ôJV=˜2Ÿ9™9ô V=ôtC˜bŸi™iô CôN˜2Ÿ9™9ô ð Ð$ð&Ð "ô0˜"Ÿ)™)ô 0ôh˜RŸY™Yô ô �r—y‘yô ôS�b—i‘iô SônZ
˜Ÿ	™	ô Z
ô|�r—y‘yô ô e2�r—y‘yô e2ôR�—‘ô ô/˜"Ÿ)™)ô /ôd^
�R—Y‘Yô ^
ôDP˜bŸi™iô Pôf6?˜_ô 6?ôr8
˜rŸy™yô 8
ôv5
Ð/ô 5
ôpn
Ð,ô n
ôbY
Ð-ô Y
ôx]
Ð)ô ]
óB4ò  _�r*   