Ë
    S^(hu ã                   ót  — d Z ddlZddlmZ ddlmZmZmZmZ ddl	Z	ddl
Z	ddl	mZ ddlmZ ddlmZmZ dd	lmZ dd
lmZ ddlmZ ddlmZmZmZmZ ddlmZ ddlmZm Z m!Z! ddl"m#Z#m$Z$m%Z%m&Z&m'Z' ddl(m)Z)m*Z*  e%jV                  e,«      Z-dZ.dZ/e G d„ de«      «       Z0 G d„ dejb                  «      Z2 G d„ dejb                  «      Z3 G d„ dejb                  «      Z4de3iZ5 G d„ dejb                  «      Z6 G d„ d ejb                  «      Z7 G d!„ d"ejb                  «      Z8 G d#„ d$ejb                  «      Z9 G d%„ d&ejb                  «      Z: G d'„ d(e«      Z;d)Z<d*Z= G d+„ d,ejb                  «      Z> G d-„ d.ejb                  «      Z? G d/„ d0ejb                  «      Z@ G d1„ d2ejb                  «      ZA G d3„ d4ejb                  «      ZBd5ZC G d6„ d7ejb                  «      ZD e#d8e<«       G d9„ d:e;«      «       ZE G d;„ d<ejb                  «      ZF e#d=e<«       G d>„ d?e;«      «       ZG e#d@e<«       G dA„ dBe;e«      «       ZHg dC¢ZIy)DzPyTorch GIT model.é    N)Ú	dataclass)ÚListÚOptionalÚTupleÚUnion)Únné   )ÚACT2FN)ÚCacheÚDynamicCache)ÚModelOutput)ÚGenerationMixin)Ú_prepare_4d_attention_mask)ÚBaseModelOutputÚBaseModelOutputWithPastÚBaseModelOutputWithPoolingÚCausalLMOutputWithPast)ÚPreTrainedModel)Úapply_chunking_to_forwardÚ find_pruneable_heads_and_indicesÚprune_linear_layer)Úadd_start_docstringsÚ%add_start_docstrings_to_model_forwardÚloggingÚreplace_return_docstringsÚ	torch_inté   )Ú	GitConfigÚGitVisionConfigzmicrosoft/git-baser   c                   óÆ   — e Zd ZU dZdZeej                     ed<   dZ	eej                     ed<   dZ
eeej                  df      ed<   dZeeej                  df      ed<   y)ÚGitVisionModelOutputaÝ  
    Base class for vision model's outputs that also contains image embeddings of the pooling of the last hidden states.

    Args:
        image_embeds (`torch.FloatTensor` of shape `(batch_size, output_dim)` *optional* returned when model is initialized with `with_projection=True`):
            The image embeddings obtained by applying the projection layer to the pooler_output.
        last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
            Sequence of hidden-states at the output of the last layer of the model.
        hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
            Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, +
            one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`.

            Hidden-states of the model at the output of each layer plus the optional initial embedding outputs.
        attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
            Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
            sequence_length)`.

            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
            heads.
    NÚimage_embedsÚlast_hidden_state.Úhidden_statesÚ
attentions)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r"   r   ÚtorchÚFloatTensorÚ__annotations__r#   r$   r   r%   © ó    úb/var/www/skyplay_api_hub/venv/lib/python3.12/site-packages/transformers/models/git/modeling_git.pyr!   r!   7   sr   … ñð* 15€L�(˜5×,Ñ,Ñ-Ó4Ø59Ð�x × 1Ñ 1Ñ2Ó9Ø=A€M�8˜E %×"3Ñ"3°SÐ"8Ñ9Ñ:ÓAØ:>€J�˜˜u×0Ñ0°#Ð5Ñ6Ñ7Ô>r.   r!   c                   óª   ‡ — e Zd ZdZˆ fd„Z	 	 	 	 d	deej                     deej                     deej                     de	dej                  f
d„Zˆ xZS )
ÚGitEmbeddingsz;Construct the embeddings from word and position embeddings.c                 óB  •— t         ‰| �  «        t        j                  |j                  |j
                  |j                  ¬«      | _        t        j                  |j                  |j
                  «      | _	        t        j                  |j
                  |j                  ¬«      | _
        t        j                  |j                  «      | _        t        |dd«      | _        | j#                  dt%        j&                  |j                  «      j)                  d«      d¬«       y )	N)Úpadding_idx©ÚepsÚposition_embedding_typeÚabsoluteÚposition_ids©r   éÿÿÿÿF©Ú
persistent)ÚsuperÚ__init__r   Ú	EmbeddingÚ
vocab_sizeÚhidden_sizeÚpad_token_idÚword_embeddingsÚmax_position_embeddingsÚposition_embeddingsÚ	LayerNormÚlayer_norm_epsÚDropoutÚhidden_dropout_probÚdropoutÚgetattrr6   Úregister_bufferr*   ÚarangeÚexpand©ÚselfÚconfigÚ	__class__s     €r/   r>   zGitEmbeddings.__init__X   sÕ   ø€ Ü‰ÑÔÜ!Ÿ|™|¨F×,=Ñ,=¸v×?QÑ?QÐ_e×_rÑ_rÔsˆÔÜ#%§<¡<°×0NÑ0NÐPV×PbÑPbÓ#cˆÔ ô Ÿ™ f×&8Ñ&8¸f×>SÑ>SÔTˆŒÜ—z‘z &×"<Ñ"<Ó=ˆŒä'.¨vÐ7PÐR\Ó']ˆÔ$Ø×ÑØœEŸL™L¨×)GÑ)GÓH×OÑOÐPWÓXÐejð 	õ 	
r.   Ú	input_idsr8   Úinputs_embedsÚpast_key_values_lengthÚreturnc                 óJ  — |�|j                  «       }n|j                  «       d d }|d   }|€| j                  d d …|||z   …f   }|€| j                  |«      }n|}| j                  dk(  r| j	                  |«      }||z  }| j                  |«      }| j                  |«      }|S )Nr:   r   r7   )Úsizer8   rC   r6   rE   rF   rJ   )	rP   rS   r8   rT   rU   Úinput_shapeÚ
seq_lengthÚ
embeddingsrE   s	            r/   ÚforwardzGitEmbeddings.forwardg   sÂ   € ð Ð Ø#Ÿ.™.Ó*‰Kà'×,Ñ,Ó.¨s°Ð3ˆKà  ‘^ˆ
àÐØ×,Ñ,ªQÐ0FÈÐVlÑIlÐ0lÐ-lÑmˆLàÐ Ø×-Ñ-¨iÓ8‰Jà&ˆJà×'Ñ'¨:Ò5Ø"&×":Ñ":¸<Ó"HÐØÐ-Ñ-ˆJØ—^‘^ JÓ/ˆ
Ø—\‘\ *Ó-ˆ
ØÐr.   )NNNr   )r&   r'   r(   r)   r>   r   r*   Ú
LongTensorr+   ÚintÚTensorr\   Ú__classcell__©rR   s   @r/   r1   r1   U   ss   ø„ ÙEô
ð" 15Ø37Ø59Ø&'ñà˜E×,Ñ,Ñ-ðð ˜u×/Ñ/Ñ0ðð   × 1Ñ 1Ñ2ð	ð
 !$ðð 
�‰÷r.   r1   c                   óü   ‡ — e Zd Zdˆ fd„	Zdej
                  dej
                  fd„Z	 	 	 	 	 ddej
                  deej                     deej                     dee	   d	ee
   d
ee
   deej
                     fd„Zˆ xZS )ÚGitSelfAttentionc                 ó  •— t         ‰| �  «        |j                  |j                  z  dk7  r2t	        |d«      s&t        d|j                  › d|j                  › d�«      ‚|| _        |€-t        j                  d| j                  j                  › d�«       |j                  | _        t        |j                  |j                  z  «      | _        | j                  | j                  z  | _        t        |j                  j                  |j                  j                   z  dz  d	z   «      | _        |j$                  �| xj"                  |j$                  z  c_        t'        j(                  |j                  | j                  «      | _        t'        j(                  |j                  | j                  «      | _        t'        j(                  |j                  | j                  «      | _        t'        j0                  |j2                  «      | _        |xs t7        |d
d«      | _        | j8                  dk(  s| j8                  dk(  rG|j:                  | _        t'        j<                  d|j:                  z  d	z
  | j                  «      | _        y y )Nr   Úembedding_sizezThe hidden size (z6) is not a multiple of the number of attention heads (ú)zInstantiating z¹ without passing a `layer_idx` is not recommended and will lead to errors during the forward call if caching is used. Please make sure to provide a `layer_idx` when creating this class.é   r   r6   r7   Úrelative_keyÚrelative_key_query) r=   r>   rA   Únum_attention_headsÚhasattrÚ
ValueErrorÚ	layer_idxÚloggerÚwarning_oncerR   r&   r^   Úattention_head_sizeÚall_head_sizeÚvision_configÚ
image_sizeÚ
patch_sizeÚimage_patch_tokensÚnum_image_with_embeddingr   ÚLinearÚqueryÚkeyÚvaluerH   Úattention_probs_dropout_probrJ   rK   r6   rD   r?   Údistance_embedding©rP   rQ   r6   rm   rR   s       €r/   r>   zGitSelfAttention.__init__†   s(  ø€ Ü‰ÑÔØ×Ñ × :Ñ :Ñ:¸aÒ?ÌÐPVÐXhÔHiÜØ# F×$6Ñ$6Ð#7ð 8Ø ×4Ñ4Ð5°Qð8óð ð #ˆŒØÐÜ×ÑØ  §¡×!8Ñ!8Ð 9ð :,ð ,ôð $*×#=Ñ#=ˆÔ Ü#& v×'9Ñ'9¸F×<VÑ<VÑ'VÓ#WˆÔ Ø!×5Ñ5¸×8PÑ8PÑPˆÔÜ"% v×';Ñ';×'FÑ'FÈ×I]ÑI]×IhÑIhÑ'hÐmnÑ&nÐqrÑ&rÓ"sˆÔØ×*Ñ*Ð6Ø×#Ò# v×'FÑ'FÑFÕ#ä—Y‘Y˜v×1Ñ1°4×3EÑ3EÓFˆŒ
Ü—9‘9˜V×/Ñ/°×1CÑ1CÓDˆŒÜ—Y‘Y˜v×1Ñ1°4×3EÑ3EÓFˆŒ
ä—z‘z &×"EÑ"EÓFˆŒØ'>ò (
Ä'ØÐ-¨zóC
ˆÔ$ð ×'Ñ'¨>Ò9¸T×=YÑ=YÐ]qÒ=qØ+1×+IÑ+IˆDÔ(Ü&(§l¡l°1°v×7UÑ7UÑ3UÐXYÑ3YÐ[_×[sÑ[sÓ&tˆDÕ#ð >rr.   ÚxrV   c                 ó¤   — |j                  «       d d | j                  | j                  fz   }|j                  |«      }|j	                  dddd«      S )Nr:   r   rg   r   r	   )rX   rj   rp   ÚviewÚpermute)rP   r~   Únew_x_shapes      r/   Útranspose_for_scoresz%GitSelfAttention.transpose_for_scores¨   sL   € Ø—f‘f“h˜s �m t×'?Ñ'?À×AYÑAYÐ&ZÑZˆØ�F‰F�;ÓˆØ�y‰y˜˜A˜q !Ó$Ð$r.   r$   Úattention_maskÚ	head_maskÚpast_key_valueÚoutput_attentionsÚpixel_values_presentc           	      óx  — | j                  |«      }|r| j                  nd}| j                  | j                  |«      «      }	| j                  | j	                  |«      «      }
|�Ž|j                  |	d d …d d …|d …d d …f   |
d d …d d …|d …d d …f   | j                  «      \  }}t        j                  |	d d …d d …d |…d d …f   |gd¬«      }	t        j                  |
d d …d d …d |…d d …f   |gd¬«      }
| j                  |«      }t        j                  ||	j                  dd«      «      }| j                  dk(  s| j                  dk(  �r—|j                  d   |	j                  d   }}|�Dt        j                  |dz
  t        j                  |j                  ¬	«      j!                  dd«      }n@t        j"                  |t        j                  |j                  ¬	«      j!                  dd«      }t        j"                  |t        j                  |j                  ¬	«      j!                  dd«      }||z
  }| j%                  || j&                  z   dz
  «      }|j)                  |j*                  ¬
«      }| j                  dk(  rt        j,                  d||«      }||z   }nE| j                  dk(  r6t        j,                  d||«      }t        j,                  d|	|«      }||z   |z   }|t/        j0                  | j2                  «      z  }|�||z   }t4        j6                  j9                  |d¬«      }| j;                  |«      }|�||z  }t        j                  ||
«      }|j=                  dddd«      j?                  «       }|jA                  «       d d | jB                  fz   }|j!                  |«      }|r||fn|f}||fz   }|S )Nr   rg   ©Údimr:   éþÿÿÿrh   ri   r   ©ÚdtypeÚdevice©rŽ   zbhld,lrd->bhlrzbhrd,lrd->bhlrr	   )"rx   ru   rƒ   ry   rz   Úupdaterm   r*   ÚcatÚmatmulÚ	transposer6   ÚshapeÚtensorÚlongr�   r€   rM   r|   rD   ÚtorŽ   ÚeinsumÚmathÚsqrtrp   r   Ú
functionalÚsoftmaxrJ   r�   Ú
contiguousrX   rq   )rP   r$   r„   r…   r†   r‡   rˆ   Úmixed_query_layerÚcutoffÚ	key_layerÚvalue_layerÚkey_layer_pastÚvalue_layer_pastÚquery_layerÚattention_scoresÚquery_lengthÚ
key_lengthÚposition_ids_lÚposition_ids_rÚdistanceÚpositional_embeddingÚrelative_position_scoresÚrelative_position_scores_queryÚrelative_position_scores_keyÚattention_probsÚcontext_layerÚnew_context_layer_shapeÚoutputss                               r/   r\   zGitSelfAttention.forward­   s™  € ð !ŸJ™J }Ó5Ðá,@�×(Ò(ÀaˆØ×-Ñ-¨d¯h©h°}Ó.EÓFˆ	Ø×/Ñ/°·
±
¸=Ó0IÓJˆØÐ%à/=×/DÑ/DØš!šQ ¡ªÐ*Ñ+¨[ººA¸v¹wÊÐ9IÑ-JÈDÏNÉNó0Ñ,ˆNÐ,ô Ÿ	™	 9ªQ²°7°F°7ºAÐ-=Ñ#>ÀÐ"OÐUVÔWˆIÜŸ)™) [²²A°w¸°wÂÐ1AÑ%BÐDTÐ$UÐ[\Ô]ˆKà×/Ñ/Ð0AÓBˆô !Ÿ<™<¨°Y×5HÑ5HÈÈRÓ5PÓQÐà×'Ñ'¨>Ò9¸T×=YÑ=YÐ]qÓ=qØ'2×'8Ñ'8¸Ñ';¸Y¿_¹_ÈQÑ=O˜*ˆLØÐ)Ü!&§¡¨j¸1©nÄEÇJÁJÐWd×WkÑWkÔ!l×!qÑ!qØ˜ó"‘ô "'§¡¨lÄ%Ç*Á*ÐUb×UiÑUiÔ!j×!oÑ!oÐprÐtuÓ!v�Ü"Ÿ\™\¨*¼E¿J¹JÈ}×OcÑOcÔd×iÑiÐjkÐmoÓpˆNØ%¨Ñ6ˆHà#'×#:Ñ#:¸8Àd×FbÑFbÑ;bÐefÑ;fÓ#gÐ Ø#7×#:Ñ#:À×ARÑARÐ#:Ó#SÐ à×+Ñ+¨~Ò=Ü+0¯<©<Ð8HÈ+ÐWkÓ+lÐ(Ø#3Ð6NÑ#NÑ Ø×-Ñ-Ð1EÒEÜ16·±Ð>NÐP[Ð]qÓ1rÐ.Ü/4¯|©|Ð<LÈiÐYmÓ/nÐ,Ø#3Ð6TÑ#TÐWsÑ#sÐ à+¬d¯i©i¸×8PÑ8PÓ.QÑQÐØÐ%à/°.Ñ@Ðô Ÿ-™-×/Ñ/Ð0@ÀbÐ/ÓIˆð Ÿ,™, Ó7ˆð Ð Ø-°	Ñ9ˆOäŸ™ _°kÓBˆà%×-Ñ-¨a°°A°qÓ9×DÑDÓFˆØ"/×"4Ñ"4Ó"6°s¸Ð";¸t×?QÑ?QÐ>SÑ"SÐØ%×*Ñ*Ð+BÓCˆá6G�= /Ñ2ÈmÐM]ˆà˜^Ð-Ñ-ˆØˆr.   ©NN©NNNFF)r&   r'   r(   r>   r*   r_   rƒ   r   r+   r   Úboolr   r\   r`   ra   s   @r/   rc   rc   …   s¶   ø„ õ uðD% e§l¡lð %°u·|±|ó %ð 7;Ø15Ø*.Ø,1Ø/4ñJà—|‘|ðJð ! ×!2Ñ!2Ñ3ðJð ˜E×-Ñ-Ñ.ð	Jð
 ! ™ðJð $ D™>ðJð ' t™nðJð 
ˆu�|‰|Ñ	÷Jr.   rc   c                   ón   ‡ — e Zd Zˆ fd„Zdej
                  dej
                  dej
                  fd„Zˆ xZS )ÚGitSelfOutputc                 ó(  •— t         ‰| �  «        t        j                  |j                  |j                  «      | _        t        j                  |j                  |j                  ¬«      | _        t        j                  |j                  «      | _
        y ©Nr4   )r=   r>   r   rw   rA   ÚdenserF   rG   rH   rI   rJ   rO   s     €r/   r>   zGitSelfOutput.__init__ü   s`   ø€ Ü‰ÑÔÜ—Y‘Y˜v×1Ñ1°6×3EÑ3EÓFˆŒ
ÜŸ™ f×&8Ñ&8¸f×>SÑ>SÔTˆŒÜ—z‘z &×"<Ñ"<Ó=ˆ�r.   r$   Úinput_tensorrV   c                 ór   — | j                  |«      }| j                  |«      }| j                  ||z   «      }|S ©N©r»   rJ   rF   ©rP   r$   r¼   s      r/   r\   zGitSelfOutput.forward  ó7   € ØŸ
™
 =Ó1ˆØŸ™ ]Ó3ˆØŸ™ }°|Ñ'CÓDˆØÐr.   ©r&   r'   r(   r>   r*   r_   r\   r`   ra   s   @r/   r¸   r¸   û   ó1   ø„ ô>ð U§\¡\ð ÀÇÁð ÐRW×R^ÑR^÷ r.   r¸   Úeagerc                   óÊ   ‡ — e Zd Zdˆ fd„	Zd„ Z	 	 	 	 	 ddej                  deej                     deej                     dee	   dee
   dee
   d	eej                     fd
„Zˆ xZS )ÚGitAttentionc                 ó    •— t         ‰| �  «        t        |j                     |||¬«      | _        t        |«      | _        t        «       | _        y )N)r6   rm   )	r=   r>   ÚGIT_SELF_ATTENTION_CLASSESÚ_attn_implementationrP   r¸   ÚoutputÚsetÚpruned_headsr}   s       €r/   r>   zGitAttention.__init__  sE   ø€ Ü‰ÑÔÜ.¨v×/JÑ/JÑKØÐ,CÈyô
ˆŒ	ô $ FÓ+ˆŒÜ›EˆÕr.   c                 ó>  — t        |«      dk(  ry t        || j                  j                  | j                  j                  | j
                  «      \  }}t        | j                  j                  |«      | j                  _        t        | j                  j                  |«      | j                  _        t        | j                  j                  |«      | j                  _	        t        | j                  j                  |d¬«      | j                  _        | j                  j                  t        |«      z
  | j                  _        | j                  j                  | j                  j                  z  | j                  _        | j
                  j                  |«      | _        y )Nr   r   rŠ   )Úlenr   rP   rj   rp   rÌ   r   rx   ry   rz   rÊ   r»   rq   Úunion)rP   ÚheadsÚindexs      r/   Úprune_headszGitAttention.prune_heads  s  € Üˆu‹:˜Š?ØÜ7Ø�4—9‘9×0Ñ0°$·)±)×2OÑ2OÐQU×QbÑQbó
‰ˆˆuô
 -¨T¯Y©Y¯_©_¸eÓDˆ�	‰	ŒÜ*¨4¯9©9¯=©=¸%Ó@ˆ�	‰	ŒÜ,¨T¯Y©Y¯_©_¸eÓDˆ�	‰	ŒÜ.¨t¯{©{×/@Ñ/@À%ÈQÔOˆ�‰Ôð )-¯	©	×(EÑ(EÌÈEË
Ñ(Rˆ�	‰	Ô%Ø"&§)¡)×"?Ñ"?À$Ç)Á)×B_ÑB_Ñ"_ˆ�	‰	ÔØ ×-Ñ-×3Ñ3°EÓ:ˆÕr.   r$   r„   r…   r†   r‡   rˆ   rV   c                 ón   — | j                  ||||||«      }| j                  |d   |«      }|f|dd  z   }	|	S )Nr   r   )rP   rÊ   )
rP   r$   r„   r…   r†   r‡   rˆ   Úself_outputsÚattention_outputr³   s
             r/   r\   zGitAttention.forward*  sT   € ð —y‘yØØØØØØ ó
ˆð  Ÿ;™; |°A¡¸ÓFÐØ#Ð%¨°Q°RÐ(8Ñ8ˆØˆr.   r´   rµ   )r&   r'   r(   r>   rÒ   r*   r_   r   r+   r   r¶   r   r\   r`   ra   s   @r/   rÆ   rÆ     s–   ø„ õ"ò;ð* 7;Ø15Ø*.Ø,1Ø/4ñà—|‘|ðð ! ×!2Ñ!2Ñ3ðð ˜E×-Ñ-Ñ.ð	ð
 ! ™ðð $ D™>ðð ' t™nðð 
ˆu�|‰|Ñ	÷r.   rÆ   c                   óV   ‡ — e Zd Zˆ fd„Zdej
                  dej
                  fd„Zˆ xZS )ÚGitIntermediatec                 ó  •— t         ‰| �  «        t        j                  |j                  |j
                  «      | _        t        |j                  t        «      rt        |j                     | _        y |j                  | _        y r¾   )r=   r>   r   rw   rA   Úintermediate_sizer»   Ú
isinstanceÚ
hidden_actÚstrr
   Úintermediate_act_fnrO   s     €r/   r>   zGitIntermediate.__init__B  s]   ø€ Ü‰ÑÔÜ—Y‘Y˜v×1Ñ1°6×3KÑ3KÓLˆŒ
Ü�f×'Ñ'¬Ô-Ü'-¨f×.?Ñ.?Ñ'@ˆDÕ$à'-×'8Ñ'8ˆDÕ$r.   r$   rV   c                 óJ   — | j                  |«      }| j                  |«      }|S r¾   )r»   rÝ   ©rP   r$   s     r/   r\   zGitIntermediate.forwardJ  s&   € ØŸ
™
 =Ó1ˆØ×0Ñ0°Ó?ˆØÐr.   rÂ   ra   s   @r/   r×   r×   A  s#   ø„ ô9ð U§\¡\ð °e·l±l÷ r.   r×   c                   ón   ‡ — e Zd Zˆ fd„Zdej
                  dej
                  dej
                  fd„Zˆ xZS )Ú	GitOutputc                 ó(  •— t         ‰| �  «        t        j                  |j                  |j
                  «      | _        t        j                  |j
                  |j                  ¬«      | _        t        j                  |j                  «      | _        y rº   )r=   r>   r   rw   rÙ   rA   r»   rF   rG   rH   rI   rJ   rO   s     €r/   r>   zGitOutput.__init__R  s`   ø€ Ü‰ÑÔÜ—Y‘Y˜v×7Ñ7¸×9KÑ9KÓLˆŒ
ÜŸ™ f×&8Ñ&8¸f×>SÑ>SÔTˆŒÜ—z‘z &×"<Ñ"<Ó=ˆ�r.   r$   r¼   rV   c                 ór   — | j                  |«      }| j                  |«      }| j                  ||z   «      }|S r¾   r¿   rÀ   s      r/   r\   zGitOutput.forwardX  rÁ   r.   rÂ   ra   s   @r/   rá   rá   Q  rÃ   r.   rá   c                   óÊ   ‡ — e Zd Zdˆ fd„	Z	 	 	 	 	 ddej
                  deej                     deej                     dee   dee	   dee	   de
ej
                     fd	„Zd
„ Zˆ xZS )ÚGitLayerc                 ó¶   •— t         ‰| �  «        |j                  | _        d| _        t	        ||¬«      | _        t        |«      | _        t        |«      | _	        y )Nr   )rm   )
r=   r>   Úchunk_size_feed_forwardÚseq_len_dimrÆ   Ú	attentionr×   Úintermediaterá   rÊ   )rP   rQ   rm   rR   s      €r/   r>   zGitLayer.__init__`  sK   ø€ Ü‰ÑÔØ'-×'EÑ'EˆÔ$ØˆÔÜ% f¸	ÔBˆŒÜ+¨FÓ3ˆÔÜ Ó'ˆ�r.   r$   r„   r…   r†   r‡   rˆ   rV   c                 óÂ   — | j                  ||||||¬«      }|d   }|dd }	|d   }
t        | j                  | j                  | j                  |«      }|f|	z   }	|	|
fz   }	|	S )N)r‡   r†   rˆ   r   r   r:   )ré   r   Úfeed_forward_chunkrç   rè   )rP   r$   r„   r…   r†   r‡   rˆ   Úself_attention_outputsrÕ   r³   Úpresent_key_valueÚlayer_outputs               r/   r\   zGitLayer.forwardh  s™   € ð "&§¡ØØØØ/Ø)Ø!5ð "0ó "
Ðð 2°!Ñ4Ðð )¨¨2Ð.ˆØ2°2Ñ6Ðä0Ø×#Ñ# T×%AÑ%AÀ4×CSÑCSÐUeó
ˆð  �/ GÑ+ˆð Ð.Ð0Ñ0ˆàˆr.   c                 óL   — | j                  |«      }| j                  ||«      }|S r¾   )rê   rÊ   )rP   rÕ   Úintermediate_outputrï   s       r/   rì   zGitLayer.feed_forward_chunkŠ  s,   € Ø"×/Ñ/Ð0@ÓAÐØ—{‘{Ð#6Ð8HÓIˆØÐr.   r¾   rµ   )r&   r'   r(   r>   r*   r_   r   r+   r   r¶   r   r\   rì   r`   ra   s   @r/   rå   rå   _  s—   ø„ õ(ð 7;Ø15Ø*.Ø,1Ø/4ñ à—|‘|ð ð ! ×!2Ñ!2Ñ3ð ð ˜E×-Ñ-Ñ.ð	 ð
 ! ™ð ð $ D™>ð ð ' t™nð ð 
ˆu�|‰|Ñ	ó öDr.   rå   c                   ó  ‡ — e Zd Zˆ fd„Z	 	 	 	 	 	 	 	 ddej
                  deej                     deej                     deee	e
e
ej                        f      dee   dee   dee   d	ee   d
ee   dee
ej
                     ef   fd„Zˆ xZS )Ú
GitEncoderc           	      óÒ   •— t         ‰| �  «        || _        t        j                  t        |j                  «      D �cg c]  }t        ||«      ‘Œ c}«      | _        d| _	        y c c}w ©NF)
r=   r>   rQ   r   Ú
ModuleListÚrangeÚnum_hidden_layersrå   ÚlayerÚgradient_checkpointing)rP   rQ   ÚirR   s      €r/   r>   zGitEncoder.__init__‘  sP   ø€ Ü‰ÑÔØˆŒÜ—]‘]ÄÀv×G_ÑG_ÓA`Ö#a¸A¤H¨V°QÕ$7Ò#aÓbˆŒ
Ø&+ˆÕ#ùò $bs   ½A$r$   r„   r…   Úpast_key_valuesÚ	use_cacher‡   Úoutput_hidden_statesrˆ   Úreturn_dictrV   c
           	      óÀ  — | j                   r%| j                  r|rt        j                  d«       d}d}
|rIt	        |t
        «      s9d}
|€t        «       }n*t        j                  |«      }t        j                  d«       |rdnd }|rdnd }d }t        | j                  «      D ]t  \  }}|r||fz   }|�||   nd }| j                   r-| j                  r!| j                  |j                  |||||«      }n |||||||«      }|d   }|r|d   }|sŒl||d   fz   }Œv |r||fz   }|r|nd }|
r|j                  «       }|	st        d	„ ||||fD «       «      S t        ||||¬
«      S )NzZ`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...FTzÿWe detected that you are passing `past_key_values` as a tuple of tuples. This is deprecated and will be removed in v4.47. Please convert your cache or use an appropriate `Cache` class (https://huggingface.co/docs/transformers/kv_cache#legacy-cache-format)r-   r   r:   r   c              3   ó$   K  — | ]  }|�|–— Œ
 y ­wr¾   r-   ©Ú.0Úvs     r/   ú	<genexpr>z%GitEncoder.forward.<locals>.<genexpr>â  s   è ø€ ò 	àð �=ô ñ	ùs   ‚©r#   rü   r$   r%   )rú   Útrainingrn   ro   rÚ   r   r   Úfrom_legacy_cacheÚ	enumeraterù   Ú_gradient_checkpointing_funcÚ__call__Úto_legacy_cacheÚtupler   )rP   r$   r„   r…   rü   rý   r‡   rþ   rˆ   rÿ   Úreturn_legacy_cacheÚall_hidden_statesÚall_self_attentionsÚnext_decoder_cacherû   Úlayer_moduleÚlayer_head_maskÚlayer_outputsÚ
next_caches                      r/   r\   zGitEncoder.forward—  sÏ  € ð ×&Ò&¨4¯=ª=ÙÜ×#Ñ#Øpôð "�	ð $ÐÙœZ¨¼Ô?Ø"&ÐØÐ&Ü".£.‘ä".×"@Ñ"@ÀÓ"Q�Ü×#Ñ#ð^ôñ #7™B¸DÐÙ$5™b¸4ÐØ!ÐÜ(¨¯©Ó4ò 	P‰OˆAˆ|Ù#Ø$5¸Ð8HÑ$HÐ!à.7Ð.C˜i¨šlÈˆOà×*Ò*¨t¯}ª}Ø $× AÑ AØ ×)Ñ)Ø!Ø"Ø#Ø#Ø%ó!‘ñ !-Ø!Ø"Ø#Ø#Ø%Ø(ó!�ð *¨!Ñ,ˆMÙØ%2°2Ñ%6Ð"Ú Ø&9¸]È1Ñ=MÐ<OÑ&OÑ#ð;	Pñ>  Ø 1°]Ð4DÑ DÐá+4Ñ'¸$ˆ
ÙØ#×3Ñ3Ó5ˆJáÜñ 	ð "ØØ%Ø'ð	ô	ó 	ð 	ô 'Ø+Ø&Ø+Ø*ô	
ð 	
r.   )NNNNFFFT)r&   r'   r(   r>   r*   r_   r   r+   r   r   r   r¶   r   r\   r`   ra   s   @r/   ró   ró   �  sõ   ø„ ô,ð 7;Ø15ØSWØ$(Ø,1Ø/4Ø/4Ø&*ñZ
à—|‘|ðZ
ð ! ×!2Ñ!2Ñ3ðZ
ð ˜E×-Ñ-Ñ.ð	Z
ð
 " %¨¨u°U¸5×;LÑ;LÑ5MÑ/NÐ(NÑ"OÑPðZ
ð ˜D‘>ðZ
ð $ D™>ðZ
ð ' t™nðZ
ð ' t™nðZ
ð ˜d‘^ðZ
ð 
ˆu�U—\‘\Ñ"Ð$;Ð;Ñ	<÷Z
r.   ró   c                   ó*   — e Zd ZdZeZdZdZdZdZ	d„ Z
y)ÚGitPreTrainedModelz†
    An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained
    models.
    ÚgitTc                 óÆ  — t        |t        «      rÒt        j                  j	                  |j
                  d| j                  j                  ¬«       t        j                  j	                  |j                  j                  | j                  j                  ¬«       t        j                  j	                  |j                  j                  | j                  j                  ¬«       t        |t        j                  «      rm|j                  j                  j	                  d| j                  j                  ¬«       |j                  �%|j                  j                  j                  «        yyt        |t        j                  «      rz|j                  j                  j	                  d| j                  j                  ¬«       |j                   �2|j                  j                  |j                      j                  «        yyt        |t        j"                  «      rJ|j                  j                  j                  «        |j                  j                  j%                  d«       yy)zInitialize the weightsg        )ÚmeanÚstd)r  Ng      ð?)rÚ   ÚGitVisionEmbeddingsr   ÚinitÚnormal_Úclass_embeddingrQ   Úinitializer_rangeÚpatch_embeddingÚweightÚposition_embeddingrw   ÚdataÚbiasÚzero_r?   r3   rF   Úfill_)rP   Úmodules     r/   Ú_init_weightsz GitPreTrainedModel._init_weights   s‹  € ä�fÔ1Ô2Ü�G‰G�O‰O˜F×2Ñ2¸À$Ç+Á+×B_ÑB_ˆOÔ`Ü�G‰G�O‰O˜F×2Ñ2×9Ñ9¸t¿{¹{×?\Ñ?\ˆOÔ]Ü�G‰G�O‰O˜F×5Ñ5×<Ñ<À$Ç+Á+×B_ÑB_ˆOÔ`Ü�fœbŸi™iÔ(ð �M‰M×Ñ×&Ñ&¨C°T·[±[×5RÑ5RÐ&ÔSØ�{‰{Ð&Ø—‘× Ñ ×&Ñ&Õ(ð 'ä˜¤§¡Ô-Ø�M‰M×Ñ×&Ñ&¨C°T·[±[×5RÑ5RÐ&ÔSØ×!Ñ!Ð-Ø—‘×"Ñ" 6×#5Ñ#5Ñ6×<Ñ<Õ>ð .ä˜¤§¡Ô-Ø�K‰K×Ñ×"Ñ"Ô$Ø�M‰M×Ñ×$Ñ$ SÕ)ð .r.   N)r&   r'   r(   r)   r   Úconfig_classÚbase_model_prefixÚsupports_gradient_checkpointingÚ_supports_cache_classÚ_supports_quantized_cacher)  r-   r.   r/   r  r  ô  s+   „ ñð
 €LØÐØ&*Ð#Ø ÐØ $Ðó*r.   r  a=  

    This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
    library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
    etc.)

    This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
    Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
    and behavior.

    Parameters:
        config ([`GitConfig`]): Model configuration class with all the parameters of the model.
            Initializing with a config file does not load the weights associated with the model, only the
            configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
an  
    Args:
        input_ids (`torch.LongTensor` of shape `({0})`):
            Indices of input sequence tokens in the vocabulary.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are input IDs?](../glossary#input-ids)
        attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*):
            Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:

            - 1 for tokens that are **not masked**,
            - 0 for tokens that are **masked**.

            [What are attention masks?](../glossary#attention-mask)

        position_ids (`torch.LongTensor` of shape `({0})`, *optional*):
            Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
            config.max_position_embeddings - 1]`.

            [What are position IDs?](../glossary#position-ids)

        pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`):
            Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See
            [`CLIPImageProcessor.__call__`] for details.

        head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*):
            Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`:

            - 1 indicates the head is **not masked**,
            - 0 indicates the head is **masked**.

        inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*):
            Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
            is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
            model's internal embedding lookup matrix.
        past_key_values (`Cache` or `tuple(tuple(torch.FloatTensor))`, *optional*):
            Pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention
            blocks) that can be used to speed up sequential decoding. This typically consists in the `past_key_values`
            returned by the model at a previous stage of decoding, when `use_cache=True` or `config.use_cache=True`.

            Two formats are allowed:
            - a [`~cache_utils.Cache`] instance, see our
            [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache);
            - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of
            shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`). This is also known as the legacy
            cache format.

            The model will output the same cache format that is fed as input. If no `past_key_values` are passed, the
            legacy cache format will be returned.

            If `past_key_values` are used, the user can optionally input only the last `input_ids` (those that don't
            have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `input_ids`
            of shape `(batch_size, sequence_length)`.
        output_attentions (`bool`, *optional*):
            Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
            tensors for more detail.
        output_hidden_states (`bool`, *optional*):
            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
            more detail.
        interpolate_pos_encoding (`bool`, *optional*, defaults `False`):
            Whether to interpolate the pre-trained position encodings.
        return_dict (`bool`, *optional*):
            Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
c                   óž   ‡ — e Zd Zdefˆ fd„Zdej                  dededej                  fd„Zd
dej                  dej                  fd	„Z
ˆ xZS )r  rQ   c                 óÚ  •— t         ‰| �  «        || _        |j                  | _        |j
                  | _        |j                  | _        t        j                  t        j                  | j                  «      «      | _        t        j                  |j                  | j                  | j                  | j                  d¬«      | _        | j
                  | j                  z  dz  | _        | j                  dz   | _        t        j"                  | j                   | j                  «      | _        | j'                  dt        j(                  | j                   «      j+                  d«      d¬«       y )NF)Úin_channelsÚout_channelsÚkernel_sizeÚstrider%  rg   r   r8   r9   r;   )r=   r>   rQ   rA   Ú	embed_dimrs   rt   r   Ú	Parameterr*   Úrandnr  ÚConv2dÚnum_channelsr!  Únum_patchesÚnum_positionsr?   r#  rL   rM   rN   rO   s     €r/   r>   zGitVisionEmbeddings.__init__k  s	  ø€ Ü‰ÑÔØˆŒØ×+Ñ+ˆŒØ ×+Ñ+ˆŒØ ×+Ñ+ˆŒä!Ÿ|™|¬E¯K©K¸¿¹Ó,GÓHˆÔä!Ÿy™yØ×+Ñ+ØŸ™ØŸ™Ø—?‘?Øô 
ˆÔð !ŸO™O¨t¯©Ñ>À1ÑDˆÔØ!×-Ñ-°Ñ1ˆÔÜ"$§,¡,¨t×/AÑ/AÀ4Ç>Á>Ó"RˆÔØ×Ñ˜^¬U¯\©\¸$×:LÑ:LÓ-M×-TÑ-TÐU\Ó-]ÐjoÐÕpr.   r[   ÚheightÚwidthrV   c                 óÒ  — |j                   d   dz
  }| j                  j                  j                  d«      }|j                   d   dz
  }t        j
                  j                  «       s%||k(  r ||k(  r| j                  | j                  «      S |dd…dd…f   }|dd…dd…f   }|j                   d   }	|| j                  z  }
|| j                  z  }t        |dz  «      }|j                  d|||	«      }|j                  dddd«      }t        j                  j                  ||
|fdd	¬
«      }|j                  dddd«      j                  dd|	«      }t	        j                   ||fd¬«      S )a   
        This method allows to interpolate the pre-trained position encodings, to be able to use the model on higher resolution
        images. This method is also adapted to support torch.jit tracing.

        Adapted from:
        - https://github.com/facebookresearch/dino/blob/de9ee3df6cf39fac952ab558447af1fa1365362a/vision_transformer.py#L174-L194, and
        - https://github.com/facebookresearch/dinov2/blob/e1277af2ba9496fbadf7aec6eba56e8d882d1e35/dinov2/models/vision_transformer.py#L179-L211
        r   r   Nr:   g      à?r	   rg   ÚbicubicF)rX   ÚmodeÚalign_cornersrŠ   )r•   r#  r"  Ú	unsqueezer*   ÚjitÚ
is_tracingr8   rt   r   Úreshaper�   r   rœ   Úinterpolater€   r’   )rP   r[   r<  r=  r:  r#  r;  Úclass_pos_embedÚpatch_pos_embedr‹   Ú
new_heightÚ	new_widthÚsqrt_num_positionss                r/   Úinterpolate_pos_encodingz,GitVisionEmbeddings.interpolate_pos_encoding�  sv  € ð !×&Ñ& qÑ)¨AÑ-ˆØ!×4Ñ4×;Ñ;×EÑEÀaÓHÐØ*×0Ñ0°Ñ3°aÑ7ˆô �y‰y×#Ñ#Ô%¨+¸Ò*FÈ6ÐUZÊ?Ø×*Ñ*¨4×+<Ñ+<Ó=Ð=à,ªQ°°°¨UÑ3ˆØ,ªQ°±¨UÑ3ˆà×Ñ˜rÑ"ˆà˜tŸ™Ñ.ˆ
Ø˜TŸ_™_Ñ,ˆ	ä& }°cÑ'9Ó:ÐØ)×1Ñ1°!Ð5GÐI[Ð]`ÓaˆØ)×1Ñ1°!°Q¸¸1Ó=ˆäŸ-™-×3Ñ3ØØ˜iÐ(ØØð	 4ó 
ˆð *×1Ñ1°!°Q¸¸1Ó=×BÑBÀ1ÀbÈ#ÓNˆä�y‰y˜/¨?Ð;ÀÔCÐCr.   Úpixel_valuesc                 ó`  — |j                   \  }}}}|sJ|| j                  k7  s|| j                  k7  r,t        d|› d|› d| j                  › d| j                  › d�	«      ‚| j                  j                  j
                  }| j                  |j                  |¬«      «      }|j                  d«      j                  dd«      }| j                  j                  |dd«      }	t        j                  |	|gd¬	«      }
|r|
| j                  |
||«      z   }
|
S |
| j                  | j                  «      z   }
|
S )
NzInput image size (Ú*z) doesn't match model (ú).r�   rg   r   r:   rŠ   )r•   rs   rl   r!  r"  rŽ   r˜   Úflattenr”   r  rN   r*   r’   rL  r#  r8   )rP   rM  rL  Ú
batch_sizeÚ_r<  r=  Útarget_dtypeÚpatch_embedsÚclass_embedsr[   s              r/   r\   zGitVisionEmbeddings.forwardª  s6  € Ø'3×'9Ñ'9Ñ$ˆ
�A�v˜uÙ'¨V°t·±Ò-FÈ%ÐSW×SbÑSbÒJbÜØ$ V H¨A¨e¨WÐ4KÈDÏOÉOÐK\Ð\]Ð^b×^mÑ^mÐ]nÐnpÐqóð ð ×+Ñ+×2Ñ2×8Ñ8ˆØ×+Ñ+¨L¯O©OÀ,¨OÓ,OÓPˆØ#×+Ñ+¨AÓ.×8Ñ8¸¸AÓ>ˆà×+Ñ+×2Ñ2°:¸qÀ"ÓEˆÜ—Y‘Y ¨lÐ;ÀÔCˆ
Ù#Ø# d×&CÑ&CÀJÐPVÐX]Ó&^Ñ^ˆJð Ðð $ d×&=Ñ&=¸d×>OÑ>OÓ&PÑPˆJØÐr.   ©F)r&   r'   r(   r   r>   r*   r_   r^   rL  r+   r\   r`   ra   s   @r/   r  r  j  sd   ø„ ðq˜õ qð,'D°5·<±<ð 'DÈð 'DÐUXð 'DÐ]b×]iÑ]ió 'DñR E×$5Ñ$5ð ÐZ_×ZfÑZf÷ r.   r  c                   óV   ‡ — e Zd Zˆ fd„Zdej
                  dej
                  fd„Zˆ xZS )ÚGitVisionMLPc                 ó  •— t         ‰| �  «        || _        t        |j                     | _        t        j                  |j                  |j                  «      | _
        t        j                  |j                  |j                  «      | _        y r¾   )r=   r>   rQ   r
   rÛ   Úactivation_fnr   rw   rA   rÙ   Úfc1Úfc2rO   s     €r/   r>   zGitVisionMLP.__init__¿  sd   ø€ Ü‰ÑÔØˆŒÜ# F×$5Ñ$5Ñ6ˆÔÜ—9‘9˜V×/Ñ/°×1IÑ1IÓJˆŒÜ—9‘9˜V×5Ñ5°v×7IÑ7IÓJˆ�r.   r$   rV   c                 ól   — | j                  |«      }| j                  |«      }| j                  |«      }|S r¾   )r\  r[  r]  rß   s     r/   r\   zGitVisionMLP.forwardÆ  s4   € ØŸ™ Ó/ˆØ×*Ñ*¨=Ó9ˆØŸ™ Ó/ˆØÐr.   rÂ   ra   s   @r/   rY  rY  ¾  s$   ø„ ôKð U§\¡\ð °e·l±l÷ r.   rY  c                   óô   ‡ — e Zd ZdZˆ fd„Zdej                  dedefd„Z	 	 	 ddej                  de	ej                     d	e	ej                     d
e	e
   deej                  e	ej                     f   f
d„Zˆ xZS )ÚGitVisionAttentionz=Multi-headed attention from 'Attention Is All You Need' paperc                 ó
  •— t         ‰| �  «        || _        |j                  | _        |j
                  | _        | j                  | j                  z  | _        | j                  | j                  z  | j                  k7  r&t        d| j                  › d| j                  › d�«      ‚| j                  dz  | _	        |j                  | _        t        j                  | j                  | j                  «      | _        t        j                  | j                  | j                  «      | _        t        j                  | j                  | j                  «      | _        t        j                  | j                  | j                  «      | _        y )Nz;embed_dim must be divisible by num_heads (got `embed_dim`: z and `num_heads`: rP  g      à¿)r=   r>   rQ   rA   r5  rj   Ú	num_headsÚhead_dimrl   ÚscaleÚattention_dropoutrJ   r   rw   Úk_projÚv_projÚq_projÚout_projrO   s     €r/   r>   zGitVisionAttention.__init__Ñ  s  ø€ Ü‰ÑÔØˆŒØ×+Ñ+ˆŒØ×3Ñ3ˆŒØŸ™¨$¯.©.Ñ8ˆŒØ�=‰=˜4Ÿ>™>Ñ)¨T¯^©^Ò;ÜØMÈdÏnÉnÐM]ð ^Ø—N‘NÐ# 2ð'óð ð —]‘] DÑ(ˆŒ
Ø×/Ñ/ˆŒä—i‘i §¡°·±Ó?ˆŒÜ—i‘i §¡°·±Ó?ˆŒÜ—i‘i §¡°·±Ó?ˆŒÜŸ	™	 $§.¡.°$·.±.ÓAˆ�r.   r–   Úseq_lenÚbszc                 óŽ   — |j                  ||| j                  | j                  «      j                  dd«      j	                  «       S )Nr   rg   )r€   rb  rc  r”   rž   )rP   r–   rj  rk  s       r/   Ú_shapezGitVisionAttention._shapeä  s7   € Ø�{‰{˜3 ¨¯©¸¿¹ÓG×QÑQÐRSÐUVÓW×bÑbÓdÐdr.   r$   r„   Úcausal_attention_maskr‡   rV   c                 ó”  — |j                  «       \  }}}| j                  |«      | j                  z  }| j                  | j	                  |«      d|«      }	| j                  | j                  |«      d|«      }
|| j                  z  d| j                  f} | j                  |||«      j                  |Ž } |	j                  |Ž }	 |
j                  |Ž }
|	j                  d«      }t        j                  ||	j                  dd«      «      }|j                  «       || j                  z  ||fk7  r/t        d|| j                  z  ||f› d|j                  «       › �«      ‚|�{|j                  «       |d||fk7  r#t        d|d||f› d|j                  «       › �«      ‚|j                  || j                  ||«      |z   }|j                  || j                  z  ||«      }|�{|j                  «       |d||fk7  r#t        d|d||f› d|j                  «       › �«      ‚|j                  || j                  ||«      |z   }|j                  || j                  z  ||«      }t        j                  j                  |d¬«      }|r?|j                  || j                  ||«      }|j                  || j                  z  ||«      }nd}t        j                  j!                  || j                   | j"                  ¬	«      }t        j                  ||
«      }|j                  «       || j                  z  || j                  fk7  r7t        d
|| j                  || j                  f› d|j                  «       › �«      ‚|j                  || j                  || j                  «      }|j                  dd«      }|j%                  |||«      }| j'                  |«      }||fS )z#Input shape: Batch x Time x Channelr:   r   rg   z$Attention weights should be of size z	, but is Nz!Attention mask should be of size rŠ   )Úpr  z `attn_output` should be of size )rX   rh  rd  rm  rf  rg  rb  rc  r€   r*   Úbmmr”   rl   r   rœ   r�   rJ   r  rE  ri  )rP   r$   r„   rn  r‡   rk  Útgt_lenr5  Úquery_statesÚ
key_statesÚvalue_statesÚ
proj_shapeÚsrc_lenÚattn_weightsÚattn_weights_reshapedÚ
attn_probsÚattn_outputs                    r/   r\   zGitVisionAttention.forwardç  sÕ  € ð #0×"4Ñ"4Ó"6ÑˆˆW�ið —{‘{ =Ó1°D·J±JÑ>ˆØ—[‘[ §¡¨]Ó!;¸RÀÓEˆ
Ø—{‘{ 4§;¡;¨}Ó#=¸rÀ3ÓGˆà˜DŸN™NÑ*¨B°·±Ð>ˆ
ØC�t—{‘{ <°¸#Ó>×CÑCÀZÐPˆØ$�Z—_‘_ jÐ1ˆ
Ø(�|×(Ñ(¨*Ð5ˆà—/‘/ !Ó$ˆÜ—y‘y ¨z×/CÑ/CÀAÀqÓ/IÓJˆà×ÑÓ 3¨¯©Ñ#7¸À'Ð"JÒJÜØ6¸¸d¿n¹nÑ8LÈgÐW^Ð7_Ð6`ð aØ ×%Ñ%Ó'Ð(ð*óð ð !Ð,Ø$×)Ñ)Ó+°°Q¸ÀÐ/IÒIÜ Ø7¸¸aÀÈ'Ð8RÐ7Sð TØ-×2Ñ2Ó4Ð5ð7óð ð (×,Ñ,¨S°$·.±.À'È7ÓSÐVkÑkˆLØ'×,Ñ,¨S°4·>±>Ñ-AÀ7ÈGÓTˆLàÐ%Ø×"Ñ"Ó$¨¨a°¸'Ð(BÒBÜ Ø7¸¸aÀÈ'Ð8RÐ7SÐS\Ð]k×]pÑ]pÓ]rÐ\sÐtóð ð (×,Ñ,¨S°$·.±.À'È7ÓSÐVdÑdˆLØ'×,Ñ,¨S°4·>±>Ñ-AÀ7ÈGÓTˆLä—}‘}×,Ñ,¨\¸rÐ,ÓBˆáð
 %1×$5Ñ$5°c¸4¿>¹>È7ÐT[Ó$\Ð!Ø0×5Ñ5°c¸D¿N¹NÑ6JÈGÐU\Ó]‰Là$(Ð!ä—]‘]×*Ñ*¨<¸4¿<¹<ÐRV×R_ÑR_Ð*Ó`ˆ
ä—i‘i 
¨LÓ9ˆà×ÑÓ #¨¯©Ñ"6¸ÀÇÁÐ!OÒOÜØ2°C¸¿¹ÈÐRV×R_ÑR_Ð3`Ð2að bØ×$Ñ$Ó&Ð'ð)óð ð
 "×&Ñ& s¨D¯N©N¸GÀTÇ]Á]ÓSˆØ!×+Ñ+¨A¨qÓ1ˆØ!×)Ñ)¨#¨w¸	ÓBˆà—m‘m KÓ0ˆàÐ1Ð1Ð1r.   )NNF)r&   r'   r(   r)   r>   r*   r_   r^   rm  r   r¶   r   r\   r`   ra   s   @r/   r`  r`  Î  s¥   ø„ ÙGôBð&e˜UŸ\™\ð e°Cð e¸có eð 26Ø8<Ø,1ñL2à—|‘|ðL2ð ! §¡Ñ.ðL2ð  (¨¯©Ñ5ð	L2ð
 $ D™>ðL2ð 
ˆu�|‰|˜X e§l¡lÑ3Ð3Ñ	4÷L2r.   r`  c                   ó    ‡ — e Zd Zdefˆ fd„Z	 d	dej                  dej                  dej                  dee   de	ej                     f
d„Zˆ xZS )
ÚGitVisionEncoderLayerrQ   c                 óD  •— t         ‰| �  «        |j                  | _        t	        |«      | _        t        j                  | j                  |j                  ¬«      | _	        t        |«      | _        t        j                  | j                  |j                  ¬«      | _        y rº   )r=   r>   rA   r5  r`  Ú	self_attnr   rF   rG   Úlayer_norm1rY  ÚmlpÚlayer_norm2rO   s     €r/   r>   zGitVisionEncoderLayer.__init__8  sm   ø€ Ü‰ÑÔØ×+Ñ+ˆŒÜ+¨FÓ3ˆŒÜŸ<™<¨¯©¸F×<QÑ<QÔRˆÔÜ Ó'ˆŒÜŸ<™<¨¯©¸F×<QÑ<QÔRˆÕr.   r$   r„   rn  r‡   rV   c                 óÎ   — |}| j                  |«      }| j                  ||||¬«      \  }}||z   }|}| j                  |«      }| j                  |«      }||z   }|f}|r||fz  }|S )aI  
        Args:
            hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)`
            attention_mask (`torch.FloatTensor`): attention mask of size
                `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values.
                `(config.encoder_attention_heads,)`.
            output_attentions (`bool`, *optional*):
                Whether or not to return the attentions tensors of all attention layers. See `attentions` under
                returned tensors for more detail.
        )r$   r„   rn  r‡   )r€  r  r‚  r�  )rP   r$   r„   rn  r‡   Úresidualrx  r³   s           r/   r\   zGitVisionEncoderLayer.forward@  s’   € ð" !ˆà×(Ñ(¨Ó7ˆØ&*§n¡nØ'Ø)Ø"7Ø/ð	 '5ó '
Ñ#ˆ�|ð ! =Ñ0ˆà ˆØ×(Ñ(¨Ó7ˆØŸ™ Ó/ˆØ  =Ñ0ˆà Ð"ˆáØ˜�Ñ&ˆGàˆr.   rW  )r&   r'   r(   r   r>   r*   r_   r   r¶   r   r+   r\   r`   ra   s   @r/   r}  r}  7  sf   ø„ ðS˜õ Sð -2ñ&à—|‘|ð&ð Ÿ™ð&ð  %Ÿ|™|ð	&ð
 $ D™>ð&ð 
ˆu× Ñ Ñ	!÷&r.   r}  c                   ó¤   ‡ — e Zd ZdZdefˆ fd„Z	 	 	 	 	 ddeej                     deej                     dee	   dee	   dee	   d	e
eef   fd
„Zˆ xZS )ÚGitVisionEncoderz·
    Transformer encoder consisting of `config.num_hidden_layers` self attention layers. Each layer is a
    [`GitVisionEncoderLayer`].

    Args:
        config: GitVisionConfig
    rQ   c                 óÐ   •— t         ‰| �  «        || _        t        j                  t        |j                  «      D �cg c]  }t        |«      ‘Œ c}«      | _        d| _	        y c c}w rõ   )
r=   r>   rQ   r   rö   r÷   rø   r}  Úlayersrú   )rP   rQ   rS  rR   s      €r/   r>   zGitVisionEncoder.__init__s  sP   ø€ Ü‰ÑÔØˆŒÜ—m‘mÌEÐRX×RjÑRjÓLkÖ$lÀqÔ%:¸6Õ%BÒ$lÓmˆŒØ&+ˆÕ#ùò %ms   ½A#r„   rn  r‡   rþ   rÿ   rV   c                 ó  — |�|n| j                   j                  }|�|n| j                   j                  }|�|n| j                   j                  }|rdnd}|rdnd}|}	t	        | j
                  «      D ]b  \  }
}|r||	fz   }| j                  r,| j                  r | j                  |j                  |	|||«      }n ||	|||¬«      }|d   }	|sŒZ||d   fz   }Œd |r||	fz   }|st        d„ |	||fD «       «      S t        |	||¬«      S )aÕ  
        Args:
            inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
                Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
                This is useful if you want more control over how to convert `input_ids` indices into associated vectors
                than the model's internal embedding lookup matrix.
            attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
                Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:

                - 1 for tokens that are **not masked**,
                - 0 for tokens that are **masked**.

                [What are attention masks?](../glossary#attention-mask)
            causal_attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
                Causal mask for the text model. Mask values selected in `[0, 1]`:

                - 1 for tokens that are **not masked**,
                - 0 for tokens that are **masked**.

                [What are attention masks?](../glossary#attention-mask)
            output_attentions (`bool`, *optional*):
                Whether or not to return the attentions tensors of all attention layers. See `attentions` under
                returned tensors for more detail.
            output_hidden_states (`bool`, *optional*):
                Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors
                for more detail.
            return_dict (`bool`, *optional*):
                Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
        Nr-   )r‡   r   r   c              3   ó&   K  — | ]	  }|€Œ|–— Œ y ­wr¾   r-   r  s     r/   r  z+GitVisionEncoder.forward.<locals>.<genexpr>Å  s   è ø€ Òe˜qÐWXÑWdœÑeùs   ‚Š©r#   r$   r%   )rQ   r‡   rþ   Úuse_return_dictr	  rˆ  rú   r  r
  r  r  r   )rP   rT   r„   rn  r‡   rþ   rÿ   Úencoder_statesÚall_attentionsr$   ÚidxÚencoder_layerr  s                r/   r\   zGitVisionEncoder.forwardy  sH  € ðL 2CÐ1NÑ-ÐTX×T_ÑT_×TqÑTqÐà$8Ð$DÑ È$Ï+É+×JjÑJjð 	ð &1Ð%<‘kÀ$Ç+Á+×B]ÑB]ˆá3™¸ˆÙ0™°dˆà%ˆÜ"+¨D¯K©KÓ"8ò 	FÑˆC�Ù#Ø!/°=Ð2BÑ!B�Ø×*Ò*¨t¯}ª}Ø $× AÑ AØ!×*Ñ*Ø!Ø"Ø)Ø%ó!‘ñ !.Ø!Ø"Ø)Ø&7ô	!�ð *¨!Ñ,ˆMâ Ø!/°=ÀÑ3CÐ2EÑ!E‘ð-	Fñ0  Ø+¨}Ð.>Ñ>ˆNáÜÑe ]°NÀNÐ$SÔeÓeÐeÜØ+¸>ÐVdô
ð 	
r.   )NNNNN)r&   r'   r(   r)   r   r>   r   r*   r_   r¶   r   r   r   r\   r`   ra   s   @r/   r†  r†  j  s•   ø„ ñð,˜õ ,ð 26Ø8<Ø,0Ø/3Ø&*ñO
ð ! §¡Ñ.ðO
ð  (¨¯©Ñ5ð	O
ð
 $ D™>ðO
ð ' t™nðO
ð ˜d‘^ðO
ð 
ˆu�oÐ%Ñ	&÷O
r.   r†  aÕ  
    Args:
        pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`):
            Pixel values. Padding will be ignored by default should you provide it. Pixel values can be obtained using
            [`AutoImageProcessor`]. See [`CLIPImageProcessor.__call__`] for details.
        output_attentions (`bool`, *optional*):
            Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
            tensors for more detail.
        output_hidden_states (`bool`, *optional*):
            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
            more detail.
        interpolate_pos_encoding (`bool`, *optional*, defaults `False`):
            Whether to interpolate the pre-trained position encodings.
        return_dict (`bool`, *optional*):
            Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
c                   ó¼   ‡ — e Zd Zdefˆ fd„Z ee«       eee¬«      	 	 	 	 	 dde	e
j                     de	e   de	e   de	e   de	e   d	eeef   fd
„«       «       Zˆ xZS )ÚGitVisionTransformerrQ   c                 ó   •— t         ‰| �  «        || _        |j                  }t	        |«      | _        t        j                  ||j                  ¬«      | _	        t        |«      | _        t        j                  ||j                  ¬«      | _        y rº   )r=   r>   rQ   rA   r  r[   r   rF   rG   Úpre_layrnormr†  ÚencoderÚpost_layernorm)rP   rQ   r5  rR   s      €r/   r>   zGitVisionTransformer.__init__ß  sj   ø€ Ü‰ÑÔØˆŒØ×&Ñ&ˆ	ä-¨fÓ5ˆŒÜŸL™L¨¸×8MÑ8MÔNˆÔÜ'¨Ó/ˆŒÜ Ÿl™l¨9¸&×:OÑ:OÔPˆÕr.   ©Úoutput_typer*  rM  r‡   rþ   rL  rÿ   rV   c                 ó°  — |�|n| j                   j                  }|�|n| j                   j                  }|�|n| j                   j                  }|€t	        d«      ‚| j                  ||¬«      }| j                  |«      }| j                  ||||¬«      }|d   }| j                  |«      }|s	|f|dd z   S t        ||j                  |j                  ¬«      S )z
        Returns:

        Nz You have to specify pixel_values©rL  )rT   r‡   rþ   rÿ   r   r   r‹  )rQ   r‡   rþ   rŒ  rl   r[   r”  r•  r–  r   r$   r%   )	rP   rM  r‡   rþ   rL  rÿ   r$   Úencoder_outputsr#   s	            r/   r\   zGitVisionTransformer.forwardé  sÿ   € ð 2CÐ1NÑ-ÐTX×T_ÑT_×TqÑTqÐà$8Ð$DÑ È$Ï+É+×JjÑJjð 	ð &1Ð%<‘kÀ$Ç+Á+×B]ÑB]ˆàÐÜÐ?Ó@Ð@àŸ™¨ÐOg˜ÓhˆØ×)Ñ)¨-Ó8ˆàŸ,™,Ø'Ø/Ø!5Ø#ð	 'ó 
ˆð ,¨AÑ.Ðà ×/Ñ/Ð0AÓBÐáØ%Ð'¨/¸!¸"Ð*=Ñ=Ð=äØ/Ø)×7Ñ7Ø&×1Ñ1ô
ð 	
r.   ©NNNFN)r&   r'   r(   r   r>   r   ÚGIT_VISION_INPUTS_DOCSTRINGr   r   r   r*   r+   r¶   r   r   r\   r`   ra   s   @r/   r’  r’  Ý  s¬   ø„ ðQ˜õ Qñ +Ð+FÓGÙ¨?ÈÔYð 59Ø,0Ø/3Ø38Ø&*ñ*
à˜u×0Ñ0Ñ1ð*
ð $ D™>ð*
ð ' t™nð	*
ð
 #+¨4¡.ð*
ð ˜d‘^ð*
ð 
ˆu�oÐ%Ñ	&ò*
ó Zó Hô*
r.   r’  zOThe vision model from CLIP, used in GIT, without any head or projection on top.c                   óÞ   ‡ — e Zd ZeZdZdefˆ fd„Zdej                  fd„Z	 e
e«       eee¬«      	 	 	 	 	 ddeej                      dee   dee   d	ed
ee   deeef   fd„«       «       Zˆ xZS )ÚGitVisionModelrM  rQ   c                 ód   •— t         ‰| �  |«       t        |«      | _        | j	                  «        y r¾   )r=   r>   r’  Úvision_modelÚ	post_initrO   s     €r/   r>   zGitVisionModel.__init__!  s'   ø€ Ü‰Ñ˜Ô Ü0°Ó8ˆÔà�‰Õr.   rV   c                 óB   — | j                   j                  j                  S r¾   )r¡  r[   r!  ©rP   s    r/   Úget_input_embeddingsz#GitVisionModel.get_input_embeddings'  s   € Ø× Ñ ×+Ñ+×;Ñ;Ð;r.   r—  r‡   rþ   rL  rÿ   c                 ób   — |�|n| j                   j                  }| j                  |||||¬«      S )a�  
        Returns:

        Examples:

        ```python
        >>> from PIL import Image
        >>> import requests
        >>> from transformers import AutoProcessor, GitVisionModel

        >>> processor = AutoProcessor.from_pretrained("microsoft/git-base")
        >>> model = GitVisionModel.from_pretrained("microsoft/git-base")

        >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
        >>> image = Image.open(requests.get(url, stream=True).raw)

        >>> inputs = processor(images=image, return_tensors="pt")

        >>> outputs = model(**inputs)
        >>> last_hidden_state = outputs.last_hidden_state
        ```)rM  r‡   rþ   rL  rÿ   )rQ   rŒ  r¡  )rP   rM  r‡   rþ   rL  rÿ   s         r/   r\   zGitVisionModel.forward*  sA   € ð> &1Ð%<‘kÀ$Ç+Á+×B]ÑB]ˆà× Ñ Ø%Ø/Ø!5Ø%=Ø#ð !ó 
ð 	
r.   rœ  )r&   r'   r(   r   r*  Úmain_input_namer>   r   ÚModuler¥  r   r�  r   r   r   r*   r+   r¶   r   r   r\   r`   ra   s   @r/   rŸ  rŸ    sÂ   ø„ ð
 #€LØ$€Oð˜õ ð< b§i¡ió <ñ +Ð+FÓGÙ¨?ÈÔYð 59Ø,0Ø/3Ø).Ø&*ñ%
à˜u×0Ñ0Ñ1ð%
ð $ D™>ð%
ð ' t™nð	%
ð
 #'ð%
ð ˜d‘^ð%
ð 
ˆu�oÐ%Ñ	&ò%
ó Zó Hô%
r.   rŸ  c                   ó\   ‡ — e Zd Zdefˆ fd„Zdej                  dej                  fd„Zˆ xZS )ÚGitProjectionrQ   c                 ó0  •— t         ‰| �  «        || _        t        j                  t        j
                  |j                  j                  |j                  «      t        j                  |j                  |j                  j                  ¬«      «      | _
        y rº   )r=   r>   rQ   r   Ú
Sequentialrw   rr   rA   rF   rG   Úvisual_projectionrO   s     €r/   r>   zGitProjection.__init__U  sf   ø€ Ü‰ÑÔØˆŒÜ!#§¡Ü�I‰I�f×*Ñ*×6Ñ6¸×8JÑ8JÓKÜ�L‰L˜×+Ñ+°×1EÑ1E×1TÑ1TÔUó"
ˆÕr.   r[   rV   c                 ó$   — | j                  |«      S r¾   )r­  )rP   r[   s     r/   r\   zGitProjection.forward]  s   € Ø×%Ñ% jÓ1Ð1r.   )	r&   r'   r(   r   r>   r*   r_   r\   r`   ra   s   @r/   rª  rª  T  s*   ø„ ð
˜yõ 
ð2 %§,¡,ð 2°5·<±<÷ 2r.   rª  z‘The bare GIT Model transformer consisting of a CLIP image encoder and text decoder outputting raw hidden-states without any specific head on top.c                   ó2  ‡ — e Zd Zˆ fd„Zd„ Zd„ Zd„ Zdedej                  dej                  dej                  fd	„Zdd
„Z eej!                  d«      «       eee¬«      	 	 	 	 	 	 	 	 	 	 	 	 ddeej                     deej                     deej                     deej                     deej                     deej                     deeeeej0                     f      dee   dee   dee   dedee   deeej                     ef   fd„«       «       Zˆ xZS )ÚGitModelc                 ól  •‡— t         ‰| �  ‰«       ‰| _        t        ‰«      | _        t        ‰j                  «      | _        t        ‰«      | _	        t        ‰«      | _        ‰j                  �6t        j                  ˆfd„t        ‰j                  «      D «       «      | _        | j#                  «        y )Nc              3   óš   •K  — | ]B  }t        j                  t        j                  d d ‰j                  j
                  «      «      –— ŒD y­w)r   N)r   r6  r*   Úzerosrr   rA   )r  rS  rQ   s     €r/   r  z$GitModel.__init__.<locals>.<genexpr>r  s;   øè ø€ ò ;àô —‘œUŸ[™[¨¨A¨v×/CÑ/C×/OÑ/OÓP×Qñ;ùs   ƒAA)r=   r>   rQ   r1   r[   rŸ  rr   Úimage_encoderró   r•  rª  r­  rv   r   ÚParameterListr÷   Úimg_temperal_embeddingr¢  rO   s    `€r/   r>   zGitModel.__init__g  s’   ù€ Ü‰Ñ˜Ô ØˆŒä'¨Ó/ˆŒÜ+¨F×,@Ñ,@ÓAˆÔÜ! &Ó)ˆŒä!.¨vÓ!6ˆÔà×*Ñ*Ð6Ü*,×*:Ñ*:ó ;ä˜v×>Ñ>Ó?ô;ó +ˆDÔ'ð 	�‰Õr.   c                 ó.   — | j                   j                  S r¾   ©r[   rC   r¤  s    r/   r¥  zGitModel.get_input_embeddingsz  s   € Ø�‰×.Ñ.Ð.r.   c                 ó&   — || j                   _        y r¾   r¸  )rP   rz   s     r/   Úset_input_embeddingszGitModel.set_input_embeddings}  s   € Ø*/ˆ�‰Õ'r.   c                 ó˜   — |j                  «       D ]7  \  }}| j                  j                  |   j                  j	                  |«       Œ9 y)z�
        Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base
        class PreTrainedModel
        N)Úitemsr•  rù   ré   rÒ   )rP   Úheads_to_prunerù   rÐ   s       r/   Ú_prune_headszGitModel._prune_heads€  sE   € ð
 +×0Ñ0Ó2ò 	C‰LˆE�5Ø�L‰L×Ñ˜uÑ%×/Ñ/×;Ñ;¸EÕBñ	Cr.   rX   rŽ   r�   rV   c                 óž   — t        j                  t        j                  ||||¬«      d¬«      }|j                  |dk(  t	        d«      «      }|S )N©r�   rŽ   r   )Údiagonalú-inf)r*   ÚtriuÚonesÚmasked_fillÚfloat)rP   rX   rŽ   r�   Úmasks        r/   Ú_generate_future_maskzGitModel._generate_future_maskˆ  sA   € ä�z‰zœ%Ÿ*™* T¨4¸ÀeÔLÐWXÔYˆØ×Ñ ¨¡	¬5°«=Ó9ˆØˆr.   c                 ó   — |j                   d   }|j                   d   }|j                  }|j                  }	t        j                  ||f||	¬«      }
t        j
                  |||z   ft        d«      |j                  |	¬«      }t        j                  ||f|	|j                  ¬«      }|dkD  rAt        j                  |j                   d   |j                   d   |z   f|	|j                  ¬«      }t        j                  |
|fd¬«      }t        j                  ||j                  |	«      fd¬«      }t        j                  ||fd¬«      d d d …f   }|€4t        j
                  |j                   d   |j                   d   fd|¬«      }|j                  t        j                  k7  rt        d	«      ‚t        j                  ||j                  ¬
«      }t        d«      ||<   |j                  |j                   d   ||z   ||z   |z   f«      }|j                  «       }|d d …d d …d |…f   }|d d …d d d …f   }||z   |d d …d d …d |…f<   |d d …d d d …d d …f   }|S )Nr   rÀ  rÂ  r�   r   rŠ   F)Ú
fill_valuer�   z1Memory key padding mask must be a boolean tensor.r�   )r•   r�   rŽ   r*   r³  ÚfullrÆ  r’   r˜   r¶   rl   Ú
zeros_likerN   Úclone)rP   ÚtgtÚmemoryÚtgt_maskrU   Úmemory_key_padding_maskÚnum_tgtÚ
num_memoryr�   rŽ   Útop_leftÚ	top_rightÚbottom_leftÚleftÚrightÚfull_attention_maskÚzero_negative_infinityÚorigin_leftr‘   s                      r/   Úcreate_attention_maskzGitModel.create_attention_maskŽ  sF  € Ø—)‘)˜A‘,ˆØ—\‘\ !‘_ˆ
Ø—‘ˆØ—	‘	ˆÜ—;‘; 
¨JÐ7ÀÈeÔTˆÜ—J‘JØ˜Ð#9Ñ9Ð:Ü�&‹MØ—:‘:Øô	
ˆ	ô —k‘kØ�jÐ!ØØ—?‘?ô
ˆð " AÒ%Ü—{‘{Ø—‘ Ñ" H§N¡N°1Ñ$5Ð8NÑ$NÐOØØ—‘ôˆHô �y‰y˜( KÐ0°aÔ8ˆÜ—	‘	˜9 h§k¡k°%Ó&8Ð9¸qÔAˆä#Ÿi™i¨¨u¨¸1Ô=¸dÂA¸gÑFÐà"Ð*Ü&+§j¡j°&·,±,¸q±/À6Ç<Á<ÐPQÁ?Ð1SÐ`eÐntÔ&uÐ#à"×(Ñ(¬E¯J©JÒ6ÜÐPÓQÐQÜ!&×!1Ñ!1Ð2IÐQT×QZÑQZÔ![ÐÜ:?À»-ÐÐ6Ñ7Ø1×8Ñ8Ø$×*Ñ*¨1Ñ-¨z¸GÑ/CÀZÐRhÑEhÐkrÑErÐsó
Ðð 2×7Ñ7Ó9ÐØ)ª!ªQ°°°Ð*;Ñ<ˆØ'ª¨4²¨
Ñ3ˆØ1<¸vÑ1EÐšAšq + : +Ð-Ñ.ð 2²!°Tº1ºa°-Ñ@Ðà"Ð"r.   úbatch_size, sequence_lengthr—  rS   r„   r8   rM  r…   rT   rü   rý   r‡   rþ   rL  rÿ   c                 óf  — |	�|	n| j                   j                  }	|
�|
n| j                   j                  }
|�|n| j                   j                  }|�|n| j                   j                  }|�|�t        d«      ‚|�#| j                  ||«       |j                  «       }n!|�|j                  «       dd }nt        d«      ‚|d   }d}|�5t        |t        «      s|d   d   j                  d   n|j                  «       }| j                  || j                   j                  «      }d}|�Ü|j                  dk(  r| j                  ||¬	«      j                   }nž|j                  d
k(  r„g }t#        |j                  d   «      D ]O  }| j                  |dd…|dd…dd…f   |¬	«      j                   }|| j$                  |   z  }|j'                  |«       ŒQ t)        j*                  |d¬«      }nt        d«      ‚| j-                  |«      }| j/                  ||||¬«      }|€It)        j0                  |j                  d   d|j                  d   f|j2                  |j4                  ¬«      }|j7                  |j                  d«      |j                  d«      z  dd«      }t)        j*                  ||fd¬«      }| j9                  ||j2                  |j4                  «      }| j;                  ||||¬«      }|�mt=        ||j2                  |d   ¬«      j?                  |j4                  «      }|dkD  r|dd…dd…| d…dd…f   }n!|dd…dd…|d    d…|d    d…fxx   |z  cc<   | jA                  ||||||	|
||du¬«	      }|d   }|s	|f|dd z   S tC        ||jD                  |jF                  |jH                  ¬«      S )a‚  
        use_cache (`bool`, *optional*):
            If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see
            `past_key_values`).

        Returns:

        Examples:

        ```python
        >>> from transformers import AutoProcessor, AutoModel
        >>> import requests
        >>> from PIL import Image

        >>> processor = AutoProcessor.from_pretrained("microsoft/git-base")
        >>> model = AutoModel.from_pretrained("microsoft/git-base")

        >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
        >>> image = Image.open(requests.get(url, stream=True).raw)

        >>> text = "this is an image of two cats"

        >>> inputs = processor(images=image, text=text, return_tensors="pt")

        >>> outputs = model(**inputs)
        >>> last_hidden_state = outputs.last_hidden_state
        ```NzDYou cannot specify both input_ids and inputs_embeds at the same timer:   z5You have to specify either input_ids or inputs_embedsr   r   rg   é   rš  é   rŠ   z#pixel_values must be of rank 4 or 5)rS   r8   rT   rU   r�   )rÎ  rÏ  rÐ  rU   )rr  )r„   r…   rü   rý   r‡   rþ   rÿ   rˆ   r  )%rQ   r‡   rþ   rý   rŒ  rl   Ú%warn_if_padding_and_no_attention_maskrX   rÚ   r   r•   Úget_seq_lengthÚget_head_maskrø   Úndimr´  r#   r÷   r¶  Úappendr*   r’   r­  r[   r³  rŽ   r�   ÚrepeatrÈ  rÜ  r   r˜   r•  r   rü   r$   r%   )rP   rS   r„   r8   rM  r…   rT   rü   rý   r‡   rþ   rL  rÿ   rY   rZ   rU   Úprojected_visual_featuresÚvisual_featuresÚ	frame_idxÚvisual_features_frameÚembedding_outputr$   rÐ  Úcombined_attention_maskÚexpanded_attn_maskr›  Úsequence_outputs                              r/   r\   zGitModel.forwardÀ  sC  € ðX 2CÐ1NÑ-ÐTX×T_ÑT_×TqÑTqÐà$8Ð$DÑ È$Ï+É+×JjÑJjð 	ð "+Ð!6‘I¸D¿K¹K×<QÑ<Qˆ	Ø%0Ð%<‘kÀ$Ç+Á+×B]ÑB]ˆàÐ  ]Ð%>ÜÐcÓdÐdØÐ"Ø×6Ñ6°yÀ.ÔQØ#Ÿ.™.Ó*‰KØÐ&Ø'×,Ñ,Ó.¨s°Ð3‰KäÐTÓUÐUà  ‘^ˆ
ð "#ÐØÐ&ô " /´5Ô9ð   Ñ" 1Ñ%×+Ñ+¨AÒ.à$×3Ñ3Ó5ð #ð ×&Ñ& y°$·+±+×2OÑ2OÓPˆ	à$(Ð!ØÐ#Ø× Ñ  AÒ%à"&×"4Ñ"4Ø Ð;Sð #5ó #ç#Ñ#ñ  ð ×"Ñ" aÒ'à"$�Ü!& |×'9Ñ'9¸!Ñ'<Ó!=ò B�IØ,0×,>Ñ,>Ø$¢Q¨	²1²aÐ%7Ñ8ÐSkð -?ó -ç'Ñ'ð *ð *¨T×-HÑ-HÈÑ-SÑSÐ)Ø#×*Ñ*Ð+@ÕAðBô #(§)¡)¨OÀÔ"C‘ô !Ð!FÓGÐGà(,×(>Ñ(>¸Ó(OÐ%àŸ?™?ØØ%Ø'Ø#9ð	 +ó 
Ðð %Ð,Ü(-¯©Ø!×'Ñ'¨Ñ*¨AÐ/?×/EÑ/EÀaÑ/HÐIØ&×,Ñ,Ø'×.Ñ.ô)Ð%ð %>×$DÑ$DØ×!Ñ! !Ó$Ð(A×(FÑ(FÀqÓ(IÑIÈ1Èaó%
Ð!ô
 Ÿ	™	Ð#<Ð>NÐ"OÐUVÔWˆð ×-Ñ-¨jÐ:J×:PÑ:PÐRb×RiÑRiÓjˆð #'×"<Ñ"<Ø Ø,ØØ#9ð	 #=ó #
Ðð Ð%ô "<ØÐ 0× 6Ñ 6ÀÈBÁô"ç‰bÐ!×(Ñ(Ó)ð ð &¨Ò)Ø%7ºº1Ð?UÐ>UÑ>VÒXYÐ8YÑ%ZÑ"à'ªª1¨{¸1©~¨oÑ.?À+ÈaÁ.ÀÑARÐ(RÓSÐWiÑiÓSàŸ,™,ØØ2ØØ+ØØ/Ø!5Ø#Ø!-°TÐ!9ð 'ó 

ˆð *¨!Ñ,ˆáØ#Ð%¨¸¸Ð(;Ñ;Ð;ä&Ø-Ø+×;Ñ;Ø)×7Ñ7Ø&×1Ñ1ô	
ð 	
r.   r¾   )NNNNNNNNNNFN)r&   r'   r(   r>   r¥  rº  r¾  r^   r*   rŽ   r�   r_   rÈ  rÜ  r   ÚGIT_INPUTS_DOCSTRINGÚformatr   r   Ú_CONFIG_FOR_DOCr   r   r   r   r+   r¶   r   r\   r`   ra   s   @r/   r°  r°  a  s°  ø„ ôò&/ò0òCð¨#ð °e·k±kð È5Ï<É<ð Ð\a×\hÑ\hó ó0#ñd +Ð+?×+FÑ+FÐGdÓ+eÓfÙÐ+EÐTcÔdð -1Ø15Ø/3Ø/3Ø,0Ø04ØKOØ$(Ø,0Ø/3Ø).Ø&*ñi
à˜EŸL™LÑ)ði
ð ! §¡Ñ.ði
ð ˜uŸ|™|Ñ,ð	i
ð
 ˜uŸ|™|Ñ,ði
ð ˜EŸL™LÑ)ði
ð   §¡Ñ-ði
ð " %¨¨t°E×4EÑ4EÑ/FÐ(FÑ"GÑHði
ð ˜D‘>ði
ð $ D™>ði
ð ' t™nði
ð #'ði
ð ˜d‘^ði
ð 
ˆu�U—\‘\Ñ"Ð$>Ð>Ñ	?òi
ó eó gôi
r.   r°  zVGIT Model with a `language modeling` head on top for autoregressive language modeling.c            !       ó  ‡ — e Zd ZdgZˆ fd„Zd„ Zd„ Z eej                  d«      «       e
ee¬«      	 	 	 	 	 	 	 	 	 	 	 	 	 ddeej                     deej                     d	eej                     d
eej                     deej                     deej                     deej                     deeeeej                     f      dee   dee   dee   dedee   deeej                     ef   fd„«       «       Z	 dd„Zd„ Zˆ xZS )ÚGitForCausalLMzoutput.weightc                 óÂ   •— t         ‰| �  |«       t        |«      | _        t	        j
                  |j                  |j                  «      | _        | j                  «        y r¾   )
r=   r>   r°  r  r   rw   rA   r@   rÊ   r¢  rO   s     €r/   r>   zGitForCausalLM.__init__t  sF   ø€ Ü‰Ñ˜Ô ä˜FÓ#ˆŒÜ—i‘i × 2Ñ 2°F×4EÑ4EÓFˆŒð 	�‰Õr.   c                 ó   — | j                   S r¾   ©rÊ   r¤  s    r/   Úget_output_embeddingsz$GitForCausalLM.get_output_embeddings}  s   € Ø�{‰{Ðr.   c                 ó   — || _         y r¾   rö  )rP   Únew_embeddingss     r/   Úset_output_embeddingsz$GitForCausalLM.set_output_embeddings€  s	   € Ø$ˆ�r.   rÝ  r—  rS   r„   r8   rM  r…   rT   Úlabelsrü   rý   r‡   rþ   rL  rÿ   rV   c                 óØ  — |�|n| j                   j                  }|�d}	| j                  ||||||||	|
|||¬«      }|d   }| j                  |«      }d}|�Ó| j                  j                  j
                  d   j                  j                  j                  }|dd…|d…dd…f   j                  «       }|dd…dd…f   j                  «       } | j                  |j                  d| j                   j                  «      |j                  d«      fd| j                   j                  i|¤Ž}|s|f|dd z   }|�|f|z   S |S t        |||j                  |j                  |j                   ¬«      S )	a±  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the left-to-right language modeling loss (next word prediction). Indices should be in
            `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are
            ignored (masked), the loss is only computed for the tokens with labels n `[0, ..., config.vocab_size]`
        use_cache (`bool`, *optional*):
            If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see
            `past_key_values`).

        Returns:

        Examples:

        Image captioning example:

        ```python
        >>> from transformers import AutoProcessor, AutoModelForCausalLM
        >>> import requests
        >>> from PIL import Image

        >>> processor = AutoProcessor.from_pretrained("microsoft/git-base-coco")
        >>> model = AutoModelForCausalLM.from_pretrained("microsoft/git-base-coco")

        >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
        >>> image = Image.open(requests.get(url, stream=True).raw)

        >>> pixel_values = processor(images=image, return_tensors="pt").pixel_values

        >>> generated_ids = model.generate(pixel_values=pixel_values, max_length=50)
        >>> generated_caption = processor.batch_decode(generated_ids, skip_special_tokens=True)[0]
        >>> print(generated_caption)
        two cats sleeping on a pink blanket next to remotes.
        ```

        Visual question answering (VQA) example:

        ```python
        >>> from transformers import AutoProcessor, AutoModelForCausalLM
        >>> from huggingface_hub import hf_hub_download
        >>> from PIL import Image

        >>> processor = AutoProcessor.from_pretrained("microsoft/git-base-textvqa")
        >>> model = AutoModelForCausalLM.from_pretrained("microsoft/git-base-textvqa")

        >>> file_path = hf_hub_download(repo_id="nielsr/textvqa-sample", filename="bus.png", repo_type="dataset")
        >>> image = Image.open(file_path).convert("RGB")

        >>> pixel_values = processor(images=image, return_tensors="pt").pixel_values

        >>> question = "what does the front of the bus say at the top?"

        >>> input_ids = processor(text=question, add_special_tokens=False).input_ids
        >>> input_ids = [processor.tokenizer.cls_token_id] + input_ids
        >>> input_ids = torch.tensor(input_ids).unsqueeze(0)

        >>> generated_ids = model.generate(pixel_values=pixel_values, input_ids=input_ids, max_length=50)
        >>> print(processor.batch_decode(generated_ids, skip_special_tokens=True))
        ['what does the front of the bus say at the top? special']
        ```

        Video captioning example:

        ```python
        >>> import av
        >>> import numpy as np
        >>> from PIL import Image
        >>> from huggingface_hub import hf_hub_download
        >>> from transformers import AutoProcessor, AutoModelForCausalLM

        >>> processor = AutoProcessor.from_pretrained("microsoft/git-base-vatex")
        >>> model = AutoModelForCausalLM.from_pretrained("microsoft/git-base-vatex")

        >>> # set seed for reproducability
        >>> np.random.seed(45)


        >>> def read_video_pyav(container, indices):
        ...     '''
        ...     Decode the video with PyAV decoder.
        ...     Args:
        ...         container (`av.container.input.InputContainer`): PyAV container.
        ...         indices (`List[int]`): List of frame indices to decode.
        ...     Returns:
        ...         result (np.ndarray): np array of decoded frames of shape (num_frames, height, width, 3).
        ...     '''
        ...     frames = []
        ...     container.seek(0)
        ...     start_index = indices[0]
        ...     end_index = indices[-1]
        ...     for i, frame in enumerate(container.decode(video=0)):
        ...         if i > end_index:
        ...             break
        ...         if i >= start_index and i in indices:
        ...             frames.append(frame)
        ...     return np.stack([x.to_ndarray(format="rgb24") for x in frames])


        >>> def sample_frame_indices(clip_len, frame_sample_rate, seg_len):
        ...     '''
        ...     Sample a given number of frame indices from the video.
        ...     Args:
        ...         clip_len (`int`): Total number of frames to sample.
        ...         frame_sample_rate (`int`): Sample every n-th frame.
        ...         seg_len (`int`): Maximum allowed index of sample's last frame.
        ...     Returns:
        ...         indices (`List[int]`): List of sampled frame indices
        ...     '''
        ...     converted_len = int(clip_len * frame_sample_rate)
        ...     end_idx = np.random.randint(converted_len, seg_len)
        ...     start_idx = end_idx - converted_len
        ...     indices = np.linspace(start_idx, end_idx, num=clip_len)
        ...     indices = np.clip(indices, start_idx, end_idx - 1).astype(np.int64)
        ...     return indices


        >>> # load video
        >>> file_path = hf_hub_download(
        ...     repo_id="nielsr/video-demo", filename="eating_spaghetti.mp4", repo_type="dataset"
        ... )
        >>> container = av.open(file_path)

        >>> # sample frames
        >>> num_frames = model.config.num_image_with_embedding
        >>> indices = sample_frame_indices(
        ...     clip_len=num_frames, frame_sample_rate=4, seg_len=container.streams.video[0].frames
        ... )
        >>> frames = read_video_pyav(container, indices)

        >>> pixel_values = processor(images=list(frames), return_tensors="pt").pixel_values

        >>> generated_ids = model.generate(pixel_values=pixel_values, max_length=50)

        >>> print("Generated caption:", processor.batch_decode(generated_ids, skip_special_tokens=True))
        Generated caption: ['a woman is sitting at a table and she is talking about the food she is holding.']
        ```
        NF)r„   r8   rM  r…   rT   rü   rý   r‡   rþ   rL  rÿ   r   r:   r   r@   )ÚlossÚlogitsrü   r$   r%   )rQ   rŒ  r  rÊ   r•  rù   ré   rP   ru   rž   Úloss_functionr€   r@   r   rü   r$   r%   )rP   rS   r„   r8   rM  r…   rT   rû  rü   rý   r‡   rþ   rL  rÿ   Úkwargsr³   rî  rþ  rý  Únum_image_tokensÚshifted_logitsrÊ   s                         r/   r\   zGitForCausalLM.forwardƒ  s—  € ðv &1Ð%<‘kÀ$Ç+Á+×B]ÑB]ˆØÐØˆIà—(‘(ØØ)Ø%Ø%ØØ'Ø+ØØ/Ø!5Ø%=Ø#ð ó 
ˆð " !™*ˆØ—‘˜_Ó-ˆàˆØÐà#Ÿx™x×/Ñ/×5Ñ5°aÑ8×BÑB×GÑG×ZÑZÐØ#¢AÐ'7¸Ð':ºAÐ$=Ñ>×IÑIÓKˆNØšA˜q™r˜E‘]×-Ñ-Ó/ˆFØ%�4×%Ñ%Ø×#Ñ# B¨¯©×(>Ñ(>Ó?Ø—‘˜B“ñð  Ÿ;™;×1Ñ1ðð ñ	ˆDñ Ø�Y ¨¨ Ñ,ˆFØ)-Ð)9�T�G˜fÑ$ÐE¸vÐEä%ØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)ô
ð 	
r.   c                 óø   — |�B|j                  «       }|j                  d   |kD  r|}n|j                  d   dz
  }|d d …|d …f   }|j                  }|€|j                  |«      }|||j                  dd «      ||dœS )Nr   rM  )rS   r„   rM  rü   rý   )râ  r•   Únew_onesÚget)	rP   rS   rü   r„   rý   r   Úpast_lengthÚremove_prefix_lengthrY   s	            r/   Úprepare_inputs_for_generationz,GitForCausalLM.prepare_inputs_for_generationM  s¡   € ð Ð&Ø)×8Ñ8Ó:ˆKð �‰˜qÑ! KÒ/Ø'2Ñ$ð (1§¡°qÑ'9¸AÑ'=Ð$à!¢!Ð%9Ñ%:Ð":Ñ;ˆIð  —o‘oˆØÐ!Ø&×/Ñ/°Ó<ˆNð #Ø,Ø"ŸJ™J ~°tÓ<Ø.Ø"ñ
ð 	
r.   c                 óJ   ‡— d}|D ]  }|t        ˆfd„|D «       «      fz  }Œ |S )Nr-   c              3   ót   •K  — | ]/  }|j                  d ‰j                  |j                  «      «      –— Œ1 y­w)r   N)Úindex_selectr˜   r�   )r  Ú
past_stateÚbeam_idxs     €r/   r  z0GitForCausalLM._reorder_cache.<locals>.<genexpr>p  s.   øè ø€ ÒnÐU_�j×-Ñ-¨a°·±¸Z×=NÑ=NÓ1O×PÑnùs   ƒ58)r  )rP   rü   r  Úreordered_pastÚ
layer_pasts     `  r/   Ú_reorder_cachezGitForCausalLM._reorder_cachel  s=   ø€ ØˆØ)ò 	ˆJØÜÓnÐcmÔnÓnðñ ‰Nð	ð Ðr.   )NNNNNNNNNNNFN)NNN)r&   r'   r(   Ú_tied_weights_keysr>   r÷  rú  r   rï  rð  r   r   rñ  r   r*   r_   r   r   r   r¶   r   r\   r  r  r`   ra   s   @r/   ró  ró  n  s¦  ø„ ð *Ð*Ðôòò%ñ +Ð+?×+FÑ+FÐGdÓ+eÓfÙÐ+AÐP_Ô`ð -1Ø15Ø/3Ø/3Ø,0Ø04Ø)-ØFJØ$(Ø,0Ø/3Ø).Ø&*ñF
à˜EŸL™LÑ)ðF
ð ! §¡Ñ.ðF
ð ˜uŸ|™|Ñ,ð	F
ð
 ˜uŸ|™|Ñ,ðF
ð ˜EŸL™LÑ)ðF
ð   §¡Ñ-ðF
ð ˜Ÿ™Ñ&ðF
ð " %¨¨t°E·L±LÑ/AÐ(AÑ"BÑCðF
ð ˜D‘>ðF
ð $ D™>ðF
ð ' t™nðF
ð #'ðF
ð ˜d‘^ðF
ð  
ˆu�U—\‘\Ñ"Ð$:Ð:Ñ	;ò!F
ó aó gðF
ðR OSó
ö>r.   ró  )ró  r°  r  rŸ  )Jr)   rš   Údataclassesr   Útypingr   r   r   r   r*   Útorch.utils.checkpointr   Úactivationsr
   Úcache_utilsr   r   Ú
file_utilsr   Ú
generationr   Úmodeling_attn_mask_utilsr   Úmodeling_outputsr   r   r   r   Úmodeling_utilsr   Úpytorch_utilsr   r   r   Úutilsr   r   r   r   r   Úconfiguration_gitr   r   Ú
get_loggerr&   rn   Ú_CHECKPOINT_FOR_DOCrñ  r!   r¨  r1   rc   r¸   rÈ   rÆ   r×   rá   rå   ró   r  ÚGIT_START_DOCSTRINGrï  r  rY  r`  r}  r†  r�  r’  rŸ  rª  r°  ró  Ú__all__r-   r.   r/   ú<module>r#     sQ  ðñ  ã Ý !ß /Ó /ã Û Ý å !ß .Ý %Ý )Ý B÷ó õ .ß lÑ l÷õ ÷ :ð 
ˆ×	Ñ	˜HÓ	%€à*Ð Ø€ð ô?˜;ó ?ó ð?ô8-�B—I‘Iô -ô`r�r—y‘yô rôl�B—I‘Iô ð ÐðÐ ô
/�2—9‘9ô /ôf�b—i‘iô ô �—	‘	ô ô.ˆr�y‰yô .ôba
�—‘ô a
ôH*˜ô *ðBÐ ð AÐ ôJP˜"Ÿ)™)ô Pôh�2—9‘9ô ô e2˜Ÿ™ô e2ôR/˜BŸI™Iô /ôf^
�r—y‘yô ^
ðBÐ ô$8
˜2Ÿ9™9ô 8
ñv ØYØóô5
Ð'ó 5
ó	ð5
ôp
2�B—I‘Iô 
2ñ ð)àóô
E
Ð!ó E
óð
E
ñP Ø`ÐbuóôAÐ'¨ó AóðAòH Q�r.   