Ë
    T^(h¢ª  ã                   óª  — d Z ddlZddlmZ ddlmZmZ ddlZddlZddlm	Z	 ddl
mZ ddlmZmZmZ dd	lmZmZmZ dd
lmZ ddlmZ ddlmZ ddlmZ ddlmZ  ej<                  e«      Z e G d„ de«      «       Z! G d„ de	jD                  «      Z# G d„ de	jD                  «      Z$ G d„ de	jD                  «      Z% G d„ de	jD                  «      Z& G d„ de	jD                  «      Z' G d„ de	jD                  «      Z( G d„ de	jD                  «      Z) G d „ d!e	jD                  «      Z* G d"„ d#e	jD                  «      Z+ G d$„ d%e	jD                  «      Z, G d&„ d'e«      Z-d(Z.d)Z/ G d*„ d+e	jD                  «      Z0 G d,„ d-e	jD                  «      Z1e0e1d.œZ2 ed/e.«       G d0„ d1e-«      «       Z3 G d2„ d3e	jD                  «      Z4 ed4e.«       G d5„ d6e-«      «       Z5g d7¢Z6y)8zPyTorch TVP Modelé    N)Ú	dataclass)ÚOptionalÚTuple)Únné   )ÚACT2FN)Úadd_start_docstringsÚ%add_start_docstrings_to_model_forwardÚreplace_return_docstrings)ÚBaseModelOutputÚBaseModelOutputWithPoolingÚModelOutput)ÚPreTrainedModel)Úprune_linear_layer)Úlogging)Úload_backboneé   )Ú	TvpConfigc                   óÆ   — e Zd ZU dZdZeej                     ed<   dZ	eej                     ed<   dZ
eeej                  df      ed<   dZeeej                  df      ed<   y)ÚTvpVideoGroundingOutputa–  
    Args:
        loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `return_loss` is `True`):
            Temporal-Distance IoU loss for video grounding.
        logits (`torch.FloatTensor` of shape `(batch_size, 2)`):
            Contains start_time/duration and end_time/duration. It is the time slot of the videos corresponding to the
            input texts.
        hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
            Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, +
            one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`. Hidden-states of
            the model at the output of each layer plus the optional initial embedding outputs.
        attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
            Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
            sequence_length)`.
    NÚlossÚlogits.Úhidden_statesÚ
attentions)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r   ÚtorchÚFloatTensorÚ__annotations__r   r   r   r   © ó    úb/var/www/skyplay_api_hub/venv/lib/python3.12/site-packages/transformers/models/tvp/modeling_tvp.pyr   r   &   sq   … ñð  )-€Dˆ(�5×$Ñ$Ñ
%Ó,Ø*.€FˆH�U×&Ñ&Ñ'Ó.Ø=A€M�8˜E %×"3Ñ"3°SÐ"8Ñ9Ñ:ÓAØ:>€J�˜˜u×0Ñ0°#Ð5Ñ6Ñ7Ô>r#   r   c                   ó:   ‡ — e Zd ZdZˆ fd„Zd„ Zd„ Zd„ Zd„ Zˆ xZ	S )ÚTvpLossa~  
    This class computes the losses for `TvpForVideoGrounding`. The process happens in two steps: 1) we compute
    hungarian assignment between ground truth boxes and the outputs of the model 2) we supervise each pair of matched
    ground-truth / prediction (supervise class and box).

    Args:
        losses (`List[str]`):
            List of all the losses to be applied.
    c                 óÊ   •— t         ‰| �  «        | j                  | j                  | j                  dœ| _        |D ]  }|| j
                  vsŒt        d|› d�«      ‚ || _        y )N©ÚiouÚdistanceÚdurationzLoss z not supported)ÚsuperÚ__init__Úloss_iouÚloss_distanceÚloss_durationÚloss_mapÚ
ValueErrorÚlosses)Úselfr3   r   Ú	__class__s      €r$   r-   zTvpLoss.__init__I   sj   ø€ Ü‰ÑÔà—=‘=Ø×*Ñ*Ø×*Ñ*ñ
ˆŒð
 ò 	?ˆDØ˜4Ÿ=™=Ò(Ü  5¨¨¨nÐ!=Ó>Ð>ð	?ð ˆ�r#   c                 óê   — t        j                  ||«      t        j                  ||«      z
  }t        j                  ||«      t        j                  ||«      z
  }d|j                  d¬«      |z  z
  }|S )z6
        Measure the intersection over union.
        r   r   ©Úmin)r   r8   ÚmaxÚclamp)	r4   Ú
start_timeÚend_timeÚcandidates_start_timeÚcandidates_end_timer+   ÚinterÚunionr)   s	            r$   r.   zTvpLoss.loss_iouV   si   € ô —	‘	Ð-¨xÓ8¼5¿9¹9ÐEZÐ\fÓ;gÑgˆÜ—	‘	Ð-¨xÓ8¼5¿9¹9ÐEZÐ\fÓ;gÑgˆØ�%—+‘+ !�+Ó$ uÑ,Ñ,ˆàˆ
r#   c                 óP  — t        j                  t        j                  ||«      d«      }t        j                  t        j                  ||«      d«      }t        j                  t        j                  ||«      t        j                  ||«      z
  |«      j                  d¬«      }|S )z5
        Measure the distance of mid points.
        g       @gš™™™™™É?r7   )r   ÚdivÚaddr9   r8   r:   )	r4   r;   r<   r=   r>   r+   Úmid_candidatesÚmid_groundtruthÚdistance_diffs	            r$   r/   zTvpLoss.loss_distance`   sƒ   € ô Ÿ™¤5§9¡9Ð-BÐDWÓ#XÐZ]Ó^ˆÜŸ)™)¤E§I¡I¨j¸(Ó$CÀSÓIˆÜŸ	™	Ü�I‰I�n oÓ6¼¿¹À>ÐSbÓ9cÑcÐemó
ç
‰%�Cˆ%‹.ð 	ð Ðr#   c                 óü   — t        j                  ||«      }t        j                  ||«      }t        j                  t        j                  t        j                  ||«      |«      «      }|j	                  d¬«      }|S )z5
        Measure the difference of duration.
        gš™™™™™Ù?r7   )r   ÚsubÚsquarerB   r:   )	r4   r;   r<   r=   r>   r+   Úduration_candidatesÚduration_groundtruthÚduration_diffs	            r$   r0   zTvpLoss.loss_durationl   sh   € ô $Ÿi™iÐ(;Ð=RÓSÐÜ$Ÿy™y¨°:Ó>ÐÜŸ™¤U§Y¡Y¬u¯y©yÐ9LÐNbÓ/cÐemÓ%nÓoˆØ%×+Ñ+°Ð+Ó4ˆàÐr#   c                 ó  — |\  }}}t        j                  ||«      }|dd…df   j                  «       |dd…df   j                  «       }}i }	| j                  D ],  }
|	j	                  |
 | j
                  |
   |||||«      i«       Œ. |	S )am  
        This performs the loss computation.

        Args:
            logits (`torch.FloatTensor`):
                The output logits of head module.
            labels (`List[torch.FloatTensor]`):
                List of tensors ([start, end, duration]), which contains start time, end time of the video corresponding to the text, and also the duration.
        Nr   r   )r   ÚmulÚfloatr3   Úupdater1   )r4   r   Úlabelsr+   r;   r<   Ú
candidatesr=   r>   Úlosses_dictr   s              r$   ÚforwardzTvpLoss.forwardw   s¤   € ð *0Ñ&ˆ�*˜hÜ—Y‘Y˜v xÓ0ˆ
Ø5?ÂÀ1ÀÑ5E×5KÑ5KÓ5MÈzÒZ[Ð]^ÐZ^ÑO_×OeÑOeÓOgÐ2ÐàˆØ—K‘Kò 	ˆDØ×ÑØÐ*�t—}‘} TÑ*¨:°xÐAVÐXkÐmuÓvÐwõð	ð
 Ðr#   )
r   r   r   r   r-   r.   r/   r0   rT   Ú__classcell__©r5   s   @r$   r&   r&   >   s!   ø„ ñôòò
ò	ör#   r&   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚTvpVisionModelc           	      ó\  •— t         ‰| �  «        t        |«      | _        |j                  �|j                  j
                  d   }n¼t        | j                  d«      rDt        | j                  j                  d«      r$| j                  j                  j
                  d   }nbt        | j                  d«      rAt        | j                  j                  d«      r!| j                  j                  j                  }nt        d«      ‚t        j                  ||j                  ddddd¬	«      | _        y )
NéÿÿÿÿÚconfigÚhidden_sizesÚhidden_sizezBackbone config not foundr   r   F)Úkernel_sizeÚstrideÚpaddingÚgroupsÚbias)r,   r-   r   ÚbackboneÚbackbone_configr\   Úhasattrr[   r]   r2   r   ÚConv2dÚgrid_encoder_conv)r4   r[   Úin_channelsr5   s      €r$   r-   zTvpVisionModel.__init__�   sç   ø€ Ü‰ÑÔÜ% fÓ-ˆŒà×!Ñ!Ð-Ø ×0Ñ0×=Ñ=¸bÑA‰KÜ�T—]‘] HÔ-´'¸$¿-¹-×:NÑ:NÐP^Ô2_ØŸ-™-×.Ñ.×;Ñ;¸BÑ?‰KÜ�T—]‘] HÔ-´'¸$¿-¹-×:NÑ:NÐP]Ô2^ØŸ-™-×.Ñ.×:Ñ:‰KäÐ8Ó9Ð9ä!#§¡ØØ×ÑØØØØØô"
ˆÕr#   c                 ó¨  — |j                   \  }}}}}|j                  ||z  |||«      }| j                  |«      d   d   }| j                  |«      }t        j
                  j                  |dd¬«      }t        j
                  j                  |d¬«      }|j                   dd  \  }	}
}|j                  |||	|
|«      }|j                  ddd	d
d«      }|S )NÚfeature_mapsr   é   )r^   r_   T)Úinplaceéýÿÿÿr   r   é   )	ÚshapeÚviewrc   rg   r   Ú
functionalÚ
max_pool2dÚreluÚpermute)r4   Úpixel_valuesÚ
batch_sizeÚ
num_framesÚnum_channelsÚheightÚwidthÚgrid_feat_outputsÚgridÚnew_channelÚ
new_heightÚ	new_widths               r$   rT   zTvpVisionModel.forward¦   sÚ   € Ø>J×>PÑ>PÑ;ˆ
�J ¨f°eà#×(Ñ(¨°jÑ)@À,ÐPVÐX]Ó^ˆØ ŸM™M¨,Ó7¸ÑGÈÑJÐØ×%Ñ%Ð&7Ó8ˆÜ�}‰}×'Ñ'¨¸!ÀAÐ'ÓFˆÜ�}‰}×!Ñ! $°Ð!Ó5ˆØ-1¯Z©Z¸¸¨_Ñ*ˆ�Z à�y‰y˜ Z°¸jÈ)ÓTˆà�|‰|˜A˜q ! Q¨Ó*ˆØˆr#   ©r   r   r   r-   rT   rU   rV   s   @r$   rX   rX   Ž   s   ø„ ô
ö.r#   rX   c                   ó~   ‡ — e Zd ZdZˆ fd„Zdej                  dededej                  fd„Zdde	fd	„Z
dde	fd
„Zˆ xZS )ÚTvpVisualInputEmbeddingz;
    Takes input of both image and video (multi-frame)
    c                 ór  •— t         ‰| �  «        t        j                  |j                  |j
                  «      | _        t        j                  |j                  |j
                  «      | _        t        j                  |j                  |j
                  «      | _
        t        j                  d|j
                  «      | _        t        j                  |j
                  |j                  ¬«      | _        t        j                  |j                   «      | _        |j                  | _        |j                  | _	        y )Nr   ©Úeps)r,   r-   r   Ú	EmbeddingÚmax_position_embeddingsr]   Úposition_embeddingsÚ max_grid_row_position_embeddingsÚrow_position_embeddingsÚ max_grid_col_position_embeddingsÚcol_position_embeddingsÚtoken_type_embeddingsÚ	LayerNormÚlayer_norm_epsÚ
layer_normÚDropoutÚhidden_dropout_probÚdropout©r4   r[   r5   s     €r$   r-   z TvpVisualInputEmbedding.__init__»   s×   ø€ Ü‰ÑÔä#%§<¡<°×0NÑ0NÐPV×PbÑPbÓ#cˆÔ Ü')§|¡|°F×4[Ñ4[Ð]c×]oÑ]oÓ'pˆÔ$Ü')§|¡|°F×4[Ñ4[Ð]c×]oÑ]oÓ'pˆÔ$Ü%'§\¡\°!°V×5GÑ5GÓ%HˆÔ"ÜŸ,™, v×'9Ñ'9¸v×?TÑ?TÔUˆŒÜ—z‘z &×"<Ñ"<Ó=ˆŒØ06×0WÑ0WˆÔ-Ø06×0WÑ0WˆÕ-r#   Ú	embeddingry   rz   Úreturnc                 ó   — dx}}|| j                   kD  r|| j                   z  }|| j                  kD  r|| j                  z  }|j                  dddd«      }t        j                  j                  |||fdd¬«      }|j                  dddd«      }|S )z¸
        This method allows to interpolate the pre-trained pad weights , to be able to use the model on collection of high
        resolution images (high resolution videos).

        r   r   r   rk   ÚbicubicF©Úscale_factorÚmodeÚalign_corners)r‰   r‹   rt   r   rq   Úinterpolate)r4   r•   ry   rz   Úh0Úw0s         r$   Úinterpolate_pos_encodingz0TvpVisualInputEmbedding.interpolate_pos_encodingÇ   s¦   € ð ˆˆˆRà�D×9Ñ9Ò9Ø˜$×?Ñ?Ñ?ˆBà�4×8Ñ8Ò8Ø˜×>Ñ>Ñ>ˆBØ×%Ñ% a¨¨A¨qÓ1ˆ	Ü—M‘M×-Ñ-ØØ˜b˜ØØð	 .ó 
ˆ	ð ×%Ñ% a¨¨A¨qÓ1ˆ	ØÐr#   r    c                 ó’  — |j                   \  }}}}t        | j                  |«      }t        j                  |t        j
                  |j                  ¬«      }| j                  |«      }	dt        |j                   «      dz
  z  |d|fz   }
 |	j                  |
Ž }	t        | j                  |«      }t        j                  |t        j
                  |j                  ¬«      }| j                  |«      }|d||f} |j                  |Ž }|	|z   }|r6|| j                  kD  s|| j                  kD  r|| j                  |||«      z   }|S ||z   }|S )af  
        Args:
            grid: (batch_size, height, width, hidden_dim)
            interpolate_pos_encoding: (`bool`, *optional*, defaults to `False`):
                Whether to interpolate the pre-trained position encodings.
        Returns:
            grid + col_position_embeddings.view(*col_shape): (batch_size, *, height, width, hidden_dim)
        ©ÚdtypeÚdevice)r   r   r   )ro   r8   r‰   r   ÚarangeÚlongr¤   rŠ   Úlenrp   r‹   rŒ   r    )r4   r|   r    rv   ry   rz   Ú
hidden_dimÚ
row_heightÚrow_position_idsrŠ   Ú	row_shapeÚ	row_widthÚcol_position_idsrŒ   Ú	col_shapeÚpositional_embeddingss                   r$   Úadd_2d_positional_embeddingsz4TvpVisualInputEmbedding.add_2d_positional_embeddingsÞ   sQ  € ð 15·
±
Ñ-ˆ
�F˜E :ô ˜×>Ñ>ÀÓGˆ
Ü Ÿ<™<¨
¼%¿*¹*ÈTÏ[É[ÔYÐà"&×">Ñ">Ð?OÓ"PÐØœC §
¡
›O¨aÑ/Ñ0°JÀÀ:Ð3NÑNˆ	à">Ð"9×">Ñ">À	Ð"JÐô ˜×=Ñ=¸uÓEˆ	Ü Ÿ<™<¨	¼¿¹ÈDÏKÉKÔXÐà"&×">Ñ">Ð?OÓ"PÐØ  I¨zÐ:ˆ	à">Ð"9×">Ñ">À	Ð"JÐà 7Ð:QÑ QÐñ $Ø�T×:Ñ:Ò:¸eÀd×FkÑFkÒ>kà˜$×7Ñ7Ð8MÈvÐW\Ó]Ñ]ˆDð ˆð Ð/Ñ/ˆDØˆr#   c                 óŠ  — |j                   \  }}}}}|j                  d«      }| j                  ||¬«      }|j                  |d|«      }|j                   dd }	|j                  }
t        j                  |	t
        j                  |
¬«      }| j                  |«      }||z   }| j                  |«      }| j                  |«      }|S )a  
        Args:
            grid: Array of shape (batch_size, num_frames, height, width, num_channels).
                It contains processed frames extracted from videos, and is generated by Tvp image preprocessor. Note,
                num_frames can be 1
            interpolate_pos_encoding: (bool, *optional*, defaults to `False`):
                Whether to interpolate the pre-trained position encodings.

        Returns:
            embeddings: The embedding of grid with size (batch_size, height*width, num_channels)

        r   ©r    rZ   Nr¢   )ro   Úmeanr°   rp   r¤   r   Úzerosr¦   r�   r�   r“   )r4   r|   r    rv   rw   ry   rz   rx   Úvisual_tokensÚvisual_tokens_shaper¤   Útoken_type_idsr�   Ú
embeddingss                 r$   rT   zTvpVisualInputEmbedding.forward  sÄ   € ð ?C¿j¹jÑ;ˆ
�J ¨¨|à�y‰y˜‹|ˆØ×0Ñ0°ÐPhÐ0ÓiˆàŸ	™	 *¨b°,Ó?ˆØ+×1Ñ1°#°2Ð6ÐØ×%Ñ%ˆô Ÿ™Ð%8ÄÇ
Á
ÐSYÔZˆØ $× :Ñ :¸>Ó JÐà"Ð%:Ñ:ˆ
Ø—_‘_ ZÓ0ˆ
Ø—\‘\ *Ó-ˆ
ØÐr#   ©F)r   r   r   r   r-   r   ÚTensorÚintr    Úboolr°   rT   rU   rV   s   @r$   r‚   r‚   ¶   sT   ø„ ñô
Xð°%·,±,ð Èð ÐTWð Ð\a×\hÑ\hó ñ.'È4ó 'ñR°d÷ r#   r‚   c                   ó*   ‡ — e Zd ZdZˆ fd„Zdd„Zˆ xZS )ÚTvpTextInputEmbeddingszGConstruct the embeddings from word, position and token_type embeddings.c                 óü  •— t         ‰| �  «        t        j                  |j                  |j
                  |j                  ¬«      | _        t        j                  |j                  |j
                  «      | _	        t        j                  |j                  |j
                  «      | _        t        j                  |j
                  |j                  ¬«      | _        t        j                  |j                   «      | _        y )N)Úpadding_idxr„   )r,   r-   r   r†   Ú
vocab_sizer]   Úpad_token_idÚword_embeddingsr‡   rˆ   Útype_vocab_sizer�   rŽ   r�   r�   r‘   r’   r“   r”   s     €r$   r-   zTvpTextInputEmbeddings.__init__*  s©   ø€ Ü‰ÑÔÜ!Ÿ|™|¨F×,=Ñ,=¸v×?QÑ?QÐ_e×_rÑ_rÔsˆÔÜ#%§<¡<°×0NÑ0NÐPV×PbÑPbÓ#cˆÔ Ü%'§\¡\°&×2HÑ2HÈ&×J\ÑJ\Ó%]ˆÔ"ÜŸ,™, v×'9Ñ'9¸v×?TÑ?TÔUˆŒÜ—z‘z &×"<Ñ"<Ó=ˆ�r#   c                 ó.  — |�|j                  «       }n|j                  «       d d }|d   }|�|j                  n|j                  }|€Ft        j                  |t        j                  |¬«      }|j                  d«      j                  |«      }|€&t        j                  |t        j                  |¬«      }|€| j                  |«      }| j                  |«      }| j                  |«      }	||z   |	z   }
| j                  |
«      }
| j                  |
«      }
|
S )NrZ   r   r¢   r   )Úsizer¤   r   r¥   r¦   Ú	unsqueezeÚexpandr´   rÃ   rˆ   r�   r�   r“   )r4   Ú	input_idsr·   Úposition_idsÚinputs_embedsÚinput_shapeÚ
seq_lengthr¤   rˆ   r�   r¸   s              r$   rT   zTvpTextInputEmbeddings.forward2  s  € ØÐ Ø#Ÿ.™.Ó*‰Kà'×,Ñ,Ó.¨s°Ð3ˆKà  ‘^ˆ
Ø%.Ð%:�×!Ò!À×@TÑ@TˆØÐÜ Ÿ<™<¨
¼%¿*¹*ÈVÔTˆLØ'×1Ñ1°!Ó4×;Ñ;¸KÓHˆLØÐ!Ü"Ÿ[™[¨¼E¿J¹JÈvÔVˆNàÐ Ø ×0Ñ0°Ó;ˆMØ"×6Ñ6°|ÓDÐØ $× :Ñ :¸>Ó JÐà"Ð%8Ñ8Ð;PÑPˆ
Ø—_‘_ ZÓ0ˆ
Ø—\‘\ *Ó-ˆ
ØÐr#   )NNNN©r   r   r   r   r-   rT   rU   rV   s   @r$   r¾   r¾   '  s   ø„ ÙQô>÷r#   r¾   c                   óf   ‡ — e Zd Zˆ fd„Zd„ Zdej                  dedefd„Z	 	 	 d	de	e
   fd„Zˆ xZS )
ÚTvpAttentionc                 óî  •— t         ‰| �  «        |j                  |j                  z  dk7  r1t	        |d«      s%t        d|j                  › d|j                  › �«      ‚|j                  | _        t        |j                  |j                  z  «      | _        | j                  | j                  z  | _        t        j                  |j                  | j                  «      | _        t        j                  |j                  | j                  «      | _        t        j                  |j                  | j                  «      | _        t        j                  |j                  «      | _        t        j                  |j                  |j                  «      | _        t        j$                  |j                  |j&                  ¬«      | _        t        j                  |j*                  «      | _        t/        «       | _        y )Nr   Úembedding_sizezThe hidden size z4 is not a multiple of the number of attention heads r„   )r,   r-   r]   Únum_attention_headsre   r2   r»   Úattention_head_sizeÚall_head_sizer   ÚLinearÚqueryÚkeyÚvaluer‘   Úattention_probs_dropout_probÚattn_dropoutÚdenserŽ   r�   r�   r’   r“   ÚsetÚpruned_headsr”   s     €r$   r-   zTvpAttention.__init__L  s‚  ø€ Ü‰ÑÔØ×Ñ × :Ñ :Ñ:¸aÒ?ÌÐPVÐXhÔHiÜØ" 6×#5Ñ#5Ð"6Ð6jÐkq÷  lFñ  lFð  kGð  Hóð ð $*×#=Ñ#=ˆÔ Ü#& v×'9Ñ'9¸F×<VÑ<VÑ'VÓ#WˆÔ Ø!×5Ñ5¸×8PÑ8PÑPˆÔä—Y‘Y˜v×1Ñ1°4×3EÑ3EÓFˆŒ
Ü—9‘9˜V×/Ñ/°×1CÑ1CÓDˆŒÜ—Y‘Y˜v×1Ñ1°4×3EÑ3EÓFˆŒ
ÜŸJ™J v×'JÑ'JÓKˆÔä—Y‘Y˜v×1Ñ1°6×3EÑ3EÓFˆŒ
ÜŸ,™, v×'9Ñ'9¸v×?TÑ?TÔUˆŒÜ—z‘z &×"<Ñ"<Ó=ˆŒÜ›EˆÕr#   c                 óN  ‡— t        |«      dk(  ry t        j                  | j                  | j                  «      }t        |«      | j                  z
  }|D ](  Š‰t        ˆfd„| j                  D «       «      z
  Šd|‰<   Œ* |j                  d«      j                  «       j                  d«      }t        j                  t        |«      «      |   j                  «       }t        | j                  |«      | _        t        | j                  |«      | _        t        | j                   |«      | _        t        | j"                  |d¬«      | _        | j                  t        |«      z
  | _        | j                  | j                  z  | _        | j                  j'                  |«      | _        y )Nr   c              3   ó0   •K  — | ]  }|‰k  rd nd–— Œ y­w)r   r   Nr"   )Ú.0ÚhÚheads     €r$   ú	<genexpr>z+TvpAttention.prune_heads.<locals>.<genexpr>h  s   øè ø€ ÒN°q 1 t¢8™a°Ó2ÑNùs   ƒrZ   r   ©Údim)r§   r   ÚonesrÓ   rÔ   rÝ   rÞ   Úsumrp   Ú
contiguousÚeqr¥   r¦   r   r×   rØ   rÙ   rÜ   rÕ   r@   )r4   ÚheadsÚmaskÚindexrã   s       @r$   Úprune_headszTvpAttention.prune_headsa  sN  ø€ Üˆu‹:˜Š?ØÜ�z‰z˜$×2Ñ2°D×4LÑ4LÓMˆÜ�E“
˜T×.Ñ.Ñ.ˆØò 	ˆDàœ#ÓN¸D×<MÑ<MÔNÓNÑNˆDØˆD�ŠJð	ð �y‰y˜‹}×'Ñ'Ó)×,Ñ,¨QÓ/ˆÜ—‘œS ›YÓ'¨Ñ-×2Ñ2Ó4ˆô (¨¯
©
°EÓ:ˆŒ
Ü% d§h¡h°Ó6ˆŒÜ'¨¯
©
°EÓ:ˆŒ
Ü'¨¯
©
°E¸qÔAˆŒ
ð $(×#;Ñ#;¼cÀ%»jÑ#HˆÔ Ø!×5Ñ5¸×8PÑ8PÑPˆÔØ ×-Ñ-×3Ñ3°EÓ:ˆÕr#   ÚtensorÚsequence_lengthrv   c                 óŽ   — |j                  ||| j                  | j                  «      j                  dd«      j	                  «       S )Nr   rk   )rp   rÓ   rÔ   Ú	transposeré   )r4   rï   rð   rv   s       r$   Ú_reshapezTvpAttention._reshapex  s7   € à�K‰K˜
 O°T×5MÑ5MÈt×OgÑOgÓhß‰Y�q˜!‹_ß‰Z‹\ð	
r#   Úoutput_attentionsc                 ó:  — |j                   d d \  }}| j                  |«      }| j                  |«      }| j                  |«      }	| j	                  |||«      }
| j	                  |||«      }| j	                  |	||«      }t        j                  |
|j                  dd«      «      }|t        j                  | j                  «      z  }|�||z   }t        j                  j                  |d¬«      }| j                  |«      }|�||z  }t        j                  ||«      }|j                  dd«      j                  «       }|j!                  ||| j"                  «      }| j%                  |«      }| j'                  |«      }| j)                  ||z   «      }|r||f}|S |f}|S )Nrk   rZ   éþÿÿÿrå   r   )ro   r×   rØ   rÙ   ró   r   Úmatmulrò   ÚmathÚsqrtrÔ   r   rq   ÚsoftmaxrÛ   ré   ÚreshaperÕ   rÜ   r“   r�   )r4   r   Úattention_maskÚ	head_maskrô   rv   rð   Úmixed_query_layerÚmixed_key_layerÚmixed_value_layerÚquery_layerÚ	key_layerÚvalue_layerÚattention_scoresÚattention_probsÚattn_outputÚoutputss                    r$   rT   zTvpAttention.forward  sŸ  € ð '4×&9Ñ&9¸"¸1Ð&=Ñ#ˆ
�OØ ŸJ™J }Ó5ÐàŸ(™( =Ó1ˆØ ŸJ™J }Ó5Ðà—m‘mÐ$5°È
ÓSˆØ—M‘M /°?ÀJÓOˆ	Ø—m‘mÐ$5°È
ÓSˆô !Ÿ<™<¨°Y×5HÑ5HÈÈRÓ5PÓQÐØ+¬d¯i©i¸×8PÑ8PÓ.QÑQÐØÐ%Ø/°.Ñ@Ðô Ÿ-™-×/Ñ/Ð0@ÀbÐ/ÓIˆð ×+Ñ+¨OÓ<ˆð Ð Ø-°	Ñ9ˆOä—l‘l ?°KÓ@ˆØ!×+Ñ+¨A¨qÓ1×<Ñ<Ó>ˆØ!×)Ñ)¨*°oÀt×GYÑGYÓZˆà—j‘j Ó-ˆØ—l‘l ;Ó/ˆØ—o‘o k°MÑ&AÓBˆá4E�; Ð0ˆØˆð MXÈ>ˆØˆr#   ©NNN)r   r   r   r-   rî   r   rº   r»   ró   r   r¼   rT   rU   rV   s   @r$   rÐ   rÐ   K  sI   ø„ ô"ò*;ð.
˜uŸ|™|ð 
¸cð 
Èsó 
ð ØØ,0ñ+ð
 $ D™>÷+r#   rÐ   c                   óV   ‡ — e Zd Zˆ fd„Zdej
                  dej
                  fd„Zˆ xZS )ÚTvpIntermediatec                 ó  •— t         ‰| �  «        t        j                  |j                  |j
                  «      | _        t        |j                  t        «      rt        |j                     | _        y |j                  | _        y ©N)r,   r-   r   rÖ   r]   Úintermediate_sizerÜ   Ú
isinstanceÚ
hidden_actÚstrr   Úintermediate_act_fnr”   s     €r$   r-   zTvpIntermediate.__init__¯  s]   ø€ Ü‰ÑÔÜ—Y‘Y˜v×1Ñ1°6×3KÑ3KÓLˆŒ
Ü�f×'Ñ'¬Ô-Ü'-¨f×.?Ñ.?Ñ'@ˆDÕ$à'-×'8Ñ'8ˆDÕ$r#   r   r–   c                 óJ   — | j                  |«      }| j                  |«      }|S r  )rÜ   r  )r4   r   s     r$   rT   zTvpIntermediate.forward·  s&   € ØŸ
™
 =Ó1ˆØ×0Ñ0°Ó?ˆØÐr#   ©r   r   r   r-   r   rº   rT   rU   rV   s   @r$   r
  r
  ®  s#   ø„ ô9ð U§\¡\ð °e·l±l÷ r#   r
  c                   ón   ‡ — e Zd Zˆ fd„Zdej
                  dej
                  dej
                  fd„Zˆ xZS )ÚTvpOutputLayerc                 ó(  •— t         ‰| �  «        t        j                  |j                  |j
                  «      | _        t        j                  |j
                  |j                  ¬«      | _	        t        j                  |j                  «      | _        y )Nr„   )r,   r-   r   rÖ   r  r]   rÜ   rŽ   r�   r�   r‘   r’   r“   r”   s     €r$   r-   zTvpOutputLayer.__init__¾  s`   ø€ Ü‰ÑÔÜ—Y‘Y˜v×7Ñ7¸×9KÑ9KÓLˆŒ
ÜŸ,™, v×'9Ñ'9¸v×?TÑ?TÔUˆŒÜ—z‘z &×"<Ñ"<Ó=ˆ�r#   r   Úinput_tensorr–   c                 ór   — | j                  |«      }| j                  |«      }| j                  ||z   «      }|S r  )rÜ   r“   r�   )r4   r   r  s      r$   rT   zTvpOutputLayer.forwardÄ  s7   € ØŸ
™
 =Ó1ˆØŸ™ ]Ó3ˆØŸ™¨¸Ñ(DÓEˆØÐr#   r  rV   s   @r$   r  r  ½  s1   ø„ ô>ð U§\¡\ð ÀÇÁð ÐRW×R^ÑR^÷ r#   r  c                   ó8   ‡ — e Zd Zˆ fd„Z	 	 	 ddee   fd„Zˆ xZS )ÚTvpEncodeLayerc                 ó‚   •— t         ‰| �  «        t        |«      | _        t	        |«      | _        t        |«      | _        y r  )r,   r-   rÐ   Ú	attentionr
  Úintermediater  Úoutputr”   s     €r$   r-   zTvpEncodeLayer.__init__Ì  s3   ø€ Ü‰ÑÔÜ% fÓ-ˆŒÜ+¨FÓ3ˆÔÜ$ VÓ,ˆ�r#   rô   c                 ó–   — | j                  ||||¬«      }|d   }|dd  }| j                  |«      }| j                  ||«      }	|	f|z   }|S )N)rô   r   r   )r  r  r  )
r4   r   rü   rý   rô   Úself_attention_outputsÚattention_outputr  Úintermediate_outputÚlayer_outputs
             r$   rT   zTvpEncodeLayer.forwardÒ  sr   € ð "&§¡ØØØØ/ð	 "0ó "
Ðð 2°!Ñ4ÐØ(¨¨Ð,ˆØ"×/Ñ/Ð0@ÓAÐØ—{‘{Ð#6Ð8HÓIˆØ�/ GÑ+ˆØˆr#   r  )r   r   r   r-   r   r¼   rT   rU   rV   s   @r$   r  r  Ë  s&   ø„ ô-ð ØØ,0ñð
 $ D™>÷r#   r  c            
       ón   ‡ — e Zd Zˆ fd„Z	 	 	 	 	 ddeej                     dee   dee   dee   fd„Zˆ xZ	S )Ú
TvpEncoderc                 óÐ   •— t         ‰| �  «        || _        t        j                  t        |j                  «      D �cg c]  }t        |«      ‘Œ c}«      | _        d| _	        y c c}w )NF)
r,   r-   r[   r   Ú
ModuleListÚrangeÚnum_hidden_layersr  ÚlayerÚgradient_checkpointing)r4   r[   Ú_r5   s      €r$   r-   zTvpEncoder.__init__è  sN   ø€ Ü‰ÑÔØˆŒÜ—]‘]ÄEÈ&×JbÑJbÓDcÖ#d¸q¤N°6Õ$:Ò#dÓeˆŒ
Ø&+ˆÕ#ùò $es   ½A#rý   rô   Úoutput_hidden_statesÚreturn_dictc                 ó   — |�|n| j                   j                  }|�|n| j                   j                  }|�|n| j                   j                  }d}d}t	        | j
                  «      D ]k  \  }	}
|r||fz   }| j                  r3| j                  r'| j                  |
j                  |||�||	   nd |«      }n |
||||	   |«      }|d   }|sŒc||d   fz   }Œm |r||fz   }|s|f}|r||fz   }|r||fz   }|S t        ||r|nd |r|¬«      S d ¬«      S )Nr"   r   r   )Úlast_hidden_stater   r   )r[   r.  rô   r-  Ú	enumerater*  r+  ÚtrainingÚ_gradient_checkpointing_funcÚ__call__r   )r4   r   rü   rý   rô   r-  r.  Úall_hidden_statesÚall_attentionsÚiÚlayer_moduleÚlayer_outputsr  s                r$   rT   zTvpEncoder.forwardî  ss  € ð &1Ð%<‘kÀ$Ç+Á+×BYÑBYˆØ1BÐ1NÑ-ÐTX×T_ÑT_×TqÑTqÐà$8Ð$DÑ È$Ï+É+×JjÑJjð 	ð ÐØˆä(¨¯©Ó4ò 	F‰OˆAˆ|Ù#Ø$5¸Ð8HÑ$HÐ!à×*Ò*¨t¯}ª}Ø $× AÑ AØ ×)Ñ)Ø!Ø"Ø%.Ð%:�Y˜q’\ÀØ%ó!‘ñ !-¨]¸NÈIÐVWÉLÐZkÓ l�à)¨!Ñ,ˆMÚ Ø!/°=ÀÑ3CÐ2EÑ!E‘ð#	Fñ(  Ø 1°]Ð4DÑ DÐáØ$Ð&ˆGÙ#Ø!Ð%6Ð$8Ñ8�Ù Ø! ^Ð$5Ñ5�ØˆNäØ+Ù/CÑ+ÈÙ):�~ô
ð 	
ð AEô
ð 	
r#   )NNNNN)
r   r   r   r-   r   r   r    r¼   rT   rU   rV   s   @r$   r%  r%  ç  s]   ø„ ô,ð Ø15Ø,0Ø/3Ø&*ñ4
ð ˜E×-Ñ-Ñ.ð	4
ð
 $ D™>ð4
ð ' t™nð4
ð ˜d‘^÷4
r#   r%  c                   óV   ‡ — e Zd Zˆ fd„Zdej
                  dej
                  fd„Zˆ xZS )Ú	TvpPoolerc                 ó²   •— t         ‰| �  «        t        j                  |j                  |j                  «      | _        t        j                  «       | _        y r  )r,   r-   r   rÖ   r]   rÜ   ÚTanhÚ
activationr”   s     €r$   r-   zTvpPooler.__init__'  s9   ø€ Ü‰ÑÔÜ—Y‘Y˜v×1Ñ1°6×3EÑ3EÓFˆŒ
ÜŸ'™'›)ˆ�r#   r   r–   c                 ó\   — |d d …df   }| j                  |«      }| j                  |«      }|S )Nr   )rÜ   r>  )r4   r   Úfirst_token_tensorÚpooled_outputs       r$   rT   zTvpPooler.forward,  s6   € ð +ª1¨a¨4Ñ0ÐØŸ
™
Ð#5Ó6ˆØŸ™¨Ó6ˆØÐr#   r  rV   s   @r$   r;  r;  &  s#   ø„ ô$ð
 U§\¡\ð °e·l±l÷ r#   r;  c                   ó"   — e Zd ZdZeZdZdZd„ Zy)ÚTvpPreTrainedModelz�An abstract class to handle weights initialization and
    a simple interface for downloading and loading pretrained models.
    ÚmodelTc                 ó&  — t        |t        j                  t        j                  f«      r<|j                  j
                  j                  d| j                  j                  ¬«       nct        |t        j                  «      rI|j                  j
                  j                  «        |j                  j
                  j                  d«       t        |t        j                  «      r0|j                  �$|j                  j
                  j                  «        t        |t        j                  «      rdt        j                  j                  |j                  dd¬«       |j                  �+t        j                  j!                  |j                  d«       yyy)	zInitialize the weightsç        )r³   Ústdg      ð?NÚfan_outrs   )r›   Únonlinearityr   )r  r   rÖ   r†   ÚweightÚdataÚnormal_r[   Úinitializer_rangerŽ   rb   Úzero_Úfill_rf   ÚinitÚkaiming_normal_Ú	constant_)r4   Úmodules     r$   Ú_init_weightsz TvpPreTrainedModel._init_weights>  s  € ä�fœrŸy™y¬"¯,©,Ð7Ô8ð �M‰M×Ñ×&Ñ&¨C°T·[±[×5RÑ5RÐ&ÕSÜ˜¤§¡Ô-Ø�K‰K×Ñ×"Ñ"Ô$Ø�M‰M×Ñ×$Ñ$ SÔ)ä�fœbŸi™iÔ(¨V¯[©[Ð-DØ�K‰K×Ñ×"Ñ"Ô$ä�fœbŸi™iÔ(Ü�G‰G×#Ñ# F§M¡M¸	ÐPVÐ#ÔWØ�{‰{Ð&Ü—‘×!Ñ! &§+¡+¨qÕ1ð 'ð )r#   N)	r   r   r   r   r   Úconfig_classÚbase_model_prefixÚsupports_gradient_checkpointingrT  r"   r#   r$   rC  rC  5  s   „ ñð €LØÐØ&*Ð#ó2r#   rC  aE  
    This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. Use it
    as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and
    behavior.

    Parameters:
        config ([`TvpConfig`]): Model configuration class with all the parameters of the model.
            Initializing with a config file does not load the weights associated with the model, only the
            configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
aâ  
    Args:
        input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
            Indices of input sequence tokens in the vocabulary. Indices can be obtained using [`AutoTokenizer`]. See
            [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input
            IDs?](../glossary#input-ids)

        pixel_values (`torch.FloatTensor` of shape `(batch_size, num_frames, num_channels, height, width)`):
            Pixel values. Pixel values can be obtained using [`TvpImageProcessor`]. See [`TvpImageProcessor.__call__`]
            for details.

        attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:
            - 1 for tokens that are **not masked**,
            - 0 for tokens that are **masked**.
            [What are attention masks?](../glossary#attention-mask)

        head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*):
            Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`:
            - 1 indicates the head is **not masked**,
            - 0 indicates the head is **masked**.

        output_attentions (`bool`, *optional*):
            Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
            tensors for more detail.

        output_hidden_states (`bool`, *optional*):
            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
            more detail.

        return_dict (`bool`, *optional*):
            Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.

        interpolate_pos_encoding (`bool`, *optional*, defaults to `False`):
            Whether to interpolate the pre-trained image pad prompter encodings and positional encodings.
c                   ó(   ‡ — e Zd ZdZˆ fd„Zd„ Zˆ xZS )ÚTvpFrameDownPadPrompterz>
    Pad frames extracted from videos only at the bottom.
    c           	      ó|  •— |j                   dvrt        d«      ‚t        ‰| �  «        |j                  | _        |j
                  | _        |j                  | _        |j                   | _         t        j                  t        j                  d|j
                  d|j                  |j                  g«      «      | _        y )N©rC   ÚreplaceÚremoveú9`visual_prompter_apply` must be in (add, replace, remove)r   r   )Úvisual_prompter_applyr2   r,   r-   Úvisual_prompt_sizeÚ	frame_numÚmax_img_sizer   Ú	Parameterr   ÚrandnÚpad_downr”   s     €r$   r-   z TvpFrameDownPadPrompter.__init__‡  s™   ø€ Ø×'Ñ'Ð/KÑKÜÐXÓYÐYä‰ÑÔØ"(×";Ñ";ˆÔØ×)Ñ)ˆŒØ"×/Ñ/ˆÔØ%+×%AÑ%AˆÔ"äŸ™Ü�K‰K˜˜F×,Ñ,¨a°×1JÑ1JÈF×L_ÑL_Ð`Óaó
ˆ�r#   c                 óŽ  — | j                   dk7  rst        j                  | j                  | j                  g|j                  |j
                  ¬«      }d|| j                  | j                  z
  | j                  …d d …f<   ||z  }| j                   dk7  r³t        j                  |j                  d   |j                  d   d| j                  | j                  g|j
                  ¬«      }| j                  | j                  z
  }| j                  |d d …d d …d d …|| j                  …d d …f<   ||j                  |j                  «      z  }|S )	NrC   r¢   rF  r]  r   r   r   ©r¤   )r_  r   rç   rb  r£   r¤   r`  r´   ro   re  Úto)r4   ru   Úvisual_prompt_maskÚpromptÚstart_points        r$   rT   zTvpFrameDownPadPrompter.forward•  s1  € Ø×%Ñ%¨Ò.Ü!&§¡Ø×"Ñ" D×$5Ñ$5Ð6¸l×>PÑ>PÐYe×YlÑYlô"Ðð fiÐ˜t×0Ñ0°4×3JÑ3JÑJÈT×M^ÑM^Ð^Ò`aÐaÑbØÐ.Ñ.ˆLØ×%Ñ%¨Ò1Ü—[‘[Ø×#Ñ# AÑ&¨×(:Ñ(:¸1Ñ(=¸qÀ$×BSÑBSÐUY×UfÑUfÐgØ#×*Ñ*ôˆFð ×+Ñ+¨d×.EÑ.EÑEˆKØBFÇ-Á-ˆF’1’aš˜K¨$×*;Ñ*;Ð;ºQÐ>Ñ?Ø˜FŸI™I l×&8Ñ&8Ó9Ñ9ˆLØÐr#   rÎ   rV   s   @r$   rY  rY  ‚  s   ø„ ñô
ör#   rY  c                   óp   ‡ — e Zd ZdZˆ fd„Zdej                  dededej                  fd„Zd
de	fd	„Z
ˆ xZS )ÚTvpFramePadPrompterz?
    Pad frames extracted from videos in the surroundings.
    c           
      ó¼  •— |j                   dvrt        d«      ‚t        ‰| �  «        |j                  | _        |j
                  | _        |j                   | _         |j
                  |j                  dz  z
  | _        t        j                  t        j                  d|j                  d|j                  |j
                  g«      «      | _        t        j                  t        j                  d|j                  d|j                  |j
                  g«      «      | _        t        j                  t        j                  d|j                  d|j
                  |j                  dz  z
  |j                  g«      «      | _        t        j                  t        j                  d|j                  d|j
                  |j                  dz  z
  |j                  g«      «      | _        y )Nr[  r^  rk   r   r   )r_  r2   r,   r-   rw   rb  r`  Ú	base_sizer   rc  r   rd  Úpad_upre  Úpad_leftÚ	pad_rightr”   s     €r$   r-   zTvpFramePadPrompter.__init__¬  s‹  ø€ Ø×'Ñ'Ð/KÑKÜÐXÓYÐYä‰ÑÔØ ×+Ñ+ˆŒØ"×/Ñ/ˆÔØ%+×%AÑ%AˆÔ"Ø×,Ñ,¨v×/HÑ/HÈ1Ñ/LÑLˆŒÜ—l‘lÜ�K‰K˜˜F×-Ñ-¨q°&×2KÑ2KÈV×M`ÑM`ÐaÓbó
ˆŒô Ÿ™Ü�K‰K˜˜F×-Ñ-¨q°&×2KÑ2KÈV×M`ÑM`ÐaÓbó
ˆŒô Ÿ™Ü�K‰KàØ×%Ñ%ØØ×'Ñ'¨&×*CÑ*CÀaÑ*GÑGØ×-Ñ-ðóó

ˆŒô Ÿ™Ü�K‰KàØ×%Ñ%ØØ×'Ñ'¨&×*CÑ*CÀaÑ*GÑGØ×-Ñ-ðóó

ˆ�r#   rj  ry   rz   r–   c                 ó  — || j                   z  || j                   z  }}|j                  \  }}}}	}
|j                  ||z  ||	|
«      }t        j                  j                  |||fdd¬«      }|j                  |||||«      }|S )z·
        This method allows to interpolate the pre-trained pad weights, to be able to use the model on collection of high
        resolution images (high resolution videos).

        r˜   Fr™   )rb  ro   rû   r   rq   r�   )r4   rj  ry   rz   rž   rŸ   Úbatchrw   ÚchannelsÚprompt_heightÚprompt_widths              r$   Úinterpolate_pad_encodingz,TvpFramePadPrompter.interpolate_pad_encodingÒ  s˜   € ð ˜$×+Ñ+Ñ+¨U°T×5FÑ5FÑ-FˆBˆàCIÇ<Á<Ñ@ˆˆz˜8 ]°Lð —‘ ¨
Ñ 2°H¸mÈ\ÓZˆÜ—‘×*Ñ*ØØ˜b˜ØØð	 +ó 
ˆð —‘  z°8¸VÀUÓKˆØˆr#   rx  c                 óZ  — |r|j                   d   |j                   d   fn| j                  | j                  f\  }}| j                  dvrt        d| j                  › �«      ‚| j                  dv r3t	        j
                  ||g|j                  |j                  ¬«      }||z  }| j                  dv rút	        j                  d| j                  d	| j                  | j                  |j                  ¬
«      }t	        j                  | j                  || j                  gd¬«      }t	        j                  | j                  || j                  gd	¬«      }t	        j                  |j!                  d«      |gz  «      }|r| j#                  |||«      }||j%                  |j                  «      z   }|S )Nrö   rZ   )rC   r]  r\  z$Invalid visual_prompter_apply value )r\  r]  r¢   )r\  rC   r   r   rg  rn   rå   r   )ro   rb  r_  r2   r   rç   r£   r¤   r´   rw   ro  Úcatrq  rr  rp  re  rÆ   rx  rh  )r4   ru   rx  ry   rz   ri  Úbaserj  s           r$   rT   zTvpFramePadPrompter.forwardê  s{  € ñ (ð ×Ñ Ñ# \×%7Ñ%7¸Ñ%;Ñ<à×#Ñ# T×%6Ñ%6Ð7ñ 	ˆ�ð
 ×%Ñ%Ð-IÑIÜÐCÀD×D^ÑD^ÐC_Ð`ÓaÐaØ×%Ñ%Ð)>Ñ>Ü!&§¡¨V°U¨OÀ<×CUÑCUÐ^j×^qÑ^qÔ!rÐØÐ.Ñ.ˆLØ×%Ñ%Ð);Ñ;Ü—;‘;˜q $§/¡/°1°d·n±nÀdÇnÁnÐ]i×]pÑ]pÔqˆDä—Y‘Y §¡¨t°T·^±^ÐDÈ!ÔLˆFÜ—Y‘Y §¡¨V°T·]±]ÐCÈÔKˆFÜ—Y‘Y˜|×0Ñ0°Ó3°v°hÑ>Ó?ˆFÙ'Ø×6Ñ6°v¸vÀuÓM�Ø'¨&¯)©)°L×4FÑ4FÓ*GÑGˆLØÐr#   r¹   )r   r   r   r   r-   r   rº   r»   rx  r¼   rT   rU   rV   s   @r$   rm  rm  §  sG   ø„ ñô$
ðL¨u¯|©|ð ÀSð ÐQTð ÐY^×YeÑYeó ñ0¸d÷ r#   rm  )ÚframedownpadÚframepadzmThe bare Tvp Model transformer outputting BaseModelOutputWithPooling object without any specific head on top.c                   ó  ‡ — e Zd Zˆ fd„Zd„ Zd„ Zd„ Z ee«       e	e
e¬«      	 	 	 	 	 	 	 	 ddeej                     deej                     deej                     d	eej                     d
ee   dee   dee   defd„«       «       Zˆ xZS )ÚTvpModelc                 ó"  •— t         ‰| �  |«       || _        t        |«      | _        t        |«      | _        t        |«      | _        t        |«      | _
        t        |«      | _        t        j                  t        j                   dd|j"                  g«      «      | _        t        j&                  |j(                  «      | _        |j,                  t.        vrt1        d«      ‚t/        |j,                     |«      | _        | j5                  «        y )Nr   é
   z:`visual_prompter_type` must be in (framedownpad, framepad))r,   r-   r[   rX   Úvision_modelr¾   r¸   r‚   Úvisual_embeddingsr%  Úencoderr;  Úpoolerr   rc  r   rd  r]   Útext_promptr‘   r’   r“   Úvisual_prompter_typeÚTVP_PROMPTER_CLASSES_MAPPINGr2   Úvisual_prompterÚ	post_initr”   s     €r$   r-   zTvpModel.__init__  sÐ   ø€ Ü‰Ñ˜Ô ØˆŒÜ*¨6Ó2ˆÔÜ0°Ó8ˆŒÜ!8¸Ó!@ˆÔÜ! &Ó)ˆŒÜ Ó'ˆŒÜŸ<™<¬¯©°Q¸¸F×<NÑ<NÐ4OÓ(PÓQˆÔÜ—z‘z &×"<Ñ"<Ó=ˆŒØ×&Ñ&Ô.JÑJÜÐYÓZÐZÜ;¸F×<WÑ<WÑXÐY_Ó`ˆÔà�‰Õr#   c                 ó.   — | j                   j                  S r  ©r¸   rÃ   )r4   s    r$   Úget_input_embeddingszTvpModel.get_input_embeddings  s   € Ø�‰×.Ñ.Ð.r#   c                 ó&   — || j                   _        y r  rŒ  )r4   rÙ   s     r$   Úset_input_embeddingszTvpModel.set_input_embeddings  s   € Ø*/ˆ�‰Õ'r#   c                 ó˜   — |j                  «       D ]7  \  }}| j                  j                  |   j                  j	                  |«       Œ9 y)z”Prunes heads of the model.
        heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base class PreTrainedModel
        N)Úitemsr„  r*  r  rî   )r4   Úheads_to_pruner*  rë   s       r$   Ú_prune_headszTvpModel._prune_heads"  sE   € ð +×0Ñ0Ó2ò 	C‰LˆE�5Ø�L‰L×Ñ˜uÑ%×/Ñ/×;Ñ;¸EÕBñ	Cr#   ©Úoutput_typerU  rÉ   ru   rü   rý   rô   r-  r.  r    c	                 ó"  — |�|n| j                   j                  }| j                  | j                  ||¬«      «      }| j	                  |¬«      }	| j                  ||¬«      }
|�¹|j                  |
j                  dd «      }t        j                  |j                  d   d«      j                  |j                  |j                  ¬«      }t        j                  |||gd	¬
«      }| j                  ||j                  «       «      j                  |j                  «      }| j                   j#                  |	j                  d   d	d	«      }t        j                  ||	|
gd¬
«      }| j%                  ||| j'                  || j                   j(                  «      |||¬«      }|r|j*                  n|d   }| j-                  |«      }| j/                  |«      }| j/                  |«      }|s
||f|dd z   S t1        |||j2                  |j4                  ¬«      S )a(  
        Returns:

        Examples:
        ```python
        >>> import torch
        >>> from transformers import AutoConfig, AutoTokenizer, TvpModel

        >>> model = TvpModel.from_pretrained("Jiqing/tiny-random-tvp")

        >>> tokenizer = AutoTokenizer.from_pretrained("Jiqing/tiny-random-tvp")

        >>> pixel_values = torch.rand(1, 1, 3, 448, 448)
        >>> text_inputs = tokenizer("This is an example input", return_tensors="pt")
        >>> output = model(text_inputs.input_ids, pixel_values, text_inputs.attention_mask)
        ```N)rx  )rÉ   r²   rk   r   r�  )r¤   r£   rZ   rå   r   )rü   rý   rô   r-  r.  )r0  Úpooler_outputr   r   )r[   r.  r‚  r‰  r¸   rƒ  Únew_onesro   r   rç   rh  r¤   r£   rz  Úget_extended_attention_maskrÆ   r†  rÈ   r„  Úget_head_maskr)  r0  r…  r“   r   r   r   )r4   rÉ   ru   rü   rý   rô   r-  r.  r    Útext_embedding_outputÚvisual_embedding_outputÚvisual_attention_maskÚpt_maskr†  Úembedding_outputÚencoder_outputsr0  rA  s                     r$   rT   zTvpModel.forward)  s   € ð: &1Ð%<‘kÀ$Ç+Á+×BYÑBYˆà×(Ñ(Ø× Ñ  ÐH`Ð Óaó
ˆð !%§¡¸) Ó DÐà"&×"8Ñ"8ØÐ3Kð #9ó #
Ðð Ð%à$2×$;Ñ$;Ð<S×<YÑ<YÐZ\Ð[\Ð<]Ó$^Ð!Ü—j‘j ×!5Ñ!5°aÑ!8¸"Ó=×@Ñ@Ø%×,Ñ,°N×4HÑ4Hð Aó ˆGô #ŸY™Y¨°ÐAVÐ'WÐ]_Ô`ˆNð "×=Ñ=¸nÈiÏnÉnÓN^Ó_×bÑbÐcl×csÑcsÓtˆNØ×&Ñ&×-Ñ-Ð.C×.IÑ.IÈ!Ñ.LÈbÐRTÓUˆä Ÿ9™9 kÐ3HÐJaÐ%bÐhiÔjÐàŸ,™,ØØ)Ø×(Ñ(¨°D·K±K×4QÑ4QÓRØ/Ø!5Ø#ð 'ó 
ˆñ BM˜O×=Ò=ÐRaÐbcÑRdÐØŸ™Ð$5Ó6ˆØ ŸL™LÐ):Ó;ÐØŸ™ ]Ó3ˆÙØ% }Ð5¸ÈÈÐ8KÑKÐKÜ)Ø/Ø'Ø)×7Ñ7Ø&×1Ñ1ô	
ð 	
r#   )NNNNNNNF)r   r   r   r-   r�  r�  r“  r
   ÚTVP_INPUTS_DOCSTRINGr   r   r   r   r   Ú
LongTensorr    r¼   rT   rU   rV   s   @r$   r  r    sï   ø„ ô
ò /ò0òCñ +Ð+?Ó@ÙÐ+EÐT]Ô^ð 15Ø48Ø59Ø15Ø,0Ø/3Ø&*Ø).ñH
à˜E×,Ñ,Ñ-ðH
ð ˜u×0Ñ0Ñ1ðH
ð ! ×!1Ñ!1Ñ2ð	H
ð
 ˜E×-Ñ-Ñ.ðH
ð $ D™>ðH
ð ' t™nðH
ð ˜d‘^ðH
ð #'òH
ó _ó AôH
r#   r  c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚTvpVideoGroundingHeadc                 ó:  •— t         ‰| �  «        t        j                  |j                  |j                  dz  «      | _        t        j                  |j                  dz  d«      | _        t        j                  «       | _        t        j                  «       | _
        y )Nrk   )r,   r-   r   rÖ   r]   Úlayer_0Úlayer_1ÚReLUÚactivation_0ÚSigmoidÚactivation_1r”   s     €r$   r-   zTvpVideoGroundingHead.__init__w  sj   ø€ Ü‰ÑÔÜ—y‘y ×!3Ñ!3°V×5GÑ5GÈ!Ñ5KÓLˆŒÜ—y‘y ×!3Ñ!3°aÑ!7¸Ó;ˆŒÜŸG™G›IˆÔÜŸJ™J›LˆÕr#   c                 ó†   — | j                  | j                  |«      «      }| j                  | j                  |«      «      }|S r  )r©  r¦  r«  r§  )r4   r—  r   s      r$   rT   zTvpVideoGroundingHead.forward~  s9   € Ø×"Ñ" 4§<¡<°Ó#>Ó?ˆØ×"Ñ" 4§<¡<°Ó#7Ó8ˆØˆr#   r€   rV   s   @r$   r¤  r¤  v  s   ø„ ô)ör#   r¤  zb
    Tvp Model with a video grounding head on top computing IoU, distance, and duration loss.
    c                   ó"  ‡ — e Zd Zˆ fd„Z ee«       eee¬«      	 	 	 	 	 	 	 	 	 dde	e
j                     de	e
j                     de	e
j                     dee
j                     de	e
j                     de	e   d	e	e   d
e	e   defd„«       «       Zˆ xZS )ÚTvpForVideoGroundingc                 ó’   •— t         ‰| �  |«       || _        t        |«      | _        t        |«      | _        | j                  «        y r  )r,   r-   r[   r  rD  r¤  Úvideo_grounding_headrŠ  r”   s     €r$   r-   zTvpForVideoGrounding.__init__‹  s:   ø€ Ü‰Ñ˜Ô ØˆŒÜ˜fÓ%ˆŒ
Ü$9¸&Ó$AˆÔ!à�‰Õr#   r”  rÉ   ru   rü   rQ   rý   rô   r-  r.  r    c
           
      óì  — |�|n| j                   j                  }| j                  ||||||||	¬«      }
|
d   }| j                  |«      }d}|�pt	        g d¢«      }|j                  | j                  «        |||«      }|d   | j                   j                  |d   z  z   | j                   j                  |d   z  z   }|s|f|
dd z   }
|�|f|
z   }
|
S t        |||
j                  |
j                  ¬	«      S )
aø  
        labels (`torch.FloatTensor` of shape `(batch_size, 3)`, *optional*):
            The labels contains duration, start time, and end time of the video corresponding to the text.
        Returns:

        Examples:
        ```python
        >>> import torch
        >>> from transformers import AutoConfig, AutoTokenizer, TvpForVideoGrounding

        >>> model = TvpForVideoGrounding.from_pretrained("Jiqing/tiny-random-tvp")

        >>> tokenizer = AutoTokenizer.from_pretrained("Jiqing/tiny-random-tvp")

        >>> pixel_values = torch.rand(1, 1, 3, 448, 448)
        >>> text_inputs = tokenizer("This is an example input", return_tensors="pt")
        >>> output = model(text_inputs.input_ids, pixel_values, text_inputs.attention_mask)
        ```N)rý   rô   r-  r.  r    r   r(   r)   r*   r+   rk   )r   r   r   r   )r[   r.  rD  r°  r&   rh  r¤   Údistance_loss_weightÚduration_loss_weightr   r   r   )r4   rÉ   ru   rü   rQ   rý   rô   r-  r.  r    r  r—  r   r   Ú	criterionÚ	loss_dicts                   r$   rT   zTvpForVideoGrounding.forward“  s-  € ð@ &1Ð%<‘kÀ$Ç+Á+×BYÑBYˆØ—*‘*ØØØØØ/Ø!5Ø#Ø%=ð ó 	
ˆð   ™
ˆØ×*Ñ*¨=Ó9ˆàˆØÐÜÒ ?Ó@ˆIØ�L‰L˜Ÿ™Ô%Ù! &¨&Ó1ˆIà˜%Ñ Ø—+‘+×2Ñ2°Y¸zÑ5JÑJñKà—+‘+×2Ñ2°Y¸zÑ5JÑJñKð ñ
 Ø�i '¨!¨" +Ñ-ˆGØÐØ˜' GÑ+�ØˆNä&ØØØ!×/Ñ/Ø×)Ñ)ô	
ð 	
r#   )	NNNNNNNNF)r   r   r   r-   r
   r¡  r   r   r   r   r   r¢  r    r   rº   r¼   rT   rU   rV   s   @r$   r®  r®  „  sö   ø„ ôñ +Ð+?Ó@ÙÐ+BÐQZÔ[ð 15Ø48Ø59Ø&*Ø15Ø,0Ø/3Ø&*Ø).ñA
à˜E×,Ñ,Ñ-ðA
ð ˜u×0Ñ0Ñ1ðA
ð ! ×!1Ñ!1Ñ2ð	A
ð
 �e—l‘lÑ#ðA
ð ˜E×-Ñ-Ñ.ðA
ð $ D™>ðA
ð ' t™nðA
ð ˜d‘^ðA
ð #'òA
ó \ó AôA
r#   r®  )r  rC  r®  )7r   rø   Údataclassesr   Útypingr   r   r   Útorch.utils.checkpointr   Úactivationsr   Ú
file_utilsr	   r
   r   Úmodeling_outputsr   r   r   Úmodeling_utilsr   Úpytorch_utilsr   Úutilsr   Úutils.backbone_utilsr   Úconfiguration_tvpr   Ú
get_loggerr   Úloggerr   ÚModuler&   rX   r‚   r¾   rÐ   r
  r  r  r%  r;  rC  ÚTVP_START_DOCSTRINGr¡  rY  rm  rˆ  r  r¤  r®  Ú__all__r"   r#   r$   ú<module>rÆ     sØ  ðñ ã Ý !ß "ã Û Ý å !ß pÑ pß XÑ XÝ -Ý /Ý Ý 1Ý (ð 
ˆ×	Ñ	˜HÓ	%€ð ô?˜kó ?ó ð?ô.Mˆb�i‰iô Mô`%�R—Y‘Yô %ôPn˜bŸi™iô nôb!˜RŸY™Yô !ôH_�2—9‘9ô _ôF�b—i‘iô ô�R—Y‘Yô ô�R—Y‘Yô ô8;
�—‘ô ;
ô~�—	‘	ô ô2˜ô 2ð8	Ð ð#Ð ôL"˜bŸi™iô "ôJW˜"Ÿ)™)ô Wðv ,Ø#ñ Ð ñ ØsØóôh
Ð!ó h
ó	ðh
ôV˜BŸI™Iô ñ ðð ó	ôL
Ð-ó L
óðL
ò^ E�r#   