Ë
    S^(håe ã                   óŒ  — d Z ddlZddlmZ ddlmZ ddlmZmZm	Z	m
Z
 ddlZddlZddlmZ ddlmZ dd	lmZmZ dd
lmZmZmZmZmZ ddlmZmZ ddlmZmZ ddlm Z m!Z!m"Z"m#Z#m$Z$ ddl%m&Z&m'Z'm(Z(  e"jR                  e*«      Z+dZ,dZ-dZ.dZ/dZ0e G d„ de«      «       Z1e G d„ de«      «       Z2 G d„ dejf                  «      Z4 G d„ dejf                  «      Z5 G d„ dejf                  «      Z6 G d„ d ejf                  «      Z7 G d!„ d"ejf                  «      Z8 G d#„ d$ejf                  «      Z9 G d%„ d&ejf                  «      Z: G d'„ d(ejf                  «      Z; G d)„ d*ejf                  «      Z< G d+„ d,ejf                  «      Z=d-e=iZ> G d.„ d/ejf                  «      Z? G d0„ d1ejf                  «      Z@ G d2„ d3ejf                  «      ZA G d4„ d5ejf                  «      ZB G d6„ d7ejf                  «      ZCdTd8„ZD G d9„ d:e«      ZE G d;„ d<eE«      ZF G d=„ d>eE«      ZG e d?e/«       G d@„ dAeE«      «       ZH G dB„ dCejf                  «      ZI G dD„ dEejf                  «      ZJ G dF„ dGejf                  «      ZK e dHe/«       G dI„ dJeE«      «       ZL e dKe/«       G dL„ dMeE«      «       ZM G dN„ dOejf                  «      ZN e dPe/«       G dQ„ dReE«      «       ZOg dS¢ZPy)UzPyTorch BridgeTower Modelé    N)ÚOrderedDict)Ú	dataclass)ÚListÚOptionalÚTupleÚUnion)Únn)ÚCrossEntropyLossé   )ÚACT2FNÚQuickGELUActivation)Ú)BaseModelOutputWithPastAndCrossAttentionsÚ,BaseModelOutputWithPoolingAndCrossAttentionsÚMaskedLMOutputÚModelOutputÚSequenceClassifierOutput)ÚPreTrainedModelÚapply_chunking_to_forward)Ú find_pruneable_heads_and_indicesÚprune_linear_layer)Úadd_start_docstringsÚ%add_start_docstrings_to_model_forwardÚloggingÚreplace_return_docstringsÚ	torch_inté   )ÚBridgeTowerConfigÚBridgeTowerTextConfigÚBridgeTowerVisionConfigr   zBridgeTower/bridgetower-baseÚRobertaTokenizeraO  
    This model is a PyTorch `torch.nn.Module <https://pytorch.org/docs/stable/nn.html#torch.nn.Module>`_ subclass. Use
    it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and
    behavior.

    Parameters:
        config ([`BridgeTowerConfig`]): Model configuration class with all the parameters of the model.
            Initializing with a config file does not load the weights associated with the model, only the
            configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
aÃ  
    Args:
        input_ids (`torch.LongTensor` of shape `({0})`):
            Indices of input sequence tokens in the vocabulary. Indices can be obtained using [`AutoTokenizer`]. See
            [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input
            IDs?](../glossary#input-ids)

        attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*):
            Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:
            - 1 for tokens that are **not masked**,
            - 0 for tokens that are **masked**.
            [What are attention masks?](../glossary#attention-mask)

        token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*):
            Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
            1]`:
            - 0 corresponds to a *sentence A* token,
            - 1 corresponds to a *sentence B* token.
            [What are token type IDs?](../glossary#token-type-ids)

        pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`):
            Pixel values. Pixel values can be obtained using [`BridgeTowerImageProcessor`]. See
            [`BridgeTowerImageProcessor.__call__`] for details.

        pixel_mask (`torch.LongTensor` of shape `(batch_size, height, width)`, *optional*):
            Mask to avoid performing attention on padding pixel values. Mask values selected in `[0, 1]`:

            - 1 for pixels that are real (i.e. **not masked**),
            - 0 for pixels that are padding (i.e. **masked**).
            `What are attention masks? <../glossary.html#attention-mask>`__

        head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*):
            Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`:
            - 1 indicates the head is **not masked**,
            - 0 indicates the head is **masked**.

        inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*):
            Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
            is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
            model's internal embedding lookup matrix.

        image_embeds (`torch.FloatTensor` of shape `(batch_size, num_patches, hidden_size)`, *optional*):
            Optionally, instead of passing `pixel_values`, you can choose to directly pass an embedded representation.
            This is useful if you want more control over how to convert `pixel_values` into patch embeddings.

        image_token_type_idx (`int`, *optional*):
            - The token type ids for images.

        output_attentions (`bool`, *optional*):
            Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
            tensors for more detail.

        output_hidden_states (`bool`, *optional*):
            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
            more detail.
        interpolate_pos_encoding (`bool`, defaults to `False`):
            Whether to interpolate the pre-trained position encodings.
        return_dict (`bool`, *optional*):
            Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
c                   óæ   — e Zd ZU dZdZeej                     ed<   dZ	eej                     ed<   dZ
eej                     ed<   dZeeej                        ed<   dZeeej                        ed<   y)ÚBridgeTowerModelOutputa™  
    Output type of [`BridgeTowerModel`].

    Args:
        text_features (`torch.FloatTensor` of shape `(batch_size, text_sequence_length, hidden_size)`):
            Sequence of hidden-states at the text output of the last layer of the model.
        image_features (`torch.FloatTensor` of shape `(batch_size, image_sequence_length, hidden_size)`):
            Sequence of hidden-states at the image output of the last layer of the model.
        pooler_output (`torch.FloatTensor` of shape `(batch_size, hidden_size x 2)`):
            Concatenation of last layer hidden-state of the first token of the text and image sequence (classification
            token), respectively, after further processing through layers used for auxiliary pretraining tasks.
        hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
            Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, +
            one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`. Hidden-states of
            the model at the output of each layer plus the optional initial embedding outputs.
        attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
            Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
            sequence_length)`.

            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
            heads.
    NÚtext_featuresÚimage_featuresÚpooler_outputÚhidden_statesÚ
attentions)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r#   r   ÚtorchÚFloatTensorÚ__annotations__r$   r%   r&   r   r'   © ó    úr/var/www/skyplay_api_hub/venv/lib/python3.12/site-packages/transformers/models/bridgetower/modeling_bridgetower.pyr"   r"      s|   … ñð. 26€M�8˜E×-Ñ-Ñ.Ó5Ø26€N�H˜U×.Ñ.Ñ/Ó6Ø15€M�8˜E×-Ñ-Ñ.Ó5Ø8<€M�8˜E %×"3Ñ"3Ñ4Ñ5Ó<Ø59€J�˜˜u×0Ñ0Ñ1Ñ2Ô9r0   r"   c                   óH  — e Zd ZU dZdZeej                     ed<   dZ	eej                     ed<   dZ
eeej                        ed<   dZeeej                        ed<   dZeeej                        ed<   dZeeej                        ed<   dZeeej                        ed	<   y)
ÚBridgeTowerContrastiveOutputaZ  
    Output type of ['BridgeTowerForContrastiveLearning']

    Args:
        loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `return_loss` is `True`:
            Image-text contrastive loss.
        logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`):
            Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
        text_embeds (`torch.FloatTensor)`, *optional*, returned when model is initialized with `with_projection=True`):
            The text embeddings obtained by applying the projection layer to the pooler_output.
        image_embeds (`torch.FloatTensor)`, *optional*, returned when model is initialized with `with_projection=True`):
            The image embeddings obtained by applying the projection layer to the pooler_output.
        cross_embeds  (`torch.FloatTensor)`, *optional*, returned when model is initialized with `with_projection=True`):
            The text-image cross-modal embeddings obtained by applying the projection layer to the pooler_output.
        hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
            Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, +
            one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`. Hidden-states of
            the model at the output of each layer plus the optional initial embedding outputs.
        attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
            Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
            sequence_length)`.
    NÚlossÚlogitsÚtext_embedsÚimage_embedsÚcross_embedsr&   r'   )r(   r)   r*   r+   r4   r   r,   r-   r.   r5   r6   r   r7   r8   r&   r'   r/   r0   r1   r3   r3   Ÿ   sµ   … ñð. )-€Dˆ(�5×$Ñ$Ñ
%Ó,Ø*.€FˆH�U×&Ñ&Ñ'Ó.Ø6:€K�˜% × 1Ñ 1Ñ2Ñ3Ó:Ø7;€L�(˜5 ×!2Ñ!2Ñ3Ñ4Ó;Ø7;€L�(˜5 ×!2Ñ!2Ñ3Ñ4Ó;Ø8<€M�8˜E %×"3Ñ"3Ñ4Ñ5Ó<Ø59€J�˜˜u×0Ñ0Ñ1Ñ2Ô9r0   r3   c                   ó–   ‡ — e Zd Zˆ fd„Zdej
                  dej
                  fd„Zddej
                  deej
                     fd„Zˆ xZ	S )ÚBridgeTowerResidualAttentionc                 óh  •— t         ‰| �  «        t        j                  |j                  |j                  dz  «      | _        t        j                  |j                  |j                  ¬«      | _        t        j                  t        dt        j                  |j                  |j                  dz  «      fdt        «       fdt        j                  |j                  dz  |j                  «      fg«      «      | _        t        j                  |j                  |j                  ¬«      | _        d | _        y )Né@   ©ÚepsÚc_fcé   ÚgeluÚc_proj)ÚsuperÚ__init__r	   ÚMultiheadAttentionÚhidden_sizeÚattnÚ	LayerNormÚlayer_norm_epsÚln_1Ú
ModuleDictr   ÚLinearr   ÚmlpÚln_2Ú	attn_mask©ÚselfÚconfigÚ	__class__s     €r1   rD   z%BridgeTowerResidualAttention.__init__Â   së   ø€ Ü‰ÑÔä×)Ñ)¨&×*<Ñ*<¸f×>PÑ>PÐTVÑ>VÓWˆŒ	Ü—L‘L ×!3Ñ!3¸×9NÑ9NÔOˆŒ	Ü—=‘=ÜàœRŸY™Y v×'9Ñ'9¸6×;MÑ;MÐPQÑ;QÓRÐSØÔ0Ó2Ð3ØœrŸy™y¨×);Ñ);¸aÑ)?À×ASÑASÓTÐUðóó
ˆŒô —L‘L ×!3Ñ!3¸×9NÑ9NÔOˆŒ	Øˆ�r0   Úhidden_stateÚattention_maskc                 ó,  — |�+|j                  t        j                  |j                  ¬«      }| j                  �1| j                  j                  |j
                  |j                  ¬«      nd | _        | j                  |||d| j                  |¬«      d   S )N©ÚdtypeÚdeviceF)Úneed_weightsrO   Úkey_padding_maskr   )Útor,   ÚboolrY   rO   rX   rG   )rQ   rT   rU   s      r1   Ú	attentionz&BridgeTowerResidualAttention.attentionÓ   sš   € ØÐ%Ø+×.Ñ.´U·Z±ZÈ×H[ÑH[Ð.Ó\ˆNð �~‰~Ð)ð �N‰N×Ñ L×$6Ñ$6¸|×?RÑ?RÐÔSàð 	Œð
 �y‰yØØØØØ—n‘nØ+ð ó 
ð ñð 	r0   c                 óÎ   — || j                  | j                  |«      |«      z   }| j                  |«      }| j                  j	                  «       D ]  \  }} ||«      }Œ ||z   }|S ©N)r^   rJ   rN   rM   Úitems)rQ   rT   rU   Úresidual_stateÚ_Úlayers         r1   Úforwardz$BridgeTowerResidualAttention.forwardä   sg   € Ø%¨¯©°t·y±yÀÓ7NÐP^Ó(_Ñ_ˆØ—y‘y Ó0ˆØŸ™Ÿ™Ó(ò 	/‰HˆAˆuÙ  Ó.‰Lð	/à%¨Ñ4ˆØÐr0   r`   )
r(   r)   r*   rD   r,   ÚTensorr^   r   re   Ú__classcell__©rS   s   @r1   r:   r:   Á   sC   ø„ ôð" e§l¡lð ÀEÇLÁLó ñ" E§L¡Lð À(È5Ï<É<ÑBX÷ r0   r:   c                   ó^   ‡ — e Zd Zˆ fd„Zddej
                  deej
                     fd„Zˆ xZS )ÚBridgeTowerTransformerc                 óÌ  •— t         ‰| �  «        |j                  | _        |j                  | _        |j                  rHt        j                  t        | j                  dz
  «      D �cg c]  }t        |«      ‘Œ c}«      | _	        nDt        j                  t        | j                  «      D �cg c]  }t        |«      ‘Œ c}«      | _	        |j                  | _
        y c c}w c c}w ©Nr   )rC   rD   rF   Únum_hidden_layersÚremove_last_layerr	   Ú
ModuleListÚranger:   Ú	resblocksÚstop_gradient©rQ   rR   rc   rS   s      €r1   rD   zBridgeTowerTransformer.__init__î   sµ   ø€ Ü‰ÑÔØ!×-Ñ-ˆÔØ!'×!9Ñ!9ˆÔØ×#Ò#ÜŸ]™]Ü?DÀT×E[ÑE[Ð^_ÑE_Ó?`Öa¸!Ô-¨fÕ5ÒaóˆD�Nô  Ÿ]™]Ü?DÀT×E[ÑE[Ó?\Ö]¸!Ô-¨fÕ5Ò]óˆDŒNð $×1Ñ1ˆÕùò bùò ^s   Á'CÂ,C!rT   rU   c                 ó¸   — g }| j                   D ]H  } |||«      }| j                  r |j                  |j                  «       «       Œ8|j                  |«       ŒJ |S r`   )rq   rr   ÚappendÚdetach)rQ   rT   rU   r&   Úblocks        r1   re   zBridgeTowerTransformer.forwardü   s\   € ØˆØ—^‘^ò 	3ˆEÙ  ¨~Ó>ˆLØ×!Ò!Ø×$Ñ$ \×%8Ñ%8Ó%:Õ;à×$Ñ$ \Õ2ð	3ð Ðr0   r`   )	r(   r)   r*   rD   r,   rf   r   re   rg   rh   s   @r1   rj   rj   í   s(   ø„ ô2ñ E§L¡Lð À(È5Ï<É<ÑBX÷ r0   rj   c                   óž   ‡ — e Zd Zdefˆ fd„Zdej                  dededej                  fd„Zd
dej                  dej                  fd	„Z
ˆ xZS )ÚBridgeTowerVisionEmbeddingsrR   c                 óÚ  •— t         ‰| �  «        || _        |j                  | _        |j
                  | _        |j                  | _        t        j                  t        j                  | j                  «      «      | _        t        j                  |j                  | j                  | j                  | j                  d¬«      | _        | j
                  | j                  z  dz  | _        | j                  dz   | _        t        j"                  | j                   | j                  «      | _        | j'                  dt        j(                  | j                   «      j+                  d«      d¬«       y )NF)Úin_channelsÚout_channelsÚkernel_sizeÚstrideÚbiasé   r   Úposition_ids©r   éÿÿÿÿ©Ú
persistent)rC   rD   rR   rF   Ú	embed_dimÚ
image_sizeÚ
patch_sizer	   Ú	Parameterr,   ÚrandnÚclass_embeddingÚConv2dÚnum_channelsÚpatch_embeddingÚnum_patchesÚnum_positionsÚ	EmbeddingÚposition_embeddingÚregister_bufferÚarangeÚexpandrP   s     €r1   rD   z$BridgeTowerVisionEmbeddings.__init__	  s	  ø€ Ü‰ÑÔØˆŒØ×+Ñ+ˆŒØ ×+Ñ+ˆŒØ ×+Ñ+ˆŒä!Ÿ|™|¬E¯K©K¸¿¹Ó,GÓHˆÔä!Ÿy™yØ×+Ñ+ØŸ™ØŸ™Ø—?‘?Øô 
ˆÔð !ŸO™O¨t¯©Ñ>À1ÑDˆÔØ!×-Ñ-°Ñ1ˆÔÜ"$§,¡,¨t×/AÑ/AÀ4Ç>Á>Ó"RˆÔØ×Ñ˜^¬U¯\©\¸$×:LÑ:LÓ-M×-TÑ-TÐU\Ó-]ÐjoÐÕpr0   Ú
embeddingsÚheightÚwidthÚreturnc                 óÒ  — |j                   d   dz
  }| j                  j                  j                  d«      }|j                   d   dz
  }t        j
                  j                  «       s%||k(  r ||k(  r| j                  | j                  «      S |dd…dd…f   }|dd…dd…f   }|j                   d   }	|| j                  z  }
|| j                  z  }t        |dz  «      }|j                  d|||	«      }|j                  dddd«      }t        j                  j                  ||
|fdd	¬
«      }|j                  dddd«      j                  dd|	«      }t	        j                   ||fd¬«      S )a   
        This method allows to interpolate the pre-trained position encodings, to be able to use the model on higher resolution
        images. This method is also adapted to support torch.jit tracing.

        Adapted from:
        - https://github.com/facebookresearch/dino/blob/de9ee3df6cf39fac952ab558447af1fa1365362a/vision_transformer.py#L174-L194, and
        - https://github.com/facebookresearch/dinov2/blob/e1277af2ba9496fbadf7aec6eba56e8d882d1e35/dinov2/models/vision_transformer.py#L179-L211
        r   r   Nrƒ   ç      à?r   r€   ÚbicubicF)ÚsizeÚmodeÚalign_corners©Údim)Úshaper’   ÚweightÚ	unsqueezer,   ÚjitÚ
is_tracingr�   rˆ   r   ÚreshapeÚpermuter	   Ú
functionalÚinterpolateÚviewÚcat)rQ   r–   r—   r˜   r�   r’   r�   Úclass_pos_embedÚpatch_pos_embedr¡   Ú
new_heightÚ	new_widthÚsqrt_num_positionss                r1   Úinterpolate_pos_encodingz4BridgeTowerVisionEmbeddings.interpolate_pos_encoding  sv  € ð !×&Ñ& qÑ)¨AÑ-ˆØ!×4Ñ4×;Ñ;×EÑEÀaÓHÐØ*×0Ñ0°Ñ3°aÑ7ˆô �y‰y×#Ñ#Ô%¨+¸Ò*FÈ6ÐUZÊ?Ø×*Ñ*¨4×+<Ñ+<Ó=Ð=à,ªQ°°°¨UÑ3ˆØ,ªQ°±¨UÑ3ˆà×Ñ˜rÑ"ˆà˜tŸ™Ñ.ˆ
Ø˜TŸ_™_Ñ,ˆ	ä& }°cÑ'9Ó:ÐØ)×1Ñ1°!Ð5GÐI[Ð]`ÓaˆØ)×1Ñ1°!°Q¸¸1Ó=ˆäŸ-™-×3Ñ3ØØ˜iÐ(ØØð	 4ó 
ˆð *×1Ñ1°!°Q¸¸1Ó=×BÑBÀ1ÀbÈ#ÓNˆä�y‰y˜/¨?Ð;ÀÔCÐCr0   Úpixel_valuesc                 ó`  — |j                   \  }}}}|sJ|| j                  k7  s|| j                  k7  r,t        d|› d|› d| j                  › d| j                  › d�	«      ‚| j                  j                  j
                  }| j                  |j                  |¬«      «      }|j                  d«      j                  dd«      }| j                  j                  |dd«      }	t        j                  |	|gd¬	«      }
|r|
| j                  |
||«      z   }
|
S |
| j                  | j                  «      z   }
|
S )
NzInput image size (Ú*z) doesn't match model (z).©rX   r€   r   rƒ   r    )r¢   r‡   Ú
ValueErrorrŽ   r£   rX   r\   ÚflattenÚ	transposer‹   r•   r,   r¬   r²   r’   r�   )rQ   r³   r²   Ú
batch_sizerc   r—   r˜   Útarget_dtypeÚpatch_embedsÚclass_embedsr–   s              r1   re   z#BridgeTowerVisionEmbeddings.forwardH  s6  € Ø'3×'9Ñ'9Ñ$ˆ
�A�v˜uÙ'¨V°t·±Ò-FÈ%ÐSW×SbÑSbÒJbÜØ$ V H¨A¨e¨WÐ4KÈDÏOÉOÐK\Ð\]Ð^b×^mÑ^mÐ]nÐnpÐqóð ð ×+Ñ+×2Ñ2×8Ñ8ˆØ×+Ñ+¨L¯O©OÀ,¨OÓ,OÓPˆØ#×+Ñ+¨AÓ.×8Ñ8¸¸AÓ>ˆà×+Ñ+×2Ñ2°:¸qÀ"ÓEˆÜ—Y‘Y ¨lÐ;ÀÔCˆ
Ù#Ø# d×&CÑ&CÀJÐPVÐX]Ó&^Ñ^ˆJð Ðð $ d×&=Ñ&=¸d×>OÑ>OÓ&PÑPˆJØÐr0   ©F)r(   r)   r*   r   rD   r,   rf   Úintr²   r-   re   rg   rh   s   @r1   ry   ry     se   ø„ ðqÐ6õ qð,'D°5·<±<ð 'DÈð 'DÐUXð 'DÐ]b×]iÑ]ió 'DñR E×$5Ñ$5ð ÐZ_×ZfÑZf÷ r0   ry   c                   óŽ   ‡ — e Zd Zˆ fd„Z	 ddej
                  defd„Z	 ddej
                  defd„Zdej
                  fd„Z	ˆ xZ
S )	ÚBridgeTowerVisionTransformerc           	      ó0  •— t         ‰| �  «        t        |«      | _        t	        j
                  |j                  |j                  ¬«      | _        t        |«      | _
        t	        j
                  |j                  |j                  ¬«      | _        |j                  | _        |j                  set	        j                  t        |j                  «      D �cg c]-  }t	        j
                  |j                  |j                  ¬«      ‘Œ/ c}«      | _        y y c c}w ©Nr=   )rC   rD   ry   r–   r	   rH   rF   rI   Úln_prerj   ÚtransformerÚln_postÚshare_layernormro   rp   rm   Úln_separaters   s      €r1   rD   z%BridgeTowerVisionTransformer.__init__\  sÅ   ø€ Ü‰ÑÔä5°fÓ=ˆŒÜ—l‘l 6×#5Ñ#5¸6×;PÑ;PÔQˆŒÜ1°&Ó9ˆÔÜ—|‘| F×$6Ñ$6¸F×<QÑ<QÔRˆŒØ%×5Ñ5ˆÔØ×%Ò%Ü!Ÿ}™}ÜV[Ð\b×\tÑ\tÓVuÖvÐQR”—‘˜f×0Ñ0°f×6KÑ6KÖLÒvó ˆDÕð &ùâvs   Ã2Dr³   r²   c                 óÊ  — | j                  ||«      }| j                  |«      }|j                  ddd«      }| j                  ||«      }t	        j
                  |d¬«      }|j                  dddd«      }| j                  r| j                  |«      }|S g }t        || j                  «      D ]  \  }} ||«      }|j                  |«       Œ  t	        j
                  |d¬«      }|S )Nr   r   r€   r    r   )r–   rÄ   r¨   rÅ   r,   ÚstackrÇ   rÆ   ÚziprÈ   ru   )rQ   r³   rU   r²   r&   Úhidden_states_stackÚlns          r1   re   z$BridgeTowerVisionTransformer.forwardi  sè   € ð Ÿ™¨Ð6NÓOˆØŸ™ MÓ2ˆà%×-Ñ-¨a°°AÓ6ˆà×(Ñ(¨¸ÓGˆäŸ™ M°qÔ9ˆà%×-Ñ-¨a°°A°qÓ9ˆØ×ÒØ ŸL™L¨Ó7ˆMð Ðð #%ÐÜ%(¨¸×8HÑ8HÓ%Iò :Ñ!�˜rÙ " =Ó 1�Ø#×*Ñ*¨=Õ9ð:ô "ŸK™KÐ(;ÀÔCˆMØÐr0   c                 ót   — | j                  ||¬«      }| j                  |«      }|j                  ddd«      }|S )N©r²   r   r   r€   )r–   rÄ   r¨   )rQ   r³   r²   r&   s       r1   Úforward_prez(BridgeTowerVisionTransformer.forward_pre„  s?   € ð
 Ÿ™¨ÐOg˜ÓhˆØŸ™ MÓ2ˆà%×-Ñ-¨a°°AÓ6ˆØÐr0   rT   c                 óN   — |j                  ddd«      }| j                  |«      }|S )Nr   r   r€   )r¨   rÆ   )rQ   rT   Úvisual_output_posts      r1   Úforward_postz)BridgeTowerVisionTransformer.forward_post�  s-   € Ø)×1Ñ1°!°Q¸Ó:ÐØ!Ÿ\™\Ð*<Ó=ÐØ!Ð!r0   r¾   )r(   r)   r*   rD   r,   rf   r]   re   rÐ   rÓ   rg   rh   s   @r1   rÁ   rÁ   [  sX   ø„ ôð" */ñ	à—l‘lðð #'ó	ð< */ñ	à—l‘lð	ð #'ó	ð"¨¯©÷ "r0   rÁ   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚBridgeTowerLinkTowerc                 ó  •— t         ‰| �  «        |j                  | _        |j                  | _        |j                  dv rª|j                  dk(  r.t	        j
                  t        j                  d«      «      | _        n<|j                  dk(  r-t	        j
                  t        j                  d«      «      | _	        t	        j                  | j                  |j                  ¬«      | _
        y t        d|j                  › d�«      ‚)	N)ÚaddÚ
scaled_addrª   rØ   ç      ð?rª   r›   r=   úlink_tower_type ú is not implemented)rC   rD   Úlink_tower_typerF   r	   r‰   r,   ÚtensorÚscaled_factorÚbetarH   rI   ÚNotImplementedErrorrP   s     €r1   rD   zBridgeTowerLinkTower.__init__–  sÃ   ø€ Ü‰ÑÔØ%×5Ñ5ˆÔØ!×-Ñ-ˆÔØ×!Ñ!Ð%IÑIØ×%Ñ%¨Ò5Ü%'§\¡\´%·,±,¸sÓ2CÓ%D�Õ"Ø×'Ñ'¨=Ò8ÜŸL™L¬¯©°cÓ):Ó;�”	ÜŸ\™\¨$×*:Ñ*:À×@UÑ@UÔVˆD�Nä%Ð(8¸×9OÑ9OÐ8PÐPcÐ&dÓeÐer0   c                 óZ  — | j                   dk(  r| j                  ||z   «      S | j                   dk(  r!| j                  || j                  z  |z   «      S | j                   dk(  r1| j                  |d| j                  z
  z  || j                  z  z   «      S t	        d| j                   › d�«      ‚)Nr×   rØ   rª   r   rÚ   rÛ   )rÜ   rH   rÞ   rß   rà   )rQ   r&   Úcross_modal_hidden_statesrU   s       r1   re   zBridgeTowerLinkTower.forward£  s¬   € Ø×Ñ 5Ò(Ø—>‘> -Ð2KÑ"KÓLÐLØ×!Ñ! \Ò1Ø—>‘> -°$×2DÑ2DÑ"DÐG`Ñ"`ÓaÐaØ×!Ñ! ]Ò2Ø—>‘> -°1°t·y±y±=Ñ"AÐD]Ð`d×`iÑ`iÑDiÑ"iÓjÐjä%Ð(8¸×9MÑ9MÐ8NÐNaÐ&bÓcÐcr0   ©r(   r)   r*   rD   re   rg   rh   s   @r1   rÕ   rÕ   •  s   ø„ ôfödr0   rÕ   c                   ón   ‡ — e Zd Zˆ fd„Zdej
                  dej
                  dej
                  fd„Zˆ xZS )ÚBridgeTowerSelfOutputc                 ó(  •— t         ‰| �  «        t        j                  |j                  |j                  «      | _        t        j                  |j                  |j                  ¬«      | _        t        j                  |j                  «      | _
        y rÃ   )rC   rD   r	   rL   rF   ÚdenserH   rI   ÚDropoutÚhidden_dropout_probÚdropoutrP   s     €r1   rD   zBridgeTowerSelfOutput.__init__°  s`   ø€ Ü‰ÑÔÜ—Y‘Y˜v×1Ñ1°6×3EÑ3EÓFˆŒ
ÜŸ™ f×&8Ñ&8¸f×>SÑ>SÔTˆŒÜ—z‘z &×"<Ñ"<Ó=ˆ�r0   r&   Úinput_tensorr™   c                 ór   — | j                  |«      }| j                  |«      }| j                  ||z   «      }|S r`   ©rç   rê   rH   ©rQ   r&   rë   s      r1   re   zBridgeTowerSelfOutput.forward¶  ó7   € ØŸ
™
 =Ó1ˆØŸ™ ]Ó3ˆØŸ™ }°|Ñ'CÓDˆØÐr0   ©r(   r)   r*   rD   r,   rf   re   rg   rh   s   @r1   rå   rå   ¯  ó1   ø„ ô>ð U§\¡\ð ÀÇÁð ÐRW×R^ÑR^÷ r0   rå   c                   óV   ‡ — e Zd Zˆ fd„Zdej
                  dej
                  fd„Zˆ xZS )ÚBridgeTowerIntermediatec                 ó  •— t         ‰| �  «        t        j                  |j                  |j
                  «      | _        t        |j                  t        «      rt        |j                     | _        y |j                  | _        y r`   )rC   rD   r	   rL   rF   Úintermediate_sizerç   Ú
isinstanceÚ
hidden_actÚstrr   Úintermediate_act_fnrP   s     €r1   rD   z BridgeTowerIntermediate.__init__¿  s]   ø€ Ü‰ÑÔÜ—Y‘Y˜v×1Ñ1°6×3KÑ3KÓLˆŒ
Ü�f×'Ñ'¬Ô-Ü'-¨f×.?Ñ.?Ñ'@ˆDÕ$à'-×'8Ñ'8ˆDÕ$r0   r&   r™   c                 óJ   — | j                  |«      }| j                  |«      }|S r`   )rç   rù   ©rQ   r&   s     r1   re   zBridgeTowerIntermediate.forwardÇ  s&   € ØŸ
™
 =Ó1ˆØ×0Ñ0°Ó?ˆØÐr0   rð   rh   s   @r1   ró   ró   ¾  s#   ø„ ô9ð U§\¡\ð °e·l±l÷ r0   ró   c                   ón   ‡ — e Zd Zˆ fd„Zdej
                  dej
                  dej
                  fd„Zˆ xZS )ÚBridgeTowerOutputc                 ó(  •— t         ‰| �  «        t        j                  |j                  |j
                  «      | _        t        j                  |j
                  |j                  ¬«      | _        t        j                  |j                  «      | _        y rÃ   )rC   rD   r	   rL   rõ   rF   rç   rH   rI   rè   ré   rê   rP   s     €r1   rD   zBridgeTowerOutput.__init__Ï  s`   ø€ Ü‰ÑÔÜ—Y‘Y˜v×7Ñ7¸×9KÑ9KÓLˆŒ
ÜŸ™ f×&8Ñ&8¸f×>SÑ>SÔTˆŒÜ—z‘z &×"<Ñ"<Ó=ˆ�r0   r&   rë   r™   c                 ór   — | j                  |«      }| j                  |«      }| j                  ||z   «      }|S r`   rí   rî   s      r1   re   zBridgeTowerOutput.forwardÕ  rï   r0   rð   rh   s   @r1   rý   rý   Î  rñ   r0   rý   c                   óV   ‡ — e Zd Zˆ fd„Zdej
                  dej
                  fd„Zˆ xZS )ÚBridgeTowerPoolerc                 ó²   •— t         ‰| �  «        t        j                  |j                  |j                  «      | _        t        j                  «       | _        y r`   )rC   rD   r	   rL   rF   rç   ÚTanhÚ
activationrP   s     €r1   rD   zBridgeTowerPooler.__init__Þ  s9   ø€ Ü‰ÑÔÜ—Y‘Y˜v×1Ñ1°6×3EÑ3EÓFˆŒ
ÜŸ'™'›)ˆ�r0   r&   r™   c                 ó\   — |d d …df   }| j                  |«      }| j                  |«      }|S )Nr   )rç   r  )rQ   r&   Úfirst_token_tensorÚpooled_outputs       r1   re   zBridgeTowerPooler.forwardã  s6   € ð +ª1¨a¨4Ñ0ÐØŸ
™
Ð#5Ó6ˆØŸ™¨Ó6ˆØÐr0   rð   rh   s   @r1   r  r  Ý  s#   ø„ ô$ð
 U§\¡\ð °e·l±l÷ r0   r  c                   óP  ‡ — e Zd Zdˆ fd„	Zdej
                  dej
                  fd„Z	 	 	 	 	 	 ddej
                  deej                     deej                     deej                     d	eej                     d
ee	e	ej                           dee
   de	ej
                     fd„Zˆ xZS )ÚBridgeTowerSelfAttentionc                 óâ  •— t         ‰| �  «        |j                  |j                  z  dk7  r2t	        |d«      s&t        d|j                  › d|j                  › d�«      ‚|j                  | _        t        |j                  |j                  z  «      | _        | j                  | j                  z  | _        t        j                  |j                  | j                  «      | _        t        j                  |j                  | j                  «      | _        t        j                  |j                  | j                  «      | _        t        j                  |j                  «      | _        |xs t#        |dd«      | _        | j$                  dk(  s| j$                  d	k(  rF|j&                  | _        t        j(                  d
|j&                  z  dz
  | j                  «      | _        |j,                  | _        y )Nr   Úembedding_sizezThe hidden size (z6) is not a multiple of the number of attention heads (ú)Úposition_embedding_typeÚabsoluteÚrelative_keyÚrelative_key_queryr€   r   )rC   rD   rF   Únum_attention_headsÚhasattrr·   r¿   Úattention_head_sizeÚall_head_sizer	   rL   ÚqueryÚkeyÚvaluerè   Úattention_probs_dropout_probrê   Úgetattrr  Úmax_position_embeddingsr‘   Údistance_embeddingÚ
is_decoder©rQ   rR   r  rS   s      €r1   rD   z!BridgeTowerSelfAttention.__init__î  s�  ø€ Ü‰ÑÔØ×Ñ × :Ñ :Ñ:¸aÒ?ÌÐPVÐXhÔHiÜØ# F×$6Ñ$6Ð#7ð 8Ø ×4Ñ4Ð5°Qð8óð ð
 $*×#=Ñ#=ˆÔ Ü#& v×'9Ñ'9¸F×<VÑ<VÑ'VÓ#WˆÔ Ø!×5Ñ5¸×8PÑ8PÑPˆÔä—Y‘Y˜v×1Ñ1°4×3EÑ3EÓFˆŒ
Ü—9‘9˜V×/Ñ/°×1CÑ1CÓDˆŒÜ—Y‘Y˜v×1Ñ1°4×3EÑ3EÓFˆŒ
ä—z‘z &×"EÑ"EÓFˆŒØ'>ò (
Ä'ØÐ-¨zóC
ˆÔ$ð ×'Ñ'¨>Ò9¸T×=YÑ=YÐ]qÒ=qØ+1×+IÑ+IˆDÔ(Ü&(§l¡l°1°v×7UÑ7UÑ3UÐXYÑ3YÐ[_×[sÑ[sÓ&tˆDÔ#à ×+Ñ+ˆ�r0   Úxr™   c                 ó¤   — |j                  «       d d | j                  | j                  fz   }|j                  |«      }|j	                  dddd«      S )Nrƒ   r   r€   r   r   )r�   r  r  r«   r¨   )rQ   r  Únew_x_shapes      r1   Útranspose_for_scoresz-BridgeTowerSelfAttention.transpose_for_scores  sL   € Ø—f‘f“h˜s �m t×'?Ñ'?À×AYÑAYÐ&ZÑZˆØ�F‰F�;ÓˆØ�y‰y˜˜A˜q !Ó$Ð$r0   r&   rU   Ú	head_maskÚencoder_hidden_statesÚencoder_attention_maskÚpast_key_valueÚoutput_attentionsc                 ó$  — | j                  |«      }|d u}	|	r|�|d   }
|d   }|}�n |	rC| j                  | j                  |«      «      }
| j                  | j                  |«      «      }|}n»|�y| j                  | j                  |«      «      }
| j                  | j                  |«      «      }t	        j
                  |d   |
gd¬«      }
t	        j
                  |d   |gd¬«      }n@| j                  | j                  |«      «      }
| j                  | j                  |«      «      }| j                  |«      }|d u}| j                  r|
|f}t	        j                  ||
j                  dd«      «      }| j                  dk(  s| j                  dk(  �r—|j                  d   |
j                  d   }}|rDt	        j                  |dz
  t        j                  |j                  ¬	«      j                  dd«      }n@t	        j                  |t        j                  |j                  ¬	«      j                  dd«      }t	        j                  |t        j                  |j                  ¬	«      j                  dd«      }||z
  }| j!                  || j"                  z   dz
  «      }|j%                  |j&                  ¬
«      }| j                  dk(  rt	        j(                  d||«      }||z   }nE| j                  dk(  r6t	        j(                  d||«      }t	        j(                  d|
|«      }||z   |z   }|t+        j,                  | j.                  «      z  }|�||z   }t0        j2                  j5                  |d¬«      }| j7                  |«      }|�||z  }t	        j                  ||«      }|j9                  dddd«      j;                  «       }|j=                  «       d d | j>                  fz   }|j                  |«      }|r||fn|f}| j                  r||fz   }|S )Nr   r   r€   r    rƒ   éþÿÿÿr  r  rW   r¶   zbhld,lrd->bhlrzbhrd,lrd->bhlrr   ) r  r!  r  r  r,   r¬   r  Úmatmulr¹   r  r¢   rÝ   ÚlongrY   r«   r”   r  r  r\   rX   ÚeinsumÚmathÚsqrtr  r	   r©   Úsoftmaxrê   r¨   Ú
contiguousr�   r  )rQ   r&   rU   r"  r#  r$  r%  r&  Úmixed_query_layerÚis_cross_attentionÚ	key_layerÚvalue_layerÚquery_layerÚ	use_cacheÚattention_scoresÚquery_lengthÚ
key_lengthÚposition_ids_lÚposition_ids_rÚdistanceÚpositional_embeddingÚrelative_position_scoresÚrelative_position_scores_queryÚrelative_position_scores_keyÚattention_probsÚcontext_layerÚnew_context_layer_shapeÚoutputss                               r1   re   z BridgeTowerSelfAttention.forward  sç  € ð !ŸJ™J }Ó5Ðð
 3¸$Ð>Ðá .Ð"<à& qÑ)ˆIØ(¨Ñ+ˆKØ3ŠNÙØ×1Ñ1°$·(±(Ð;PÓ2QÓRˆIØ×3Ñ3°D·J±JÐ?TÓ4UÓVˆKØ3‰NØÐ'Ø×1Ñ1°$·(±(¸=Ó2IÓJˆIØ×3Ñ3°D·J±J¸}Ó4MÓNˆKÜŸ	™	 >°!Ñ#4°iÐ"@ÀaÔHˆIÜŸ)™) ^°AÑ%6¸Ð$DÈ!ÔL‰Kà×1Ñ1°$·(±(¸=Ó2IÓJˆIØ×3Ñ3°D·J±J¸}Ó4MÓNˆKà×/Ñ/Ð0AÓBˆà"¨$Ð.ˆ	Ø�?Š?ð (¨Ð5ˆNô !Ÿ<™<¨°Y×5HÑ5HÈÈRÓ5PÓQÐà×'Ñ'¨>Ò9¸T×=YÑ=YÐ]qÓ=qØ'2×'8Ñ'8¸Ñ';¸Y¿_¹_ÈQÑ=O˜*ˆLÙÜ!&§¡¨j¸1©nÄEÇJÁJÐWd×WkÑWkÔ!l×!qÑ!qØ˜ó"‘ô "'§¡¨lÄ%Ç*Á*ÐUb×UiÑUiÔ!j×!oÑ!oÐprÐtuÓ!v�Ü"Ÿ\™\¨*¼E¿J¹JÈ}×OcÑOcÔd×iÑiÐjkÐmoÓpˆNØ%¨Ñ6ˆHà#'×#:Ñ#:¸8Àd×FbÑFbÑ;bÐefÑ;fÓ#gÐ Ø#7×#:Ñ#:À×ARÑARÐ#:Ó#SÐ à×+Ñ+¨~Ò=Ü+0¯<©<Ð8HÈ+ÐWkÓ+lÐ(Ø#3Ð6NÑ#NÑ Ø×-Ñ-Ð1EÒEÜ16·±Ð>NÐP[Ð]qÓ1rÐ.Ü/4¯|©|Ð<LÈiÐYmÓ/nÐ,Ø#3Ð6TÑ#TÐWsÑ#sÐ à+¬d¯i©i¸×8PÑ8PÓ.QÑQÐØÐ%à/°.Ñ@Ðô Ÿ-™-×/Ñ/Ð0@ÀbÐ/ÓIˆð Ÿ,™, Ó7ˆð Ð Ø-°	Ñ9ˆOäŸ™ _°kÓBˆà%×-Ñ-¨a°°A°qÓ9×DÑDÓFˆØ"/×"4Ñ"4Ó"6°s¸Ð";¸t×?QÑ?QÐ>SÑ"SÐØ%×*Ñ*Ð+BÓCˆá6G�= /Ñ2ÈmÐM]ˆà�?Š?Ø Ð 1Ñ1ˆGØˆr0   r`   ©NNNNNF)r(   r)   r*   rD   r,   rf   r!  r   r-   r   r]   re   rg   rh   s   @r1   r	  r	  í  så   ø„ õ,ð4% e§l¡lð %°u·|±|ó %ð 7;Ø15Ø=AØ>BØDHØ,1ñcà—|‘|ðcð ! ×!2Ñ!2Ñ3ðcð ˜E×-Ñ-Ñ.ð	cð
  (¨×(9Ñ(9Ñ:ðcð !)¨×):Ñ):Ñ ;ðcð !  u¨U×->Ñ->Ñ'?Ñ!@ÑAðcð $ D™>ðcð 
ˆu�|‰|Ñ	÷cr0   r	  Úeagerc                   ó  ‡ — e Zd Zdˆ fd„	Zd„ Z	 	 	 	 	 	 ddej                  deej                     deej                     deej                     deej                     dee	e	ej                           d	ee
   d
e	ej                     fd„Zˆ xZS )ÚBridgeTowerAttentionc                 óž   •— t         ‰| �  «        t        |j                     ||¬«      | _        t        |«      | _        t        «       | _        y )N©r  )	rC   rD   Ú#BRIDGE_TOWER_SELF_ATTENTION_CLASSESÚ_attn_implementationrQ   rå   ÚoutputÚsetÚpruned_headsr  s      €r1   rD   zBridgeTowerAttention.__init__z  sC   ø€ Ü‰ÑÔÜ7¸×8SÑ8SÑTØÐ,Cô
ˆŒ	ô ,¨FÓ3ˆŒÜ›EˆÕr0   c                 ó>  — t        |«      dk(  ry t        || j                  j                  | j                  j                  | j
                  «      \  }}t        | j                  j                  |«      | j                  _        t        | j                  j                  |«      | j                  _        t        | j                  j                  |«      | j                  _	        t        | j                  j                  |d¬«      | j                  _        | j                  j                  t        |«      z
  | j                  _        | j                  j                  | j                  j                  z  | j                  _        | j
                  j                  |«      | _        y )Nr   r   r    )Úlenr   rQ   r  r  rN  r   r  r  r  rL  rç   r  Úunion)rQ   ÚheadsÚindexs      r1   Úprune_headsz BridgeTowerAttention.prune_heads‚  s  € Üˆu‹:˜Š?ØÜ7Ø�4—9‘9×0Ñ0°$·)±)×2OÑ2OÐQU×QbÑQbó
‰ˆˆuô
 -¨T¯Y©Y¯_©_¸eÓDˆ�	‰	ŒÜ*¨4¯9©9¯=©=¸%Ó@ˆ�	‰	ŒÜ,¨T¯Y©Y¯_©_¸eÓDˆ�	‰	ŒÜ.¨t¯{©{×/@Ñ/@À%ÈQÔOˆ�‰Ôð )-¯	©	×(EÑ(EÌÈEË
Ñ(Rˆ�	‰	Ô%Ø"&§)¡)×"?Ñ"?À$Ç)Á)×B_ÑB_Ñ"_ˆ�	‰	ÔØ ×-Ñ-×3Ñ3°EÓ:ˆÕr0   r&   rU   r"  r#  r$  r%  r&  r™   c           	      óp   — | j                  |||||||«      }| j                  |d   |«      }	|	f|dd  z   }
|
S )Nr   r   )rQ   rL  )rQ   r&   rU   r"  r#  r$  r%  r&  Úself_outputsÚattention_outputrC  s              r1   re   zBridgeTowerAttention.forward”  sW   € ð —y‘yØØØØ!Ø"ØØó
ˆð  Ÿ;™; |°A¡¸ÓFÐØ#Ð%¨°Q°RÐ(8Ñ8ˆØˆr0   r`   rD  )r(   r)   r*   rD   rT  r,   rf   r   r-   r   r]   re   rg   rh   s   @r1   rG  rG  y  sÆ   ø„ õ"ò;ð* 7;Ø15Ø=AØ>BØDHØ,1ñà—|‘|ðð ! ×!2Ñ!2Ñ3ðð ˜E×-Ñ-Ñ.ð	ð
  (¨×(9Ñ(9Ñ:ðð !)¨×):Ñ):Ñ ;ðð !  u¨U×->Ñ->Ñ'?Ñ!@ÑAðð $ D™>ðð 
ˆu�|‰|Ñ	÷r0   rG  c                   ó6   ‡ — e Zd Zˆ fd„Z	 	 	 	 	 dd„Zd„ Zˆ xZS )ÚBridgeTowerBertCrossLayerc                 ó  •— t         ‰| �  «        |j                  | _        d| _        t	        |«      | _        |j                  | _        |j                  | _        t	        |«      | _        t        |«      | _
        t        |«      | _        y rl   )rC   rD   Úchunk_size_feed_forwardÚseq_len_dimrG  r^   r  Úadd_cross_attentionÚcrossattentionró   Úintermediaterý   rL  rP   s     €r1   rD   z"BridgeTowerBertCrossLayer.__init__­  sq   ø€ Ü‰ÑÔØ'-×'EÑ'EˆÔ$ØˆÔÜ-¨fÓ5ˆŒØ ×+Ñ+ˆŒØ#)×#=Ñ#=ˆÔ Ü2°6Ó:ˆÔÜ3°FÓ;ˆÔÜ'¨Ó/ˆ�r0   c           	      óô   — | j                  ||d |d ¬«      }|d   }	|dd  }
| j                  |	||||||¬«      }|d   }	|
|dd z   }
t        | j                  | j                  | j
                  |	«      }|f|
z   }
|
S )N)rU   r"  r&  r%  r   r   )rU   r"  r#  r$  r%  r&  rƒ   )r^   r^  r   Úfeed_forward_chunkr[  r\  )rQ   r&   r#  rU   r"  r$  r%  r&  Úself_attention_outputsrW  rC  Úcross_attention_outputsÚlayer_outputs                r1   re   z!BridgeTowerBertCrossLayer.forward¸  sÅ   € ð "&§¡ØØ)ØØ/Øð "0ó "
Ðð 2°!Ñ4Ðð )¨¨Ð,ˆà"&×"5Ñ"5ØØ)ØØ"7Ø#9Ø)Ø/ð #6ó #
Ðð 3°1Ñ5ÐàÐ3°A°bÐ9Ñ9ˆä0Ø×#Ñ# T×%AÑ%AÀ4×CSÑCSÐUeó
ˆð  �/ GÑ+ˆàˆr0   c                 óL   — | j                  |«      }| j                  ||«      }|S r`   ©r_  rL  ©rQ   rW  Úintermediate_outputrd  s       r1   ra  z,BridgeTowerBertCrossLayer.feed_forward_chunkä  ó,   € Ø"×/Ñ/Ð0@ÓAÐØ—{‘{Ð#6Ð8HÓIˆØÐr0   )NNNNF)r(   r)   r*   rD   re   ra  rg   rh   s   @r1   rY  rY  ¬  s$   ø„ ô	0ð ØØ#ØØó*öXr0   rY  c                   ó  ‡ — e Zd Zˆ fd„Z	 	 	 	 	 	 ddej
                  deej                     deej                     deej                     deej                     deeeej                           dee	   d	eej
                     fd
„Z
d„ Zˆ xZS )ÚBridgeTowerTextLayerc                 óf  •— t         ‰| �  «        |j                  | _        d| _        t	        |«      | _        |j                  | _        |j                  | _        | j                  r,| j                  st        | › d�«      ‚t	        |d¬«      | _	        t        |«      | _        t        |«      | _        y )Nr   z> should be used as a decoder model if cross attention is addedr  rI  )rC   rD   r[  r\  rG  r^   r  r]  r·   r^  ró   r_  rý   rL  rP   s     €r1   rD   zBridgeTowerTextLayer.__init__ë  s—   ø€ Ü‰ÑÔØ'-×'EÑ'EˆÔ$ØˆÔÜ-¨fÓ5ˆŒØ ×+Ñ+ˆŒØ#)×#=Ñ#=ˆÔ Ø×#Ò#Ø—?’?Ü  D 6Ð)gÐ!hÓiÐiÜ"6°vÐWaÔ"bˆDÔÜ3°FÓ;ˆÔÜ'¨Ó/ˆ�r0   r&   rU   r"  r#  r$  r%  r&  r™   c           	      óÒ  — |�|d d nd }| j                  |||||¬«      }	|	d   }
| j                  r|	dd }|	d   }n|	dd  }d }| j                  rT|�Rt        | d«      st        d| › d�«      ‚|�|d	d  nd }| j	                  |
||||||«      }|d   }
||dd z   }|d   }|z   }t        | j                  | j                  | j                  |
«      }|f|z   }| j                  r|fz   }|S )
Nr€   )r&  r%  r   r   rƒ   r^  z'If `encoder_hidden_states` are passed, z` has to be instantiated with cross-attention layers by setting `config.add_cross_attention=True`r(  )	r^   r  r  r·   r^  r   ra  r[  r\  )rQ   r&   rU   r"  r#  r$  r%  r&  Úself_attn_past_key_valuerb  rW  rC  Úpresent_key_valueÚcross_attn_present_key_valueÚcross_attn_past_key_valuerc  rd  s                    r1   re   zBridgeTowerTextLayer.forwardù  s}  € ð :HÐ9S >°"°1Ñ#5ÐY]Ð Ø!%§¡ØØØØ/Ø3ð "0ó "
Ðð 2°!Ñ4Ðð �?Š?Ø,¨Q¨rÐ2ˆGØ 6°rÑ :Ñà,¨Q¨RÐ0ˆGà'+Ð$Ø�?Š?Ð4Ð@Ü˜4Ð!1Ô2Ü Ø=¸d¸Vð DDð Dóð ð @NÐ?Y¨°r°sÑ(;Ð_cÐ%Ø&*×&9Ñ&9Ø ØØØ%Ø&Ø)Ø!ó'Ð#ð  7°qÑ9ÐØÐ 7¸¸"Ð =Ñ=ˆGð ,CÀ2Ñ+FÐ(Ø 1Ð4PÑ PÐä0Ø×#Ñ# T×%AÑ%AÀ4×CSÑCSÐUeó
ˆð  �/ GÑ+ˆð �?Š?ØÐ!2Ð 4Ñ4ˆGàˆr0   c                 óL   — | j                  |«      }| j                  ||«      }|S r`   rf  rg  s       r1   ra  z'BridgeTowerTextLayer.feed_forward_chunk:  ri  r0   rD  )r(   r)   r*   rD   r,   rf   r   r-   r   r]   re   ra  rg   rh   s   @r1   rk  rk  ê  sÇ   ø„ ô0ð" 7;Ø15Ø=AØ>BØDHØ,1ñ?à—|‘|ð?ð ! ×!2Ñ!2Ñ3ð?ð ˜E×-Ñ-Ñ.ð	?ð
  (¨×(9Ñ(9Ñ:ð?ð !)¨×):Ñ):Ñ ;ð?ð !  u¨U×->Ñ->Ñ'?Ñ!@ÑAð?ð $ D™>ð?ð 
ˆu�|‰|Ñ	ó?öBr0   rk  c                   óD  ‡ — e Zd Zˆ fd„Z	 	 	 	 	 	 	 	 	 ddej
                  deej                     deej                     deej                     deej                     deeeej                           dee	   d	ee	   d
ee	   dee	   de
eej
                     ef   fd„Zˆ xZS )ÚBridgeTowerTextEncoderc                 óÐ   •— t         ‰| �  «        || _        t        j                  t        |j                  «      D �cg c]  }t        |«      ‘Œ c}«      | _        d| _	        y c c}w ©NF)
rC   rD   rR   r	   ro   rp   rm   rk  rd   Úgradient_checkpointingrs   s      €r1   rD   zBridgeTowerTextEncoder.__init__B  sP   ø€ Ü‰ÑÔØˆŒÜ—]‘]Ì%ÐPV×PhÑPhÓJiÖ#jÀQÔ$8¸Õ$@Ò#jÓkˆŒ
Ø&+ˆÕ#ùò $ks   ½A#r&   rU   r"  r#  r$  Úpast_key_valuesr5  r&  Úoutput_hidden_statesÚreturn_dictr™   c                 óš  — |	rdnd }|rdnd }|r| j                   j                  rdnd }| j                  r%| j                  r|rt        j                  d«       d}|rdnd }t        | j                  «      D ]¤  \  }}|	r||fz   }|�||   nd }|�||   nd }| j                  r/| j                  r#| j                  |j                  |||||||«      }n ||||||||«      }|d   }|r	||d   fz  }|sŒ|||d   fz   }| j                   j                  sŒœ||d   fz   }Œ¦ |	r||fz   }|
st        d„ |||||fD «       «      S t        |||||¬	«      S )
Nr/   zZ`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...Fr   rƒ   r   r€   c              3   ó$   K  — | ]  }|�|–— Œ
 y ­wr`   r/   ©Ú.0Úvs     r1   ú	<genexpr>z1BridgeTowerTextEncoder.forward.<locals>.<genexpr>Š  s   è ø€ ò 
àð �=ô ñ
ùó   ‚)Úlast_hidden_staterx  r&   r'   Úcross_attentions)rR   r]  rw  ÚtrainingÚloggerÚwarning_onceÚ	enumeraterd   Ú_gradient_checkpointing_funcÚ__call__Útupler   )rQ   r&   rU   r"  r#  r$  rx  r5  r&  ry  rz  Úall_hidden_statesÚall_self_attentionsÚall_cross_attentionsÚnext_decoder_cacheÚiÚlayer_moduleÚlayer_head_maskr%  Úlayer_outputss                       r1   re   zBridgeTowerTextEncoder.forwardH  sÎ  € ñ #7™B¸DÐÙ$5™b¸4ÐÙ%6¸4¿;¹;×;ZÒ;Z™rÐ`dÐà×&Ò&¨4¯=ª=ÙÜ×#Ñ#Øpôð "�	á#,™R°$ÐÜ(¨¯©Ó4ò #	V‰OˆAˆ|Ù#Ø$5¸Ð8HÑ$HÐ!à.7Ð.C˜i¨šlÈˆOØ3BÐ3N˜_¨QÒ/ÐTXˆNà×*Ò*¨t¯}ª}Ø $× AÑ AØ ×)Ñ)Ø!Ø"Ø#Ø)Ø*Ø"Ø%ó	!‘ñ !-Ø!Ø"Ø#Ø)Ø*Ø"Ø%ó!�ð *¨!Ñ,ˆMÙØ" }°RÑ'8Ð&:Ñ:Ð"Ú Ø&9¸]È1Ñ=MÐ<OÑ&OÐ#Ø—;‘;×2Ó2Ø+?À=ÐQRÑCSÐBUÑ+UÑ(ðG#	VñJ  Ø 1°]Ð4DÑ DÐáÜñ 
ð "Ø&Ø%Ø'Ø(ðô
ó 
ð 
ô 9Ø+Ø.Ø+Ø*Ø1ô
ð 	
r0   )	NNNNNNFFT)r(   r)   r*   rD   r,   rf   r   r-   r   r]   r   r   re   rg   rh   s   @r1   rt  rt  A  s  ø„ ô,ð 7;Ø15Ø=AØ>BØEIØ$(Ø,1Ø/4Ø&*ñS
à—|‘|ðS
ð ! ×!2Ñ!2Ñ3ðS
ð ˜E×-Ñ-Ñ.ð	S
ð
  (¨×(9Ñ(9Ñ:ðS
ð !)¨×):Ñ):Ñ ;ðS
ð " %¨¨e×.?Ñ.?Ñ(@Ñ"AÑBðS
ð ˜D‘>ðS
ð $ D™>ðS
ð ' t™nðS
ð ˜d‘^ðS
ð 
ˆu�U—\‘\Ñ"Ð$MÐMÑ	N÷S
r0   rt  c                   ó2   ‡ — e Zd ZdZˆ fd„Z	 dd„Zd„ Zˆ xZS )ÚBridgeTowerTextEmbeddingszV
    Same as BertEmbeddings with a tiny tweak for positional embeddings indexing.
    c                 óÖ  •— t         ‰| �  «        t        j                  |j                  |j
                  |j                  ¬«      | _        t        j                  |j                  |j
                  «      | _	        t        j                  |j                  |j
                  «      | _        t        j                  |j
                  |j                  ¬«      | _        t        j                  |j                  «      | _        t#        |dd«      | _        | j'                  dt)        j*                  |j                  «      j-                  d«      d¬«       | j'                  d	t)        j.                  | j0                  j3                  «       t(        j4                  ¬
«      d¬«       |j                  | _        t        j                  |j                  |j
                  | j6                  ¬«      | _	        y )N)Úpadding_idxr=   r  r  r�   r‚   Fr„   Útoken_type_idsr¶   )rC   rD   r	   r‘   Ú
vocab_sizerF   Úpad_token_idÚword_embeddingsr  Úposition_embeddingsÚtype_vocab_sizeÚtoken_type_embeddingsrH   rI   rè   ré   rê   r  r  r“   r,   r”   r•   Úzerosr�   r�   r*  r–  rP   s     €r1   rD   z"BridgeTowerTextEmbeddings.__init__¥  si  ø€ Ü‰ÑÔÜ!Ÿ|™|¨F×,=Ñ,=¸v×?QÑ?QÐ_e×_rÑ_rÔsˆÔÜ#%§<¡<°×0NÑ0NÐPV×PbÑPbÓ#cˆÔ Ü%'§\¡\°&×2HÑ2HÈ&×J\ÑJ\Ó%]ˆÔ"ô Ÿ™ f×&8Ñ&8¸f×>SÑ>SÔTˆŒÜ—z‘z &×"<Ñ"<Ó=ˆŒä'.¨vÐ7PÐR\Ó']ˆÔ$Ø×ÑØœEŸL™L¨×)GÑ)GÓH×OÑOÐPWÓXÐejð 	ô 	
ð 	×ÑØœeŸk™k¨$×*;Ñ*;×*@Ñ*@Ó*BÌ%Ï*É*ÔUÐbgð 	ô 	
ð
 "×.Ñ.ˆÔÜ#%§<¡<Ø×*Ñ*¨F×,>Ñ,>ÈD×L\ÑL\ô$
ˆÕ r0   c                 ó€  — |€+|�t        || j                  |«      }n| j                  |«      }|�|j                  «       }n|j                  «       d d }|d   }|€st	        | d«      r-| j
                  d d …d |…f   }|j                  |d   |«      }	|	}n:t        j                  |t        j                  | j                  j                  ¬«      }|€| j                  |«      }| j                  |«      }
||
z   }| j                  dk(  r| j                  |«      }||z  }| j!                  |«      }| j#                  |«      }|S )Nrƒ   r   r—  r   rW   r  )Ú"create_position_ids_from_input_idsr–  Ú&create_position_ids_from_inputs_embedsr�   r  r—  r•   r,   rž  r*  r�   rY   rš  r�  r  r›  rH   rê   )rQ   Ú	input_idsr—  r�   Úinputs_embedsÚpast_key_values_lengthÚinput_shapeÚ
seq_lengthÚbuffered_token_type_idsÚ buffered_token_type_ids_expandedr�  r–   r›  s                r1   re   z!BridgeTowerTextEmbeddings.forward¾  sR  € ð ÐØÐ$äAÀ)ÈT×M]ÑM]Ð_uÓv‘à#×JÑJÈ=ÓY�àÐ Ø#Ÿ.™.Ó*‰Kà'×,Ñ,Ó.¨s°Ð3ˆKà  ‘^ˆ
ð
 Ð!Ü�tÐ-Ô.Ø*.×*=Ñ*=ºaÀÀ*À¸nÑ*MÐ'Ø3J×3QÑ3QÐR]Ð^_ÑR`ÐblÓ3mÐ0Ø!A‘ä!&§¡¨[ÄÇ
Á
ÐSW×SdÑSd×SkÑSkÔ!l�àÐ Ø ×0Ñ0°Ó;ˆMØ $× :Ñ :¸>Ó JÐà"Ð%:Ñ:ˆ
Ø×'Ñ'¨:Ò5Ø"&×":Ñ":¸<Ó"HÐØÐ-Ñ-ˆJØ—^‘^ JÓ/ˆ
Ø—\‘\ *Ó-ˆ
ØÐr0   c                 ó  — |j                  «       dd }|d   }t        j                  | j                  dz   || j                  z   dz   t        j                  |j
                  ¬«      }|j                  d«      j                  |«      S )z×
        We are provided embeddings directly. We cannot infer which are padded so just generate sequential position ids.

        Args:
            inputs_embeds: torch.Tensor

        Returns: torch.Tensor
        Nrƒ   r   rW   r   )r�   r,   r”   r–  r*  rY   r¤   r•   )rQ   r£  r¥  Úsequence_lengthr�   s        r1   r¡  z@BridgeTowerTextEmbeddings.create_position_ids_from_inputs_embedsæ  s€   € ð $×(Ñ(Ó*¨3¨BÐ/ˆØ% a™.ˆä—|‘|Ø×Ñ˜qÑ  /°D×4DÑ4DÑ"DÀqÑ"HÔPU×PZÑPZÐcp×cwÑcwô
ˆð ×%Ñ% aÓ(×/Ñ/°Ó<Ð<r0   )NNNNr   )r(   r)   r*   r+   rD   re   r¡  rg   rh   s   @r1   r”  r”  Ÿ  s   ø„ ñô

ð4 rsó&öP=r0   r”  c                 ó¾   — | j                  |«      j                  «       }t        j                  |d¬«      j	                  |«      |z   |z  }|j                  «       |z   S )a  
    Replace non-padding symbols with their position numbers. Position numbers begin at padding_idx+1. Padding symbols
    are ignored. This is modified from fairseq's `utils.make_positions`.

    Args:
        x: torch.Tensor x:

    Returns: torch.Tensor
    r   r    )Úner¿   r,   ÚcumsumÚtype_asr*  )r¢  r–  r¤  ÚmaskÚincremental_indicess        r1   r   r   ù  sW   € ð �<‰<˜Ó$×(Ñ(Ó*€DÜ Ÿ<™<¨°!Ô4×<Ñ<¸TÓBÐE[Ñ[Ð_cÑcÐØ×#Ñ#Ó%¨Ñ3Ð3r0   c                   ó.   — e Zd ZdZeZdZdZddgZdZ	d„ Z
y)	ÚBridgeTowerPreTrainedModelz†
    An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained
    models.
    ÚbridgetowerFr	  r:   rx  c                 óò  — t        |t        «      �r¿|j                  j                  j                  dz  d|j                  j                  j
                  z  dz  z  }|j                  j                  j                  dz  }d|j                  j                  j                  z  dz  }|j                  j                  j                  D �]Q  }t        j                  j                  |j                  j                  || j                  j                  z  ¬«       t        j                  j                  |j                  j                  j                  || j                  j                  z  ¬«       t        j                  j                  |j                   j"                  j                  || j                  j                  z  ¬«       t        j                  j                  |j                   j$                  j                  || j                  j                  z  ¬«       �ŒT t        j                  j                  |j                  j&                  j(                  || j                  j                  z  ¬«       t        j                  j                  |j                  j&                  j*                  j                  || j                  j                  z  ¬«       nÛt        |t        j,                  t        j.                  t        j0                  f«      r?|j                  j2                  j                  dd| j                  j                  z  ¬«       nct        |t        j4                  «      rI|j6                  j2                  j9                  «        |j                  j2                  j;                  d«       t        |t        j,                  «      r2|j6                  �%|j6                  j2                  j9                  «        y y y )Ng      à¿r€   )Ústdg        gš™™™™™©?)Úmeanrµ  rÙ   )rö   ÚBridgeTowerVisionModelÚvisualrÅ   rF   rm   rq   r	   ÚinitÚnormal_rG   Úin_proj_weightrR   Úinitializer_factorÚout_projr£   rM   r?   rB   r–   r‹   r’   rL   rŒ   r‘   ÚdatarH   r   Úzero_Úfill_)rQ   ÚmoduleÚproj_stdÚattn_stdÚfc_stdrw   s         r1   Ú_init_weightsz(BridgeTowerPreTrainedModel._init_weights  s   € Ü�fÔ4Õ5ØŸ™×1Ñ1×=Ñ=¸tÑCØ�V—]‘]×.Ñ.×@Ñ@Ñ@ÀTÑIñˆHð —}‘}×0Ñ0×<Ñ<¸dÑBˆHØ˜&Ÿ-™-×3Ñ3×?Ñ?Ñ?ÀDÑHˆFØŸ™×2Ñ2×<Ñ<ó h�Ü—‘—‘ §
¡
× 9Ñ 9¸xÈ$Ï+É+×JhÑJhÑ?h�ÔiÜ—‘—‘ §
¡
× 3Ñ 3× :Ñ :ÀÈ4Ï;É;×KiÑKiÑ@i�ÔjÜ—‘—‘ §	¡	§¡× 5Ñ 5¸6ÀDÇKÁK×DbÑDbÑ;b�ÔcÜ—‘—‘ §	¡	× 0Ñ 0× 7Ñ 7¸XÈÏÉ×HfÑHfÑ=f�Ögð	hô �G‰G�O‰O˜FŸM™M×4Ñ4×DÑDÈ(ÐUY×U`ÑU`×UsÑUsÑJsˆOÔtÜ�G‰G�O‰OØ—‘×(Ñ(×;Ñ;×BÑBÈÐSW×S^ÑS^×SqÑSqÑHqð õ ô ˜¤§¡¬B¯I©I´r·|±|Ð DÔEØ�M‰M×Ñ×&Ñ&¨C°T¸D¿K¹K×<ZÑ<ZÑ5ZÐ&Õ[Ü˜¤§¡Ô-Ø�K‰K×Ñ×"Ñ"Ô$Ø�M‰M×Ñ×$Ñ$ SÔ)ä�fœbŸi™iÔ(¨V¯[©[Ð-DØ�K‰K×Ñ×"Ñ"Õ$ð .EÐ(r0   N)r(   r)   r*   r+   r   Úconfig_classÚbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_skip_keys_device_placementrÅ  r/   r0   r1   r²  r²  	  s1   „ ñð
 %€LØ%ÐØ&+Ð#Ø3Ð5SÐTÐØ"3Ðó%r0   r²  c                   ó:   ‡ — e Zd ZeZˆ fd„Zed„ «       Zdd„Zˆ xZ	S )r·  c                 óD   •— t         ‰| �  |«       t        |«      | _        y r`   )rC   rD   rÁ   r¸  rP   s     €r1   rD   zBridgeTowerVisionModel.__init__3  s   ø€ Ü‰Ñ˜Ô Ü2°6Ó:ˆ�r0   c                 ój   — | j                   j                  j                  j                  j                  S r`   )r¸  r–   rŽ   r£   rX   ©rQ   s    r1   rX   zBridgeTowerVisionModel.dtype7  s$   € à�{‰{×%Ñ%×5Ñ5×<Ñ<×BÑBÐBr0   c                 óZ   — | j                  |j                  | j                  «      ||«      S r`   )r¸  ÚtyperX   )rQ   ÚimageÚ
image_maskr²   s       r1   re   zBridgeTowerVisionModel.forward;  s#   € Ø�{‰{˜5Ÿ:™: d§j¡jÓ1°:Ð?WÓXÐXr0   rv  )
r(   r)   r*   r   rÆ  rD   ÚpropertyrX   re   rg   rh   s   @r1   r·  r·  0  s)   ø„ Ø*€Lô;ð ñCó ðC÷Yr0   r·  c                   óÂ  ‡ — e Zd ZdZeZdˆ fd„	Zd„ Zd„ Zd„ Z		 	 	 	 	 	 	 	 	 	 	 	 	 dde
ej                     de
ej                     de
ej                     d	e
ej                     d
e
ej                     de
ej                     de
ej                     de
ej                     de
eej                        de
e   de
e   de
e   de
e   deeej                     ef   fd„Zˆ xZS )ÚBridgeTowerTextModela*  

    The model can behave as an encoder (with only self-attention) as well as a decoder, in which case a layer of
    cross-attention is added between the self-attention layers, following the architecture described in *Attention is
    all you need*_ by Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit, Llion Jones, Aidan N. Gomez, Lukasz
    Kaiser and Illia Polosukhin.

    To behave as an decoder the model needs to be initialized with the `is_decoder` argument of the configuration set
    to `True`. To be used in a Seq2Seq model, the model needs to initialized with both `is_decoder` argument and
    `add_cross_attention` set to `True`; an `encoder_hidden_states` is then expected as an input to the forward pass.

    .. _*Attention is all you need*: https://arxiv.org/abs/1706.03762

    c                 óº   •— t         ‰| �  |«       || _        t        |«      | _        t        |«      | _        |rt        |«      nd | _        | j                  «        y r`   )
rC   rD   rR   r”  r–   rt  Úencoderr  ÚpoolerÚ	post_init)rQ   rR   Úadd_pooling_layerrS   s      €r1   rD   zBridgeTowerTextModel.__init__Q  sL   ø€ Ü‰Ñ˜Ô ØˆŒä3°FÓ;ˆŒÜ-¨fÓ5ˆŒá3DÔ'¨Ô/È$ˆŒð 	�‰Õr0   c                 ó.   — | j                   j                  S r`   ©r–   rš  rÎ  s    r1   Úget_input_embeddingsz)BridgeTowerTextModel.get_input_embeddings]  s   € Ø�‰×.Ñ.Ð.r0   c                 ó&   — || j                   _        y r`   rÜ  ©rQ   r  s     r1   Úset_input_embeddingsz)BridgeTowerTextModel.set_input_embeddings`  s   € Ø*/ˆ�‰Õ'r0   c                 ó˜   — |j                  «       D ]7  \  }}| j                  j                  |   j                  j	                  |«       Œ9 y)z�
        Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base
        class PreTrainedModel
        N)ra   r×  rd   r^   rT  )rQ   Úheads_to_prunerd   rR  s       r1   Ú_prune_headsz!BridgeTowerTextModel._prune_headsc  sE   € ð
 +×0Ñ0Ó2ò 	C‰LˆE�5Ø�L‰L×Ñ˜uÑ%×/Ñ/×;Ñ;¸EÕBñ	Cr0   r¢  rU   r—  r�   r"  r£  r#  r$  rx  r5  r&  ry  rz  r™   c                 óœ  — |�|n| j                   j                  }|�|n| j                   j                  }|�|n| j                   j                  }| j                   j                  r|
�|
n| j                   j
                  }
nd}
|�|�t        d«      ‚|�#| j                  ||«       |j                  «       }n!|�|j                  «       dd }nt        d«      ‚|\  }}|�|j                  n|j                  }|	�|	d   d   j                  d   nd}|€t        j                  |||z   f|¬«      }|€pt        | j                  d	«      r4| j                  j                  dd…d|…f   }|j!                  ||«      }|}n&t        j"                  |t        j$                  |¬
«      }| j'                  ||«      }| j                   j                  rE|�C|j                  «       \  }}}||f}|€t        j                  ||¬«      }| j)                  |«      }nd}| j+                  || j                   j,                  «      }| j                  |||||¬«      }| j/                  ||||||	|
|||¬«
      }|d   }| j0                  �| j1                  |«      nd}|s
||f|dd z   S t3        |||j4                  |j6                  |j8                  |j:                  ¬«      S )a  
        encoder_hidden_states  (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
            Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if
            the model is configured as a decoder.
        encoder_attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in
            the cross-attention if the model is configured as a decoder. Mask values selected in `[0, 1]`:

            - 1 for tokens that are **not masked**,
            - 0 for tokens that are **masked**.
        past_key_values (`tuple(tuple(torch.FloatTensor))` of length `config.n_layers` with each tuple having 4 tensors of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`):
            Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding.

            If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that
            don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all
            `decoder_input_ids` of shape `(batch_size, sequence_length)`.
        use_cache (`bool`, *optional*):
            If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see
            `past_key_values`).
        NFzDYou cannot specify both input_ids and inputs_embeds at the same timerƒ   z5You have to specify either input_ids or inputs_embedsr   r€   ©rY   r—  rW   )r¢  r�   r—  r£  r¤  )	rU   r"  r#  r$  rx  r5  r&  ry  rz  r   )r‚  r%   rx  r&   r'   rƒ  )rR   r&  ry  Úuse_return_dictr  r5  r·   Ú%warn_if_padding_and_no_attention_maskr�   rY   r¢   r,   Úonesr  r–   r—  r•   rž  r*  Úget_extended_attention_maskÚinvert_attention_maskÚget_head_maskrm   r×  rØ  r   rx  r&   r'   rƒ  )rQ   r¢  rU   r—  r�   r"  r£  r#  r$  rx  r5  r&  ry  rz  r¥  rº   r¦  rY   r¤  r§  r¨  Úextended_attention_maskÚencoder_batch_sizeÚencoder_sequence_lengthrc   Úencoder_hidden_shapeÚencoder_extended_attention_maskÚembedding_outputÚencoder_outputsÚsequence_outputr  s                                  r1   re   zBridgeTowerTextModel.forwardl  s  € ðH 2CÐ1NÑ-ÐTX×T_ÑT_×TqÑTqÐà$8Ð$DÑ È$Ï+É+×JjÑJjð 	ð &1Ð%<‘kÀ$Ç+Á+×B]ÑB]ˆà�;‰;×!Ò!Ø%.Ð%:™	ÀÇÁ×@UÑ@U‰IàˆIàÐ  ]Ð%>ÜÐcÓdÐdØÐ"Ø×6Ñ6°yÀ.ÔQØ#Ÿ.™.Ó*‰KØÐ&Ø'×,Ñ,Ó.¨s°Ð3‰KäÐTÓUÐUà!,Ñˆ
�JØ%.Ð%:�×!Ò!À×@TÑ@Tˆð DSÐC^ °Ñ!3°AÑ!6×!<Ñ!<¸QÒ!?ÐdeÐàÐ!Ü"ŸZ™Z¨*°jÐCYÑ6YÐ)ZÐdjÔkˆNàÐ!Ü�t—‘Ð(8Ô9Ø*.¯/©/×*HÑ*HÊÈKÈZÈKÈÑ*XÐ'Ø3J×3QÑ3QÐR\Ð^hÓ3iÐ0Ø!A‘ä!&§¡¨[ÄÇ
Á
ÐSYÔ!Z�ð 15×0PÑ0PÐQ_ÐalÓ0mÐð �;‰;×!Ò!Ð&;Ð&GØ=R×=WÑ=WÓ=YÑ:ÐÐ 7¸Ø$6Ð8OÐ#PÐ Ø%Ð-Ü).¯©Ð4HÐQWÔ)XÐ&Ø.2×.HÑ.HÐI_Ó.`Ñ+à.2Ð+ð ×&Ñ& y°$·+±+×2OÑ2OÓPˆ	àŸ?™?ØØ%Ø)Ø'Ø#9ð +ó 
Ðð Ÿ,™,ØØ2ØØ"7Ø#BØ+ØØ/Ø!5Ø#ð 'ó 
ˆð *¨!Ñ,ˆØ8<¿¹Ð8O˜Ÿ™ OÔ4ÐUYˆáØ# ]Ð3°oÀaÀbÐ6IÑIÐIä;Ø-Ø'Ø+×;Ñ;Ø)×7Ñ7Ø&×1Ñ1Ø,×=Ñ=ô
ð 	
r0   )T)NNNNNNNNNNNNN)r(   r)   r*   r+   r   rÆ  rD   rÝ  rà  rã  r   r,   rf   r   r-   r]   r   r   r   re   rg   rh   s   @r1   rÕ  rÕ  ?  sg  ø„ ñð )€Lõ
ò/ò0òCð -1Ø15Ø15Ø/3Ø,0Ø04Ø8<Ø9=Ø=AØ$(Ø,0Ø/3Ø&*ñ@
à˜EŸL™LÑ)ð@
ð ! §¡Ñ.ð@
ð ! §¡Ñ.ð	@
ð
 ˜uŸ|™|Ñ,ð@
ð ˜EŸL™LÑ)ð@
ð   §¡Ñ-ð@
ð  (¨¯©Ñ5ð@
ð !)¨¯©Ñ 6ð@
ð " $ u×'8Ñ'8Ñ"9Ñ:ð@
ð ˜D‘>ð@
ð $ D™>ð@
ð ' t™nð@
ð ˜d‘^ð@
ð 
ˆu�U—\‘\Ñ"Ð$PÐPÑ	Q÷@
r0   rÕ  zqThe bare BridgeTower Model transformer outputting BridgeTowerModelOutput object without any specific head on top.c            #       óè  ‡ — e Zd Zˆ fd„Zd„ Zd„ Z ee«       ee	e
¬«      	 	 	 	 	 	 	 	 	 	 	 	 	 	 ddeej                     deej                     deej                     deej                     d	eej                     d
eej                     deej                     deej                     dee   dee   dee   dee   deej                     dedeeej&                     e	f   fd„«       «       Zd„ Zˆ xZS )ÚBridgeTowerModelc           	      ó¶  •— t         ‰| �  |«       || _        |j                  }|j                  }|j
                  r_t        j                  |j                  |j                  «      | _	        t        j                  |j                  |j                  «      | _
        nÆt        j                  t        |j                  «      D �cg c],  }t        j                  |j                  |j                  «      ‘Œ. c}«      | _	        t        j                  t        |j                  «      D �cg c],  }t        j                  |j                  |j                  «      ‘Œ. c}«      | _
        t        j                  d|j                  «      | _        t!        |«      | _        t%        |«      | _        |j(                  s·|j*                  r«| j"                  j,                  j.                  D ]ˆ  }| j"                  j,                  j0                  j2                  j4                  |j2                  _        | j"                  j,                  j0                  j6                  j4                  |j6                  _        ŒŠ t        j                  t        |j                  «      D �cg c]  }t9        |«      ‘Œ c}«      | _        t        j                  t        |j                  «      D �cg c]  }t9        |«      ‘Œ c}«      | _        t?        |«      | _         t?        |«      | _!        t        jD                  |j                  |jF                  ¬«      | _$        t        jD                  |j                  |jF                  ¬«      | _%        |jL                  r!tO        |«      | _(        tO        |«      | _)        nŽt        j                  t        |j                  dz
  «      D �cg c]  }tO        |«      ‘Œ c}«      | _(        t        j                  t        |j                  dz
  «      D �cg c]  }tO        |«      ‘Œ c}«      | _)        | jU                  «        y c c}w c c}w c c}w c c}w c c}w c c}w )Nr€   r=   r   )+rC   rD   rR   Úvision_configÚtext_configÚ$share_cross_modal_transformer_layersr	   rL   rF   Úcross_modal_text_transformÚcross_modal_image_transformro   rp   rm   r‘   r�  r·  Úvision_modelrÕ  Ú
text_modelrÇ   Ú"init_layernorm_from_vision_encoderr¸  Úcross_modal_ln_separaterÆ   r£   r¾  r   rY  Úcross_modal_image_layersÚcross_modal_text_layersr  Úcross_modal_image_poolerÚcross_modal_text_poolerrH   rI   Úcross_modal_text_layernormÚcross_modal_image_layernormÚshare_link_tower_layersrÕ   Úcross_modal_text_link_towerÚcross_modal_image_link_towerrÙ  )rQ   rR   r÷  rø  rc   rÍ   rS   s         €r1   rD   zBridgeTowerModel.__init__õ  s9  ø€ Ü‰Ñ˜Ô ØˆŒØ×,Ñ,ˆØ×(Ñ(ˆà×6Ò6Ü.0¯i©i¸×8OÑ8OÐQW×QcÑQcÓ.dˆDÔ+Ü/1¯y©y¸×9RÑ9RÐTZ×TfÑTfÓ/gˆDÕ,ä.0¯m©mÜQVÐW]×WoÑWoÓQpÖqÈA”—‘˜;×2Ñ2°F×4FÑ4FÕGÒqó/ˆDÔ+ô 02¯}©}ÜSXÐY_×YqÑYqÓSrÖsÈa”—‘˜=×4Ñ4°f×6HÑ6HÕIÒsó0ˆDÔ,ô &(§\¡\°!°V×5GÑ5GÓ%HˆÔ"ä2°=ÓAˆÔä.¨{Ó;ˆŒà×,Ò,°×1ZÒ1ZØ×'Ñ'×.Ñ.×FÑFò J�Ø!%×!2Ñ!2×!9Ñ!9×!AÑ!A×!HÑ!H×!MÑ!M�—	‘	”Ø#×0Ñ0×7Ñ7×?Ñ?×DÑD×IÑI�—‘•ðJô )+¯©Ü=BÀ6×C[ÑC[Ó=\Ö]¸Ô& {Õ3Ò]ó)
ˆÔ%ô (*§}¡}Ü=BÀ6×C[ÑC[Ó=\Ö]¸Ô& {Õ3Ò]ó(
ˆÔ$ô
 ):¸&Ó(AˆÔ%Ü'8¸Ó'@ˆÔ$ô +-¯,©,°v×7IÑ7IÈv×OdÑOdÔ*eˆÔ'Ü+-¯<©<¸×8JÑ8JÐPV×PeÑPeÔ+fˆÔ(à×)Ò)Ü/CÀFÓ/KˆDÔ,Ü0DÀVÓ0LˆDÕ-ä/1¯}©}Ü7<¸V×=UÑ=UÐXYÑ=YÓ7ZÖ[°!Ô% fÕ-Ò[ó0ˆDÔ,ô 13·±Ü7<¸V×=UÑ=UÐXYÑ=YÓ7ZÖ[°!Ô% fÕ-Ò[ó1ˆDÔ-ð 	�‰ÕùòW rùò tùò ^ùò ^ùò  \ùò \s$   Ã1P=Ä$1QÊQËQÏQÐQc                 ó6   — | j                   j                  «       S r`   )rý  rÝ  rÎ  s    r1   rÝ  z%BridgeTowerModel.get_input_embeddings-  s   € Ø�‰×3Ñ3Ó5Ð5r0   c                 ó:   — | j                   j                  |«       y r`   )rý  rà  rß  s     r1   rà  z%BridgeTowerModel.set_input_embeddings0  s   € Ø�‰×,Ñ,¨UÕ3r0   ©Úoutput_typerÆ  r¢  rU   r—  r³   Ú
pixel_maskr"  r£  r7   Úimage_token_type_idxr&  ry  rz  Úlabelsr²   r™   c                 óª  — |
�|
n| j                   j                  }
|�|n| j                   j                  }|rdnd}|rdnd}|rdnd}|rdnd}|
rdnd}|�|€t        d«      ‚|�|n| j                   j                  }|	r|	nd}	|j                  «       }| j                  j                  |¬«      }|r||fz  }|€0t        j                  |t        j                  |j                  ¬«      }| j                  j                  ||«      j                  |j                  «      }t        | j                  j                  j                   «      | j                   j"                  z
  dz   }| j                  j                  j                   d| D ]  } |||«      d   }|sŒ||fz  }Œ |€K| j$                  j&                  j)                  |j+                  | j$                  j,                  «      |¬«      }n|j/                  ddd	«      }|r||fz  }| j$                  j&                  j0                  j2                  d| D ]  } ||«      }|sŒ||fz  }Œ | j$                  j&                  j5                  |j+                  | j$                  j,                  «      «      }| j7                  |«      }| j9                  t        j:                  dt        j                  |j                  ¬«      «      j=                  |«      }| j?                  ||z   «      }| jA                  |«      }| j9                  t        jB                  d
|	t        j                  |j                  ¬«      «      j=                  |«      }||z   }| jE                  |«      }t        j                  |j                  d«      |j                  d«      ft        j                  |j                  ¬«      }| j                  j                  ||j                  «       «      j                  |j                  «      } | jF                  d   |||||
¬«      } | d   }! | jH                  d   |||||
¬«      }"|"d   }#|r||!|#ffz  }|
r|| d   |"d   ffz  }d}$tK        |t        | j                  j                  j                   «      «      D �]r  }% | j                  j                  j                   |%   ||«      d   } | j$                  j&                  j0                  j2                  |%   |«      j+                  | j$                  j,                  «      }| jA                  | j$                  j&                  j5                  |«      «      |z   }| jL                  |$   }&| jN                  |$   }' |&| j7                  |«      |z   |!|«      }( |'||#|«      }) | jF                  |$dz      |(|)|||
¬«      } | d   }! | jH                  |$dz      |)|(|||
¬«      }"|"d   }#|$dz  }$|r||fz  }||fz  }||!|#ffz  }|
s�Œe|| d   |"d   ffz  }�Œu |!|#}+}*| jQ                  |*|+«      },|r|||f}|stS        d„ |*|+|,||fD «       «      S tU        |*|+|,||¬«      S )a1  
        output_hidden_states (`bool`, *optional*):
            If set to `True`, hidden states are returned as a list containing the hidden states of text, image, and
            cross-modal components respectively. i.e. `(hidden_states_text, hidden_states_image,
            hidden_states_cross_modal)` where each element is a list of the hidden states of the corresponding
            modality. `hidden_states_txt/img` are a list of tensors corresponding to unimodal hidden states and
            `hidden_states_cross_modal` is a list of tuples containing `cross_modal_text_hidden_states` and
            `cross_modal_image_hidden_states` of each brdige layer.
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels are currently not supported.
        Returns:

        Examples:

        ```python
        >>> from transformers import BridgeTowerProcessor, BridgeTowerModel
        >>> from PIL import Image
        >>> import requests

        >>> # prepare image and text
        >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
        >>> image = Image.open(requests.get(url, stream=True).raw)
        >>> text = "hello world"
        >>> processor = BridgeTowerProcessor.from_pretrained("BridgeTower/bridgetower-base")
        >>> model = BridgeTowerModel.from_pretrained("BridgeTower/bridgetower-base")

        >>> inputs = processor(image, text, return_tensors="pt")
        >>> outputs = model(**inputs)
        >>> outputs.keys()
        odict_keys(['text_features', 'image_features', 'pooler_output'])
        ```Nr/   zYBridgeTowerModel does not use `inputs_embeds`.  Make sure to pass in `input_ids` instead.r   )r¢  rW   r   rÏ   r€   ©r   )rU   r$  r&  c              3   ó$   K  — | ]  }|�|–— Œ
 y ­wr`   r/   r}  s     r1   r€  z+BridgeTowerModel.forward.<locals>.<genexpr>  s   è ø€ ò àØ�=ô ñùr�  )r#   r$   r%   r&   r'   )+rR   r&  ry  rà   ræ  r�   rý  r–   r,   rè  r*  rY   ré  r\   rP  r×  rd   rm   rü  r¸  rÐ   rÐ  rX   r¨   rÅ   rq   rÓ   rú  r�  rž  Ú	expand_asr  rû  Úfullr  r  r   rp   r  r  Úget_cls_featuresrŠ  r"   )-rQ   r¢  rU   r—  r³   r  r"  r£  r7   r  r&  ry  rz  r  r²   Úall_hidden_states_textÚall_hidden_states_imageÚall_hidden_states_crossr‹  rŒ  r¥  r6   Úextend_text_masksÚsplit_indexrd   rw   Úimage_embeds_with_lnÚcross_modal_textÚtext_token_type_embeddingsÚimage_token_type_embeddingsÚcross_modal_imageÚextend_image_masksÚlayer_outputs_textÚcross_text_featuresÚlayer_outputs_imageÚcross_image_featuresÚlink_layer_indexr�  Útext_link_towerÚimage_link_towerÚcross_text_features_Úcross_image_features_r#   r$   Úcls_featuress-                                                r1   re   zBridgeTowerModel.forward3  s  € ðd 2CÐ1NÑ-ÐTX×T_ÑT_×TqÑTqÐà$8Ð$DÑ È$Ï+É+×JjÑJjð 	ñ (<¡ÀÐÙ(<¡"À$ÐÙ(<¡"À$ÐÙ"6™B¸DÐÙ$5™b¸4ÐàÐ$¨Ð):Ü%Økóð ð &1Ð%<‘kÀ$Ç+Á+×B]ÑB]ˆÙ7KÑ3ÐQRÐØ—n‘nÓ&ˆØ—o‘o×0Ñ0¸9Ð0ÓEˆáØ" { nÑ4Ð"àÐ!Ü"ŸZ™Z¨¼5¿:¹:Èi×N^ÑN^Ô_ˆNØ ŸO™O×GÑGÈÐXcÓd×gÑgØ×Ñó
Ðô
 ˜$Ÿ/™/×1Ñ1×7Ñ7Ó8¸4¿;¹;×;XÑ;XÑXÐ[\Ñ\ˆð —_‘_×,Ñ,×2Ñ2°<°KÐ@ò 	9ˆEÙ Ð->Ó?ÀÑBˆKâ#Ø&¨;¨.Ñ8Ñ&ð		9ð ÐØ×,Ñ,×3Ñ3×?Ñ?Ø×!Ñ! $×"3Ñ"3×"9Ñ"9Ó:ÐUmð @ó ‰Lð
 (×/Ñ/°°1°aÓ8ˆLáØ#¨ Ñ6Ð#ð ×&Ñ&×-Ñ-×9Ñ9×CÑCÀLÀ[ÐQò 	;ˆEÙ  Ó.ˆLÚ#Ø'¨L¨?Ñ:Ñ'ð	;ð
  $×0Ñ0×7Ñ7×DÑDÀ\×EVÑEVÐW[×WhÑWh×WnÑWnÓEoÓpÐð  ×:Ñ:¸;ÓGÐà%)×%?Ñ%?Ü�K‰K˜¤§¡°I×4DÑ4DÔEó&
ç
‰)Ð$Ó
%ð 	#ð  ×:Ñ:Ð;KÐNhÑ;hÓiÐà#×?Ñ?Ð@TÓUÐØ&*×&@Ñ&@Ü�J‰J�tÐ1¼¿¹ÈI×L\ÑL\Ô]ó'
ç
‰)Ð(Ó
)ð 	$ð  4Ð6QÑQÐØ ×<Ñ<Ð=QÓRÐä—Z‘ZØ×#Ñ# AÓ&Ð(9×(>Ñ(>¸qÓ(AÐBÜ—*‘*Ø×#Ñ#ô
ˆ
ð
 "Ÿ_™_×HÑHÈÐU_×UdÑUdÓUfÓg×jÑjØ×Ñó
Ðð =˜T×9Ñ9¸!Ñ<ØØØ,Ø#5Ø/ô
Ðð 1°Ñ3Ðà>˜d×;Ñ;¸AÑ>ØØØ-Ø#4Ø/ô
Ðð  3°1Ñ5ÐáØ#Ð)<Ð>RÐ(SÐ'UÑUÐ#áØÐ%7¸Ñ%:Ð<OÐPQÑ<RÐ$SÐ#UÑUÐàÐô �{¤C¨¯©×(?Ñ(?×(EÑ(EÓ$FÓGó 0	ZˆAØ:˜$Ÿ/™/×1Ñ1×7Ñ7¸Ñ:¸;ÐHYÓZÐ[\Ñ]ˆKØL˜4×,Ñ,×3Ñ3×?Ñ?×IÑIÈ!ÑLÈ\ÓZ×_Ñ_Ø×!Ñ!×'Ñ'óˆLð ×0Ñ0°×1BÑ1B×1IÑ1I×1VÑ1VÐWcÓ1dÓeØ-ñ.ð !ð
 #×>Ñ>Ð?OÑPˆOØ#×@Ñ@ÐAQÑRÐñ $3Ø×/Ñ/°Ó<Ð?YÑYØ#Ø!ó$Ð ñ
 %5Ð5IÐK_ÐasÓ$tÐ!ð "T ×!=Ñ!=Ð>NÐQRÑ>RÑ!SØ$Ø%Ø0Ø'9Ø"3ô"Ðð #5°QÑ"7Ðà"U $×"?Ñ"?Ð@PÐSTÑ@TÑ"UØ%Ø$Ø1Ø'8Ø"3ô#Ðð $7°qÑ#9Ð à Ñ!Ðá#Ø&¨;¨.Ñ8Ð&Ø'¨L¨?Ñ:Ð'Ø'Ð-@ÐBVÐ,WÐ+YÑYÐ'ã Ø#Ð);¸AÑ)>Ð@SÐTUÑ@VÐ(WÐ'YÑYÒ#ða0	Zðf )<Ð=Q�~ˆØ×,Ñ,¨]¸NÓKˆáØ!7Ð9PÐRiÐ jÐáÜñ à'¨¸ÐGXÐZmÐnôó ð ô &Ø'Ø)Ø&Ø+Ø*ô
ð 	
r0   c                 óx   — | j                  |«      }| j                  |«      }t        j                  ||gd¬«      S )Nrƒ   r    )r  r  r,   r¬   )rQ   r#   r$   Úcls_features_textÚcls_features_images        r1   r  z!BridgeTowerModel.get_cls_features  s<   € Ø ×8Ñ8¸ÓGÐØ!×:Ñ:¸>ÓJÐÜ�y‰yÐ+Ð-?Ð@ÀbÔIÐIr0   )NNNNNNNNNNNNNF)r(   r)   r*   rD   rÝ  rà  r   ÚBRIDGETOWER_INPUTS_DOCSTRINGr   r"   Ú_CONFIG_FOR_DOCr   r,   Ú
LongTensorr-   r¿   r]   r   r   rf   re   r  rg   rh   s   @r1   rõ  rõ  ï  sŸ  ø„ ô6òp6ò4ñ +Ð+GÓHÙÐ+AÐP_Ô`ð 15Ø6:Ø59Ø48Ø15Ø15Ø59Ø48Ø.2Ø,0Ø/3Ø&*Ø-1Ø).ñf
à˜E×,Ñ,Ñ-ðf
ð ! ×!2Ñ!2Ñ3ðf
ð ! ×!1Ñ!1Ñ2ð	f
ð
 ˜u×0Ñ0Ñ1ðf
ð ˜U×-Ñ-Ñ.ðf
ð ˜E×-Ñ-Ñ.ðf
ð   × 1Ñ 1Ñ2ðf
ð ˜u×0Ñ0Ñ1ðf
ð ' s™mðf
ð $ D™>ðf
ð ' t™nðf
ð ˜d‘^ðf
ð ˜×)Ñ)Ñ*ðf
ð #'ðf
ð  
ˆu�U—\‘\Ñ"Ð$:Ð:Ñ	;ò!f
ó aó Iðf
öPJr0   rõ  c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )Ú"BridgeTowerPredictionHeadTransformc                 óh  •— t         ‰| �  «        t        j                  |j                  |j                  «      | _        t        |j                  t        «      rt        |j                     | _
        n|j                  | _
        t        j                  |j                  |j                  ¬«      | _        y rÃ   )rC   rD   r	   rL   rF   rç   rö   r÷   rø   r   Útransform_act_fnrH   rI   rP   s     €r1   rD   z+BridgeTowerPredictionHeadTransform.__init__%  s{   ø€ Ü‰ÑÔÜ—Y‘Y˜v×1Ñ1°6×3EÑ3EÓFˆŒ
Ü�f×'Ñ'¬Ô-Ü$*¨6×+<Ñ+<Ñ$=ˆDÕ!à$*×$5Ñ$5ˆDÔ!ÜŸ™ f×&8Ñ&8¸f×>SÑ>SÔTˆ�r0   c                 ól   — | j                  |«      }| j                  |«      }| j                  |«      }|S r`   )rç   r4  rH   rû   s     r1   re   z*BridgeTowerPredictionHeadTransform.forward.  s4   € ØŸ
™
 =Ó1ˆØ×-Ñ-¨mÓ<ˆØŸ™ }Ó5ˆØÐr0   rã   rh   s   @r1   r2  r2  $  s   ø„ ôUör0   r2  c                   ó&   ‡ — e Zd Zdˆ fd„	Zd„ Zˆ xZS )ÚBridgeTowerMLMHeadc                 óp  •— t         ‰| �  «        || _        t        |«      | _        t        j                  |j                  |j                  j                  d¬«      | _
        t        j                  t        j                  |j                  j                  «      «      | _        |�|| j                  _        y y )NF)r   )rC   rD   rR   r2  Ú	transformr	   rL   rF   rø  r˜  Údecoderr‰   r,   rž  r   r£   )rQ   rR   r£   rS   s      €r1   rD   zBridgeTowerMLMHead.__init__6  s„   ø€ Ü‰ÑÔØˆŒÜ;¸FÓCˆŒÜ—y‘y ×!3Ñ!3°V×5GÑ5G×5RÑ5RÐY^Ô_ˆŒÜ—L‘L¤§¡¨V×-?Ñ-?×-JÑ-JÓ!KÓLˆŒ	ØÐØ"(ˆD�L‰LÕð r0   c                 ód   — | j                  |«      }| j                  |«      | j                  z   }|S r`   )r9  r:  r   )rQ   r  Ú	mlm_scores      r1   re   zBridgeTowerMLMHead.forward?  s-   € Ø—N‘N 1Ó%ˆ	Ø—L‘L Ó+¨d¯i©iÑ7ˆ	ØÐr0   r`   rã   rh   s   @r1   r7  r7  5  s   ø„ õ)ör0   r7  c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚBridgeTowerITMHeadc                 óX   •— t         ‰| �  «        t        j                  |d«      | _        y ©Nr€   ©rC   rD   r	   rL   Úfc)rQ   rF   rS   s     €r1   rD   zBridgeTowerITMHead.__init__F  s    ø€ Ü‰ÑÔÜ—)‘)˜K¨Ó+ˆ�r0   c                 ó(   — | j                  |«      }|S r`   ©rB  )rQ   r  Ú	itm_scores      r1   re   zBridgeTowerITMHead.forwardJ  s   € Ø—G‘G˜A“Jˆ	ØÐr0   rã   rh   s   @r1   r>  r>  E  s   ø„ ô,ör0   r>  z\
    BridgeTower Model with a language modeling head on top as done during pretraining.
    c                    óô  ‡ — e Zd ZdgZˆ fd„Zd„ Zd„ Z eej                  d«      «       e
ee¬«      	 	 	 	 	 	 	 	 	 	 	 	 ddeej                     deej                      d	eej                     d
eej                      deej                     deej                      deej                      deej                      dee   dee   dee   deej                     deeeej                      f   fd„«       «       Zˆ xZS )ÚBridgeTowerForMaskedLMzmlm_score.decoder.weightc                 ó„   •— t         ‰| �  |«       t        |«      | _        t	        |«      | _        | j                  «        y r`   )rC   rD   rõ  r³  r7  r<  rÙ  rP   s     €r1   rD   zBridgeTowerForMaskedLM.__init__X  s5   ø€ Ü‰Ñ˜Ô ä+¨FÓ3ˆÔÜ+¨FÓ3ˆŒð 	�‰Õr0   c                 ó.   — | j                   j                  S r`   ©r<  r:  rÎ  s    r1   Úget_output_embeddingsz,BridgeTowerForMaskedLM.get_output_embeddingsa  s   € Ø�~‰~×%Ñ%Ð%r0   c                 ó&   — || j                   _        y r`   rJ  )rQ   Únew_embeddingss     r1   Úset_output_embeddingsz,BridgeTowerForMaskedLM.set_output_embeddingsd  s   € Ø!/ˆ�‰Õr0   zbatch_size, sequence_lengthr  r¢  rU   r—  r³   r  r"  r£  r7   r&  ry  rz  r  r™   c                 ó  — |�|n| j                   j                  }| j                  |||||||||	|
|¬«      }| j                  |r|j                  n|d   «      }d}|�kt        «       }|j                  |j                  «      } ||j                  d| j                   j                  j                  «      |j                  d«      «      }|st        |«      }|�|f|z   S |S t        |||j                  |j                  ¬«      S )aQ  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
            config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
            loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
        Returns:

        Examples:

        ```python
        >>> from transformers import BridgeTowerProcessor, BridgeTowerForMaskedLM
        >>> from PIL import Image
        >>> import requests

        >>> url = "http://images.cocodataset.org/val2017/000000360943.jpg"
        >>> image = Image.open(requests.get(url, stream=True).raw).convert("RGB")
        >>> text = "a <mask> looking out of the window"

        >>> processor = BridgeTowerProcessor.from_pretrained("BridgeTower/bridgetower-base-itm-mlm")
        >>> model = BridgeTowerForMaskedLM.from_pretrained("BridgeTower/bridgetower-base-itm-mlm")

        >>> # prepare inputs
        >>> encoding = processor(image, text, return_tensors="pt")

        >>> # forward pass
        >>> outputs = model(**encoding)

        >>> results = processor.decode(outputs.logits.argmax(dim=-1).squeeze(0).tolist())

        >>> print(results)
        .a cat looking out of the window.
        ```N©
rU   r—  r³   r  r"  r£  r7   r&  ry  rz  r   rƒ   ©r4   r5   r&   r'   )rR   ræ  r³  r<  r#   r
   r\   rY   r«   rø  r˜  rŠ  r   r&   r'   )rQ   r¢  rU   r—  r³   r  r"  r£  r7   r&  ry  rz  r  rC  Ú
mlm_logitsÚmasked_lm_lossÚloss_fctrL  s                     r1   re   zBridgeTowerForMaskedLM.forwardg  s  € ðb &1Ð%<‘kÀ$Ç+Á+×B]ÑB]ˆØ×"Ñ"ØØ)Ø)Ø%Ø!ØØ'Ø%Ø/Ø!5Ø#ð #ó 
ˆð —^‘^¹[ G×$9Ò$9ÈgÐVWÉjÓYˆ
ØˆØÐÜ'Ó)ˆHà—Y‘Y˜z×0Ñ0Ó1ˆFÙ% j§o¡o°b¸$¿+¹+×:QÑ:Q×:\Ñ:\Ó&]Ð_e×_jÑ_jÐkmÓ_nÓoˆNáÜ˜:Ó&ˆFØ3AÐ3M�^Ð%¨Ñ.ÐYÐSYÐYäØØØ!×/Ñ/Ø×)Ñ)ô	
ð 	
r0   ©NNNNNNNNNNNN)r(   r)   r*   Ú_tied_weights_keysrD   rK  rN  r   r.  Úformatr   r   r/  r   r,   r0  r-   r]   r   r   re   rg   rh   s   @r1   rG  rG  O  s�  ø„ ð 5Ð5Ðôò&ò0ñ +Ð+G×+NÑ+NÐOlÓ+mÓnÙ¨>ÈÔXð 15Ø6:Ø59Ø48Ø15Ø15Ø59Ø48Ø,0Ø/3Ø&*Ø-1ñO
à˜E×,Ñ,Ñ-ðO
ð ! ×!2Ñ!2Ñ3ðO
ð ! ×!1Ñ!1Ñ2ð	O
ð
 ˜u×0Ñ0Ñ1ðO
ð ˜U×-Ñ-Ñ.ðO
ð ˜E×-Ñ-Ñ.ðO
ð   × 1Ñ 1Ñ2ðO
ð ˜u×0Ñ0Ñ1ðO
ð $ D™>ðO
ð ' t™nðO
ð ˜d‘^ðO
ð ˜×)Ñ)Ñ*ðO
ð 
ˆ~˜u U×%6Ñ%6Ñ7Ð7Ñ	8òO
ó Yó oôO
r0   rG  zª
    BridgeTower Model transformer with a classifier head on top (a linear layer on top of the final hidden state of the
    [CLS] token) for image-to-text matching.
    c                    óÄ  ‡ — e Zd Zˆ fd„Z ee«       eee¬«      	 	 	 	 	 	 	 	 	 	 	 	 dde	e
j                     de	e
j                     de	e
j                     de	e
j                     de	e
j                     de	e
j                     d	e	e
j                     d
e	e
j                     de	e   de	e   de	e   de	e
j                     deeee
j                     f   fd„«       «       Zˆ xZS )Ú#BridgeTowerForImageAndTextRetrievalc                 óž   •— t         ‰| �  |«       t        |«      | _        t	        |j
                  dz  «      | _        | j                  «        y r@  )rC   rD   rõ  r³  r>  rF   rE  rÙ  rP   s     €r1   rD   z,BridgeTowerForImageAndTextRetrieval.__init__Ã  s@   ø€ Ü‰Ñ˜Ô ä+¨FÓ3ˆÔä+¨F×,>Ñ,>ÀÑ,BÓCˆŒð 	�‰Õr0   r  r¢  rU   r—  r³   r  r"  r£  r7   r&  ry  rz  r  r™   c                 óŽ  — |�|n| j                   j                  }| j                  |||||||||	|
|¬«      }|r|j                  n|d   }| j	                  |«      }d}|�.t        «       }|j                  |j                  «      } |||«      }|st        |«      }|�|f|z   S |S t        |||j                  |j                  ¬«      S )aÏ  
        labels (`torch.LongTensor` of shape `(batch_size, 1)`, *optional*):
            Labels for computing the image-text matching loss. 0 means the pairs don't match and 1 means they match.
            The pairs with 0 will be skipped for calculation.
        Returns:

        Examples:

        ```python
        >>> from transformers import BridgeTowerProcessor, BridgeTowerForImageAndTextRetrieval
        >>> import requests
        >>> from PIL import Image

        >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
        >>> image = Image.open(requests.get(url, stream=True).raw)
        >>> texts = ["An image of two cats chilling on a couch", "A football player scoring a goal"]

        >>> processor = BridgeTowerProcessor.from_pretrained("BridgeTower/bridgetower-base-itm-mlm")
        >>> model = BridgeTowerForImageAndTextRetrieval.from_pretrained("BridgeTower/bridgetower-base-itm-mlm")

        >>> # forward pass
        >>> scores = dict()
        >>> for text in texts:
        ...     # prepare inputs
        ...     encoding = processor(image, text, return_tensors="pt")
        ...     outputs = model(**encoding)
        ...     scores[text] = outputs.logits[0, 1].item()
        ```NrP  r€   rQ  )rR   ræ  r³  r%   rE  r
   r\   rY   rŠ  r   r&   r'   )rQ   r¢  rU   r—  r³   r  r"  r£  r7   r&  ry  rz  r  rC  r%   r5   Úitm_lossrT  rL  s                      r1   re   z+BridgeTowerForImageAndTextRetrieval.forwardÍ  sí   € ðZ &1Ð%<‘kÀ$Ç+Á+×B]ÑB]ˆà×"Ñ"ØØ)Ø)Ø%Ø!ØØ'Ø%Ø/Ø!5Ø#ð #ó 
ˆñ 2=˜×-Ò-À'È!Á*ˆà—‘ Ó.ˆàˆØÐÜ'Ó)ˆHà—Y‘Y˜vŸ}™}Ó-ˆFÙ ¨Ó/ˆHáÜ˜6“]ˆFØ-5Ð-A�X�K &Ñ(ÐMÀvÐMä'ØØØ!×/Ñ/Ø×)Ñ)ô	
ð 	
r0   rU  )r(   r)   r*   rD   r   r.  r   r   r/  r   r,   r0  r-   r]   r   r   re   rg   rh   s   @r1   rY  rY  »  so  ø„ ôñ +Ð+GÓHÙÐ+CÐRaÔbð 15Ø6:Ø59Ø48Ø15Ø15Ø59Ø48Ø,0Ø/3Ø&*Ø-1ñO
à˜E×,Ñ,Ñ-ðO
ð ! ×!2Ñ!2Ñ3ðO
ð ! ×!1Ñ!1Ñ2ð	O
ð
 ˜u×0Ñ0Ñ1ðO
ð ˜U×-Ñ-Ñ.ðO
ð ˜E×-Ñ-Ñ.ðO
ð   × 1Ñ 1Ñ2ðO
ð ˜u×0Ñ0Ñ1ðO
ð $ D™>ðO
ð ' t™nðO
ð ˜d‘^ðO
ð ˜×)Ñ)Ñ*ðO
ð 
Ð'¨¨u×/@Ñ/@Ñ)AÐAÑ	BòO
ó có IôO
r0   rY  c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚBridgeTowerContrastiveHeadc                 óX   •— t         ‰| �  «        t        j                  ||«      | _        y r`   rA  )rQ   rF   Ú
embed_sizerS   s      €r1   rD   z#BridgeTowerContrastiveHead.__init__"  s    ø€ Ü‰ÑÔÜ—)‘)˜K¨Ó4ˆ�r0   c                 ó(   — | j                  |«      }|S r`   rD  )rQ   r  s     r1   re   z"BridgeTowerContrastiveHead.forward&  s   € Ø�G‰G�A‹JˆØˆr0   rã   rh   s   @r1   r^  r^  !  s   ø„ ô5ör0   r^  zl
    BridgeTower Model with a image-text contrastive head on top computing image-text contrastive loss.
    c                    ó°  ‡ — e Zd Zˆ fd„Z ee«       eee¬«      	 	 	 	 	 	 	 	 	 	 	 	 dde	e
j                     de	e
j                     de	e
j                     de	e
j                     de	e
j                     de	e
j                     d	e	e
j                     d
e	e
j                     de	e   de	e   de	e   de	e   deeee
j                     f   fd„«       «       Zˆ xZS )Ú!BridgeTowerForContrastiveLearningc                 óÊ  •— t         ‰| �  |«       t        |«      | _        t	        |j
                  |j                  «      | _        t	        |j
                  |j                  «      | _        t	        |j
                  dz  |j                  «      | _	        t        j                  t        j                  | j                  j                  «      «      | _        | j#                  «        y r@  )rC   rD   rõ  r³  r^  rF   Úcontrastive_hidden_sizeÚitc_text_headÚitc_image_headÚitc_cross_modal_headr	   r‰   r,   rÝ   rR   Úlogit_scale_init_valueÚlogit_scalerÙ  rP   s     €r1   rD   z*BridgeTowerForContrastiveLearning.__init__2  s£   ø€ Ü‰Ñ˜Ô ä+¨FÓ3ˆÔä7¸×8JÑ8JÈF×LjÑLjÓkˆÔÜ8¸×9KÑ9KÈV×MkÑMkÓlˆÔÜ$>¸v×?QÑ?QÐTUÑ?UÐW]×WuÑWuÓ$vˆÔ!äŸ<™<¬¯©°T·[±[×5WÑ5WÓ(XÓYˆÔà�‰Õr0   r  r¢  rU   r—  r³   r  r"  r£  r7   r&  ry  rz  Úreturn_lossr™   c                 óH  — |�|n| j                   j                  }| j                  |||||||||	d|¬«      }|r|j                  n|d   }|r|j                  n|d   \  }}}|d   }|d   }| j                  j
                  j                  j                  |«      }| j                  j                  t        j                  ddt        j                  | j                  j                  j                  j                  ¬	«      «      j                  |«      }| j                  j                  |«      |z   }t         j"                  j%                  | j'                  |dd…d
dd…f   «      dd¬«      }t         j"                  j%                  | j)                  |dd…d
dd…f   «      dd¬«      j+                  |j                  ¬«      }t         j"                  j%                  | j-                  |«      dd¬«      j+                  |j                  ¬«      }t        j.                  |||gd¬«      }| j0                  j3                  «       j+                  |j                  ¬«      }t        j4                  ||j7                  «       «      |z  }t        j4                  ||j7                  «       «      |z  }t        j4                  ||j7                  «       «      |z  }d}|r•t        j8                  t;        |«      |j                  ¬«      }t         j"                  j=                  ||«      }t         j"                  j=                  ||«      }t         j"                  j=                  ||«      }||z   |z   dz  }|s||||f|dd z   } |�|f| z   S | S t?        ||||||j                  |j@                  ¬«      S )aˆ  
        return_loss (`bool`, *optional*):
            Whether or not to return the contrastive loss.
        Returns:

        Examples:

        ```python
        >>> from transformers import BridgeTowerProcessor, BridgeTowerForContrastiveLearning
        >>> import requests
        >>> from PIL import Image
        >>> import torch

        >>> image_urls = [
        ...     "https://farm4.staticflickr.com/3395/3428278415_81c3e27f15_z.jpg",
        ...     "http://images.cocodataset.org/val2017/000000039769.jpg",
        ... ]
        >>> texts = ["two dogs in a car", "two cats sleeping on a couch"]
        >>> images = [Image.open(requests.get(url, stream=True).raw) for url in image_urls]

        >>> processor = BridgeTowerProcessor.from_pretrained("BridgeTower/bridgetower-large-itm-mlm-itc")
        >>> model = BridgeTowerForContrastiveLearning.from_pretrained("BridgeTower/bridgetower-large-itm-mlm-itc")

        >>> inputs = processor(images, texts, padding=True, return_tensors="pt")
        >>> loss = model(**inputs, return_loss=True).loss

        >>> inputs = processor(images, texts[::-1], padding=True, return_tensors="pt")
        >>> loss_swapped = model(**inputs, return_loss=True).loss

        >>> print("Loss", round(loss.item(), 4))
        Loss 0.0019

        >>> print("Loss with swapped images", round(loss_swapped.item(), 4))
        Loss with swapped images 2.126
        ```NTrP  r€   r   rƒ   r  r   rW   r   )r¡   Úprå  r(  r    g      @)r4   r5   r6   r7   r8   r&   r'   )!rR   ræ  r³  r%   r&   rü  r¸  rÓ   r�  r,   r  r*  r£   rY   r  rû  r	   r©   Ú	normalizerf  rg  r\   rh  rÊ   rj  Úexpr)  Útr”   rP  Úcross_entropyr3   r'   )!rQ   r¢  rU   r—  r³   r  r"  r£  r7   r&  ry  rz  rk  rC  r%   Úhidden_states_txtÚhidden_states_imgÚhidden_states_cross_modalr6   r  r  r8   r5   rj  Úlogits_text_to_imageÚlogits_text_to_crossÚlogits_image_to_crossÚitc_lossr  Útext_to_image_lossÚtext_to_cross_lossÚimage_to_cross_lossrL  s!                                    r1   re   z)BridgeTowerForContrastiveLearning.forward?  sz  € ðh &1Ð%<‘kÀ$Ç+Á+×B]ÑB]ˆà×"Ñ"ØØ)Ø)Ø%Ø!ØØ'Ø%Ø/Ø!%Ø#ð #ó 
ˆñ 2=˜×-Ò-À'È!Á*ˆá%0ˆG×!Ò!°g¸a±jñ 	HÐÐ,Ð.Gð (¨Ñ+ˆØ(¨Ñ,ˆà#×/Ñ/×<Ñ<×CÑC×PÑPÐQ]Ó^ÐØ&*×&6Ñ&6×&LÑ&LÜ�J‰J�t˜Q¤e§j¡j¸×9IÑ9I×9_Ñ9_×9fÑ9f×9mÑ9mÔnó'
ç
‰)Ð(Ó
)ð 	$ð ×'Ñ'×CÑCÐDXÓYÐ\wÑwˆô —m‘m×-Ñ-¨d×.@Ñ.@ÀÊQÐPQÒSTÈWÑAUÓ.VÐ\^ÐbcÐ-ÓdˆÜ—}‘}×.Ñ.¨t×/BÑ/BÀ<ÒPQÐSTÒVWÐPWÑCXÓ/YÐ_aÐefÐ.Óg×jÑjØ×%Ñ%ð kó 
ˆô —}‘}×.Ñ.¨t×/HÑ/HÈÓ/WÐ]_ÐcdÐ.Óe×hÑhØ×%Ñ%ð ió 
ˆô —‘˜k¨<¸ÐFÈBÔOˆà×&Ñ&×*Ñ*Ó,×/Ñ/°{×7IÑ7IÐ/ÓJˆÜ$Ÿ|™|¨K¸¿¹Ó9IÓJÈ[ÑXÐÜ$Ÿ|™|¨K¸¿¹Ó9IÓJÈ[ÑXÐÜ %§¡¨\¸<¿>¹>Ó;KÓ LÈ{Ñ ZÐàˆáÜ—\‘\¤# f£+°f·m±mÔDˆFÜ!#§¡×!<Ñ!<Ð=QÐSYÓ!ZÐÜ!#§¡×!<Ñ!<Ð=QÐSYÓ!ZÐÜ"$§-¡-×"=Ñ"=Ð>SÐU[Ó"\ÐØ*Ð-?Ñ?ÐBUÑUÐY\Ñ\ˆHáØ˜k¨<¸ÐFÈÐQRÐQSÈÑTˆFØ-5Ð-A�X�K &Ñ(ÐMÀvÐMä+ØØØ#Ø%Ø%Ø!×/Ñ/Ø×)Ñ)ô
ð 	
r0   )NNNNNNNNNTNN)r(   r)   r*   rD   r   r.  r   r3   r/  r   r,   r0  r-   r]   r   r   re   rg   rh   s   @r1   rc  rc  +  sh  ø„ ôñ +Ð+GÓHÙÐ+GÐVeÔfð 15Ø6:Ø59Ø48Ø15Ø15Ø59Ø48Ø,0Ø/3Ø&*Ø&*ñv
à˜E×,Ñ,Ñ-ðv
ð ! ×!2Ñ!2Ñ3ðv
ð ! ×!1Ñ!1Ñ2ð	v
ð
 ˜u×0Ñ0Ñ1ðv
ð ˜U×-Ñ-Ñ.ðv
ð ˜E×-Ñ-Ñ.ðv
ð   × 1Ñ 1Ñ2ðv
ð ˜u×0Ñ0Ñ1ðv
ð $ D™>ðv
ð ' t™nðv
ð ˜d‘^ðv
ð ˜d‘^ðv
ð 
Ð+¨U°5×3DÑ3DÑ-EÐEÑ	Fòv
ó gó Iôv
r0   rc  )rc  rY  rG  rõ  r²  )r   )Qr+   r,  Úcollectionsr   Údataclassesr   Útypingr   r   r   r   r,   Útorch.utils.checkpointr	   Útorch.nnr
   Úactivationsr   r   Úmodeling_outputsr   r   r   r   r   Úmodeling_utilsr   r   Úpytorch_utilsr   r   Úutilsr   r   r   r   r   Úconfiguration_bridgetowerr   r   r   Ú
get_loggerr(   r…  r/  Ú_CHECKPOINT_FOR_DOCÚ_TOKENIZER_FOR_DOCÚBRIDGETOWER_START_DOCSTRINGr.  r"   r3   ÚModuler:   rj   ry   rÁ   rÕ   rå   ró   rý   r  r	  rJ  rG  rY  rk  rt  r”  r   r²  r·  rÕ  rõ  r2  r7  r>  rG  rY  r^  rc  Ú__all__r/   r0   r1   ú<module>r�     s  ðñ  ã Ý #Ý !ß /Ó /ã Û Ý Ý %ç 6÷õ ÷ Iß Q÷õ ÷ iÑ hð 
ˆ×	Ñ	˜HÓ	%€à%€Ø4Ð Ø'Ð ð	Ð ð; Ð ð| ô:˜[ó :ó ð:ð> ô: ;ó :ó ð:ôB) 2§9¡9ô )ôX˜RŸY™Yô ô6P "§)¡)ô Pôf7" 2§9¡9ô 7"ôtd˜2Ÿ9™9ô dô4˜BŸI™Iô ô˜bŸi™iô ô ˜Ÿ	™	ô ô˜Ÿ	™	ô ô C˜rŸy™yô CðN Ð%ð'Ð #ô0˜2Ÿ9™9ô 0ôf; §	¡	ô ;ô|S˜2Ÿ9™9ô SônZ
˜RŸY™Yô Z
ô|V= §	¡	ô V=ót4ô $% ô $%ôNYÐ7ô Yôm
Ð5ô m
ñ` ðàóô
lJÐ1ó lJóð
lJô`	¨¯©ô ô"˜Ÿ™ô ô ˜Ÿ™ô ñ ðð  ó	ôc
Ð7ó c
óðc
ñL ðð  óô\
Ð*Dó \
óð\
ô~ §¡ô ñ ðð  ó	ôF
Ð(Bó F
óðF
òR�r0   