Ë
    T^(h½9 ã                  óè  — d Z ddlmZ ddlmZ ddlmZmZmZm	Z	 ddl
ZddlmZ ddlmZ dd	lmZ dd
lmZmZmZmZmZ ddlmZmZ ddlmZmZmZmZ ddl m!Z! ddl"m#Z# ddl$m%Z%  ejL                  e'«      Z(dZ)e G d„ de«      «       Z*e G d„ de«      «       Z+	 	 	 	 d:d„Z,d„ Z-d;d„Z.g fd„Z/ G d„ dej`                  jb                  jd                  «      Z3 G d„ dej`                  jb                  jh                  «      Z5d<d„Z6d;d„Z7 G d „ d!ej`                  jb                  jh                  «      Z8 G d"„ d#ej`                  jb                  jh                  «      Z9d$„ Z:d%„ Z; G d&„ d'ej`                  jb                  jh                  «      Z< G d(„ d)ej`                  jb                  jh                  «      Z= G d*„ d+ej`                  jb                  jh                  «      Z> G d,„ d-ej`                  jb                  jh                  «      Z?d.Z@ ed/e@«       G d0„ d1e«      «       ZAd2ZB ed/e@«      e G d3„ d4ej`                  jb                  jh                  «      «       «       ZC G d5„ d6eA«      ZD G d7„ d8ee«      ZEg d9¢ZFy)=zTF 2.0 Idefics model.é    )Úannotations)Ú	dataclass)ÚListÚOptionalÚTupleÚUnionNé   )ÚTFPreTrainedModel)Úget_tf_activation)ÚModelOutput)ÚTFCausalLanguageModelingLossÚTFModelInputTypeÚkeras_serializableÚ
shape_listÚunpack_inputs)Úinvert_attention_maskÚscaled_dot_product_attention)Úadd_start_docstringsÚ%add_start_docstrings_to_model_forwardÚloggingÚreplace_return_docstringsé   )ÚIdeficsConfig)ÚTFIdeficsPerceiverResampler)ÚTFIdeficsVisionTransformerr   c                  óX   — e Zd ZU dZdZded<   dZded<   dZded<   dZded	<   dZ	ded
<   y)Ú TFIdeficsBaseModelOutputWithPastaž	  
    Base class for Idefics model's outputs that may also contain a past key/values (to speed up sequential decoding).

    Args:
        last_hidden_state (`tf.Tensor` of shape `(batch_size, sequence_length, hidden_size)`):
            Sequence of hidden-states at the output of the last layer of the model.

            If `past_key_values` is used only the last hidden-state of the sequences of shape `(batch_size, 1,
            hidden_size)` is output.
        past_key_values (`tuple(tuple(tf.Tensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
            Tuple of `tuple(tf.Tensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape
            `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and optionally if
            `config.is_encoder_decoder=True` 2 additional tensors of shape `(batch_size, num_heads,
            encoder_sequence_length, embed_size_per_head)`.

            Contains pre-computed hidden-states (key and values in the self-attention blocks and optionally if
            `config.is_encoder_decoder=True` in the cross-attention blocks) that can be used (see `past_key_values`
            input) to speed up sequential decoding.
        hidden_states (`tuple(tf.Tensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
            Tuple of `tf.Tensor` (one for the output of the embeddings, if the model has an embedding layer, +
            one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`.

            Hidden-states of the model at the output of each layer plus the optional initial embedding outputs.
        attentions (`tuple(tf.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
            Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
            sequence_length)`.

            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
            heads.
        image_hidden_states (`tuple(tf.Tensor)`, *optional*):
            Tuple of `tf.Tensor` (one for the output of the image embeddings, `(batch_size, num_images,
            sequence_length, hidden_size)`.

            image_hidden_states of the model produced by the vision encoder, and optionally by the perceiver
    NúOptional[tf.Tensor]Úlast_hidden_statez!Optional[Tuple[Tuple[tf.Tensor]]]Úpast_key_valuesúOptional[Tuple[tf.Tensor]]Úhidden_statesÚ
attentionsÚimage_hidden_states)
Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   Ú__annotations__r    r"   r#   r$   © ó    úm/var/www/skyplay_api_hub/venv/lib/python3.12/site-packages/transformers/models/idefics/modeling_tf_idefics.pyr   r   8   sC   … ñ"ðH .2ÐÐ*Ó1Ø9=€OÐ6Ó=Ø04€MÐ-Ó4Ø-1€JÐ*Ó1Ø6:ÐÐ3Ô:r+   r   c                  óf   — e Zd ZU dZdZded<   dZded<   dZded<   dZded	<   dZ	ded
<   dZ
ded<   y)ÚTFIdeficsCausalLMOutputWithPasta„  
    Base class for Idefics causal language model (or autoregressive) outputs.

    Args:
        loss (`tf.Tensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
            Language modeling loss (for next-token prediction).
        logits (`tf.Tensor` of shape `(batch_size, sequence_length, config.vocab_size)`):
            Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
        past_key_values (`tuple(tuple(tf.Tensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
            Tuple of `tuple(tf.Tensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape
            `(batch_size, num_heads, sequence_length, embed_size_per_head)`)

            Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see
            `past_key_values` input) to speed up sequential decoding.
        hidden_states (`tuple(tf.Tensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
            Tuple of `tf.Tensor` (one for the output of the embeddings, if the model has an embedding layer, +
            one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`.

            Hidden-states of the model at the output of each layer plus the optional initial embedding outputs.
        attentions (`tuple(tf.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
            Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
            sequence_length)`.

            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
            heads.
        image_hidden_states (`tuple(tf.Tensor)`, *optional*):
            Tuple of `tf.Tensor` (one for the output of the image embeddings, `(batch_size, num_images,
            sequence_length, hidden_size)`.

            image_hidden_states of the model produced by the vision encoder, and optionally by the perceiver
    Nr   ÚlossÚlogitsúOptional[List[tf.Tensor]]r    r!   r"   r#   r$   )r%   r&   r'   r(   r/   r)   r0   r    r"   r#   r$   r*   r+   r,   r.   r.   e   sM   … ñð@ !%€DÐ
Ó$Ø"&€FÐÓ&Ø15€OÐ.Ó5Ø04€MÐ-Ó4Ø-1€JÐ*Ó1Ø6:ÐÐ3Ô:r+   r.   c           	     ó  — t        j                  t        j                  t        j                  t        j                  | «      d   «      |«      dg«      }t        j
                  | |«      } |j                  dd «      |d<   |j                  dd «      |d<   |j                  dd «      |d<   |j                  dd «      |d<   d|v r|d   }t        j
                  ||«      |d<   |�t        j
                  ||«      |d<   |d   �t        j
                  |d   |«      |d<   |d   � t        j
                  |d   |«      |d<   | |fS |d   � t        j
                  |d   |«      |d<   | |fS |d   �t        j
                  |d   |«      |d<   | |fS )	Nr   éÿÿÿÿÚpixel_valuesÚimage_encoder_embeddingsÚperceiver_embeddingsÚimage_attention_maskÚtoken_type_idsÚattention_mask)ÚtfÚreshapeÚrepeatÚrangeÚshapeÚgatherÚget)Ú	input_idsÚexpand_sizeÚis_encoder_decoderr9   Úencoder_outputsÚmodel_kwargsÚexpanded_return_idxr8   s           r,   Úexpand_inputs_for_generationrG   �   sÆ  € ô Ÿ*™*¤R§Y¡Y¬r¯x©x¼¿¹ÀÓ8KÈAÑ8NÓ/OÐQ\Ó%]Ð`bÐ_cÓdÐÜ—	‘	˜)Ð%8Ó9€IØ#/×#3Ñ#3°NÀDÓ#I€L�Ñ Ø/;×/?Ñ/?Ð@ZÐ\`Ó/a€LÐ+Ñ,Ø+7×+;Ñ+;Ð<RÐTXÓ+Y€LÐ'Ñ(Ø+7×+;Ñ+;Ð<RÐTXÓ+Y€LÐ'Ñ(à˜<Ñ'Ø%Ð&6Ñ7ˆÜ)+¯©°>ÐCVÓ)WˆÐ%Ñ&àÐ!Ü)+¯©°>ÐCVÓ)WˆÐ%Ñ&àÐ*Ñ+Ð7Ü/1¯y©y¸ÐF\Ñ9]Ð_rÓ/sˆÐ+Ñ,à�NÑ#Ð/Ü')§y¡y°¸nÑ1MÐObÓ'cˆ�^Ñ$ð �lÐ"Ð"ð 
Ð0Ñ	1Ð	=Ü35·9±9ØÐ3Ñ4Ð6Ió4
ˆÐ/Ñ0ð �lÐ"Ð"ð 
Ð,Ñ	-Ð	9Ü/1¯y©y¸ÐF\Ñ9]Ð_rÓ/sˆÐ+Ñ,à�lÐ"Ð"r+   c           	     ól  — d| v r| j                   |d<   nd |d<   d|v r+|d   }t        j                  ||d d …dd …df   gd¬«      |d<   d|v r>|d   }t        j                  |t        j                  |d d …dd …df   «      gd¬«      |d<   d|v r|d   }|d d …dd …df   }||d<   | j                  |d<   |S )	Nr    r8   r3   .©Úaxisr9   r7   r$   )r    r:   ÚconcatÚ	ones_liker$   )ÚoutputsrE   r8   r9   r7   Ú	last_masks         r,   Ú"update_model_kwargs_for_generationrO   ¶   sþ   € à˜GÑ#Ø*1×*AÑ*AˆÐ&Ò'à*.ˆÐ&Ñ'ð ˜<Ñ'Ø%Ð&6Ñ7ˆÜ)+¯©°NÀNÒSTÐVXÑVYÐ[^ÐS^ÑD_Ð3`ÐgiÔ)jˆÐ%Ñ&ð ˜<Ñ'Ø%Ð&6Ñ7ˆÜ)+¯©ØœRŸ\™\¨.º¸B¹CÀ¸Ñ*EÓFÐGÈbô*
ˆÐ%Ñ&ð  Ñ-Ø+Ð,BÑCÐØ(ª¨B©C°¨Ñ5ˆ	Ø/8ˆÐ+Ñ,ð +2×*EÑ*E€LÐ&Ñ'àÐr+   c                ót  — |j                  dd «      }|�| d d …dd …f   } |�|d d …dd …f   }|j                  dd «      }|j                  dd «      }|�p|€nt        j                  j                  t        j                  |t        j
                  ¬«      d¬«      dz
  }t        j                  |dk(  d|«      }|�|d d …dd …f   }|j                  d	d «      }|j                  d
d «      }|j                  dd «      }|j                  dd «      }	|j                  dd«      }
| ||j                  d«      |||||||	|
dœS )Nr8   r3   r9   Úposition_ids©ÚdtyperI   r   r   r4   r5   r6   r7   Úinterpolate_pos_encodingFÚ	use_cache)rA   r    rU   rQ   r9   r8   r4   r5   r6   r7   rT   )r@   r:   ÚmathÚcumsumÚcastÚint64Úwhere)rA   r    Úkwargsr8   r9   rQ   r4   r5   r6   r7   rT   s              r,   Úprepare_inputs_for_generationr\   Ó   sR  € Ø—Z‘ZÐ 0°$Ó7€NàÐ"Øša ¡˜fÑ%ˆ	ØÐ%Ø+ªA¨r©s¨FÑ3ˆNà—Z‘ZÐ 0°$Ó7€NØ—:‘:˜n¨dÓ3€LàÐ! lÐ&:ä—w‘w—~‘~¤b§g¡g¨nÄBÇHÁHÔ&MÐTV�~ÓWÐZ[Ñ[ˆÜ—x‘x °!Ñ 3°Q¸ÓEˆØÐ&Ø'ª¨2©3¨Ñ/ˆLà—:‘:˜n¨dÓ3€LØ%Ÿz™zÐ*DÀdÓKÐØ!Ÿ:™:Ð&<¸dÓCÐØ!Ÿ:™:Ð&<¸dÓCÐØ%Ÿz™zÐ*DÀeÓLÐð Ø*Ø—Z‘Z Ó,Ø$Ø(Ø(Ø$Ø$<Ø 4Ø 4Ø$<ñð r+   c                ó¤  ‡— t         j                  j                  j                  t         j                  j                  j                  t         j                  j                  j
                  dœ}|D �cg c]  }||   ‘Œ	 }}t        | d«      s	d| _        | S | j                  D ]'  Š|rt        ˆfd„|D «       «      rd‰_        Œ!d‰_        Œ) | S c c}w )N)Ú	LayerNormÚDenseÚ	EmbeddingÚlayersFc              3  ó6   •K  — | ]  }t        ‰|«      –— Œ y ­w©N)Ú
isinstance)Ú.0ÚtÚlayers     €r,   ú	<genexpr>zfreeze_model.<locals>.<genexpr>  s   øè ø€ Ò$\¸a¤Z°°q×%9Ñ$\ùs   ƒT)	r:   Úkerasra   ÚLayerNormalizationr_   r`   ÚhasattrÚ	trainableÚany)ÚmodelÚmodule_exceptionsÚmappingÚmÚmodule_exceptions_mappedrg   s        @r,   Úfreeze_modelrs   ú   s­   ø€ ä—X‘X—_‘_×7Ñ7Ü—‘—‘×&Ñ&Ü—X‘X—_‘_×.Ñ.ñ€Gð
 5FÖF¨q ¨£
ÐFÐÐFÜ�5˜(Ô#ØˆŒØˆØ—‘ò $ˆÙ¤Ó$\ÐC[Ô$\Ô!\Ø"ˆE�Oà#ˆE�Oð	$ð
 €Lùò  Gs   Á2Cc                  óB   ‡ — e Zd ZdZ	 	 d	 	 	 dˆ fd„Zˆ fd„Zdd„Zˆ xZS )ÚTFIdeficsDecoupledEmbeddingaÌ  
    Implements a decoupling of parameters to allow freezing (or not) a subset of the embeddings. In practise, the
    regular `weight` can be trained or frozen (i.e. `partially_freeze=True`), and if `num_additional_embeddings` > 0,
    then it will create `num_additional_embeddings` additional parameters that are always trained. If
    `num_additional_embeddings=0`, then the module defaults back to the regular behavior of `tf.keras.layers.Embedding`.
    c                ó   •— t        ‰| �  d|||dœ|¤Ž || _        || _        || _        |rd| _        | j                  dkD  r=t        j                  j                  j                  | j                  ||d¬«      | _
        yy)aÏ  
        Args:
            num_embeddings (`int`):
                Size of the dictionary of embeddings
            num_additional_embeddings (`int`):
                Number of additional embeddings. Only useful when you `partially_freeze=True`.
            embedding_dim (`int`):
                The size of each embedding vector
            partially_freeze: (`bool`, *optional*, defaults to `False`):
                If `True`, the regular `weight` will be frozen. `additional_weight` is never frozen.

        Note: there are a lot of other parameters to initialize a standard `tf.keras.layers.Embedding` such as `mask_zero`,
        `input_length` or `embeddings_initializer`. We are not supporting these.
        )Ú	input_dimÚ
output_dimrS   Fr   Úadditional_embedding)rw   rx   rS   ÚnameNr*   )ÚsuperÚ__init__Únum_embeddingsÚnum_additional_embeddingsÚpartially_freezerl   r:   ri   ra   r`   ry   )Úselfr}   r~   Úembedding_dimr   rS   r[   Ú	__class__s          €r,   r|   z$TFIdeficsDecoupledEmbedding.__init__  s”   ø€ ô. 	‰Ñð 	
Ø$Ø$Øñ	
ð ò		
ð -ˆÔØ)BˆÔ&Ø 0ˆÔáØ"ˆDŒNà×)Ñ)¨AÒ-Ü(*¯©¯©×(AÑ(AØ×8Ñ8Ø(ØØ+ð	 )Bó )ˆDÕ%ð .r+   c           	     óþ  •— | j                   dk(  rt        ‰| �	  |«      S t        j                  |«      }t        j
                  || j                  k\  «      }t        j                  ||«      }| j                  || j                  z
  «      }t        j                  ||t        j                  t        j                  |«      d   |j                  ¬«      «      }t        ‰| �	  |«      }t        j                  |||«      }|S )aû  
        we have 2 embeddings, with different indices - one pretrained self.weight and another
        self.additional_embedding.weight that is being trained.

        in order to make a lookup of the input ids, we:
        1. find out the indices of the entries belonging to the 2nd embedding
        2. extract those values while subtracting the size of the first embedding (num_embeddings), since the 2nd
           embedding starts from 0 and not num_embeddings
        3. perform the 2nd embedding lookup
        4. now we handle the 1st embedding, we overwrite indices belonging to the 2nd embedding with a padding index
        5. perform the 1st embedding lookup
        6. now we overwrite the values in the 1st embedding lookup with the values of the 2nd embedding lookup

        note: for the 1st embedding lookup we could have looked up only the low indices and not do the padding, but
        then we have to create a new tensor and populate it with 2 tensors that are spread out across various indices -
        i.e. not a simple concat - I haven't benchmarked the complex case if it's any faster, given that seqlens are
        usually relatively short it's probably not faster or if faster not by much - but might be a good idea to
        measure.

        r   rR   )r~   r{   Úcallr:   ÚidentityrZ   r}   Ú	gather_ndry   Útensor_scatter_nd_updateÚzerosr>   rS   )r€   rA   Úadditional_vocab_indicesÚinput_ids_additional_vocabÚadditional_embeddingsÚfull_vectorr‚   s         €r,   r„   z TFIdeficsDecoupledEmbedding.call@  sã   ø€ ð* ×)Ñ)¨QÒ.Ü‘7‘< 	Ó*Ð*ô —K‘K 	Ó*ˆ	Ü#%§8¡8¨I¸×9LÑ9LÑ,LÓ#MÐ Ü%'§\¡\°)Ð=UÓ%VÐ"Ø $× 9Ñ 9Ð:TÐW[×WjÑWjÑ:jÓ kÐô ×/Ñ/ØØ$ä�H‰H”R—X‘XÐ6Ó7¸Ñ:À)Ç/Á/ÔRó	
ˆ	ô ‘g‘l 9Ó-ˆô ×1Ñ1°+Ð?WÐYnÓoˆàÐr+   c                óz   — dj                  | j                  | j                  | j                  | j                  «      S )NzVnum_embeddings={}, num_additional_embeddings={}, embedding_dim={}, partially_freeze={})Úformatr}   r~   rx   r   ©r€   s    r,   Ú
extra_reprz&TFIdeficsDecoupledEmbedding.extra_reprl  s7   € Øg×nÑnØ×ÑØ×*Ñ*Ø�O‰OØ×!Ñ!ó	
ð 	
r+   )FN)r   úOptional[bool]ÚreturnÚNone©r’   Ústr)r%   r&   r'   r(   r|   r„   r�   Ú__classcell__©r‚   s   @r,   ru   ru     s5   ø„ ñð ,1Øð*ð
 )ð*ð 
õ*ôX*÷X
r+   ru   c                  ót   ‡ — e Zd ZdZ	 	 	 d	 	 	 	 	 	 	 	 	 	 	 d	ˆ fd„Zd
d„Zˆ fd„Zdd„Zed„ «       Z	dd„Z
ˆ xZS )ÚTFIdeficsDecoupledLinearaÐ  
    Implements a decoupling of parameters to allow freezing (or not) a subset of the parameters. In practise, the
    regular `weight` can be trained or frozen (i.e. `partially_freeze=True`), and if `out_additional_features` > 0,
    then it will create `out_additional_features * in_features` additional parameters that are always trained. If
    `out_additional_features=0`, then the module defaults back to the regular behavior of `tf.keras.layers.Dense`.
    c                óØ   •— t        ‰| �  di |¤Ž || _        || _        || _        || _        || _        |dkD  r2t        j                  j                  j                  ||d¬«      | _        yy)aS  
        out_additional_features: int. Number of additional trainable dimensions. Only makes sense when
        `partially_freeze=True`. partially_freeze: bool. If True, the regular `weight` will be frozen and extra
        parameters (if any) will be trainable. If False, default to the regular behavior of tf.keras.layers.Dense.
        r   Úadditional_fc)ÚunitsÚuse_biasrz   Nr*   )r{   r|   Úout_additional_featuresr   Úin_featuresÚout_featuresr�   r:   ri   ra   r_   r›   )r€   rŸ   r    rž   Úbiasr   r[   r‚   s          €r,   r|   z!TFIdeficsDecoupledLinear.__init__}  sp   ø€ ô 	‰ÑÑ"˜6Ò"Ø'>ˆÔ$Ø 0ˆÔà&ˆÔØ(ˆÔØˆŒà" QÒ&Ü!#§¡§¡×!6Ñ!6Ø-¸À?ð "7ó "ˆDÕð 'r+   c                ó<  — t         j                  j                  || j                  d¬«      }| j                  �*t         j
                  j                  || j                  «      }| j                  dkD  r*| j                  |«      }t        j                  ||gd¬«      }|S )NT)ÚaÚbÚtranspose_br   r3   rI   )
r:   ÚlinalgÚmatmulÚweightr¡   ÚnnÚbias_addrž   r›   rK   )r€   ÚinputsÚoutputÚadditional_featuress       r,   r„   zTFIdeficsDecoupledLinear.call˜  s}   € Ü—‘×!Ñ! F¨d¯k©kÀtÐ!ÓLˆØ�9‰9Ð Ü—U‘U—^‘^ F¨D¯I©IÓ6ˆFà×'Ñ'¨!Ò+Ø"&×"4Ñ"4°VÓ"<ÐÜ—Y‘Y Ð(;Ð<À2ÔFˆFàˆr+   c                óº   •— t         ‰| �  «       }|j                  | j                  | j                  | j
                  | j                  d u| j                  dœ«       |S )N)rŸ   r    rž   r¡   r   )r{   Ú
get_configÚupdaterŸ   r    rž   r¡   r   )r€   Úconfigr‚   s     €r,   r¯   z#TFIdeficsDecoupledLinear.get_config£  sX   ø€ Ü‘Ñ#Ó%ˆØ�‰à#×/Ñ/Ø $× 1Ñ 1Ø+/×+GÑ+GØŸ	™	¨Ð-Ø$(×$9Ñ$9ñô	
ð ˆr+   c                ó”   — dj                  | j                  | j                  | j                  | j                  du| j
                  «      S )z=Overwriting `nn.Linear.extra_repr` to include new parameters.zYin_features={}, out_features={}, out_additional_features={}, bias={}, partially_freeze={}N)rŽ   rŸ   r    rž   r¡   r   r�   s    r,   r�   z#TFIdeficsDecoupledLinear.extra_repr°  sE   € àj×qÑqØ×ÑØ×ÑØ×(Ñ(Ø�I‰I˜TÐ!Ø×!Ñ!ó
ð 	
r+   c                ó   —  | di |¤ŽS )Nr*   r*   )Úclsr±   s     r,   Úfrom_configz$TFIdeficsDecoupledLinear.from_configº  s   € á‰}�V‰}Ðr+   c                ó  — | j                   ry d| _         | j                  | j                  | j                  f| j                   d¬«      | _        | j                  r0| j                  | j                  f| j                   d¬«      | _        nd | _        t        | dd «      �Xt        j                  | j                  j                  «      5  | j                  j                  | j                  «       d d d «       y y # 1 sw Y   y xY w)NTr¨   )r>   rl   rz   r¡   r›   )ÚbuiltÚ
add_weightr    rŸ   r   r¨   r�   r¡   Úgetattrr:   Ú
name_scoper›   rz   Úbuild©r€   Úinput_shapes     r,   r»   zTFIdeficsDecoupledLinear.build¾  sä   € Ø�:Š:ØØˆŒ
Ø—o‘oØ×$Ñ$ d×&6Ñ&6Ð7Àt×G\ÑG\ÐC\Ðckð &ó 
ˆŒð �=Š=ØŸ™¨t×/@Ñ/@Ð.BÐRV×RgÑRgÐNgÐnt˜ÓuˆD�IàˆDŒIÜ�4˜¨$Ó/Ð;Ü—‘˜t×1Ñ1×6Ñ6Ó7ñ ;Ø×"Ñ"×(Ñ(¨×)9Ñ)9Ô:÷;ð ;ð <÷;ð ;ús   Ã&C8Ã8D)r   TT)rŸ   Úintr    r¾   rž   r¾   r¡   Úboolr   r¿   r’   r“   )r«   ú	tf.Tensorr’   rÀ   r”   rc   )r%   r&   r'   r(   r|   r„   r¯   r�   Úclassmethodrµ   r»   r–   r—   s   @r,   r™   r™   u  sz   ø„ ñð ()ØØ!%ðàðð ðð "%ð	ð
 ðð ðð 
õó6	ôó
ð ñó ð÷;r+   r™   c                óV  — | \  }}t        j                  ||ft         j                  j                  |«      j                  «      }t        j
                  |«      }t        j                  |dd…df   |ddd…f   k\  d|«      }|dkD  r0t        j                  t        j                  ||f|¬«      |gd¬«      }|€Ht        j                  |d«      }t        j                  |d«      }t        j                  ||dddg«      }|S t        j                  |dddd…dd…f   |d|||z   f«      }|S )zm
    Make causal mask used for bi-directional self-attention, supporting both static and dynamic shapes.
    Nç        r   rR   r3   rI   r   )r:   ÚfillÚdtypesÚas_dtypeÚminr=   rZ   rK   rˆ   Úexpand_dimsÚtileÚbroadcast_to)Úinput_ids_shaperS   Úpast_key_values_lengthÚbszÚtgt_lenÚmaskÚ	mask_conds          r,   Ú_make_causal_maskrÑ   Î  s  € ð #�L€Cˆô �7‰7�G˜WÐ%¤r§y¡y×'9Ñ'9¸%Ó'@×'DÑ'DÓE€DÜ—‘˜Ó!€IÜ�8‰8�Iša ˜gÑ&¨)°Dº!°GÑ*<Ñ<¸cÀ4ÓH€Dà Ò!Ü�y‰yœ"Ÿ(™( GÐ-CÐ#DÈEÔRÐTXÐYÐ`bÔcˆà
€{ô �~‰~˜d AÓ&ˆÜ�~‰~˜d AÓ&ˆÜ�w‰w�t˜c 1 a¨˜^Ó,ˆð
 €Kô �‰˜t D¨$²²1Ð$4Ñ5¸¸QÀÈÐSiÑIiÐ7jÓkˆà€Kr+   c                ó®  — t        | «      \  }}|�|n|}t        j                  t        j                  | d«      d«      }t        j                  ||d||g«      }dt        j                  ||«      z
  }t        j
                  t        j                  |t        «      t        j                  t        |«      t        j                  j                  ¬«      |«      S )z_
    Expands attention_mask from `[bsz, seq_len]` to `[bsz, 1, tgt_seq_len, src_seq_len]`.
    r   ç      ð?)ÚdimsÚvalue)
r   r:   rÈ   rÊ   rX   rZ   r¿   rÄ   Úfloat32rÇ   )rÏ   rS   rÎ   rÍ   Úsrc_lenÚexpanded_maskÚinverted_masks          r,   Ú_expand_maskrÚ   é  s¨   € ô ˜dÓ#�L€CˆØ Ð,‰g°'€Gä—N‘N¤2§>¡>°$¸Ó#:¸AÓ>€MÜ—O‘O M°C¸¸GÀWÐ3MÓN€Màœ"Ÿ'™' -°Ó7Ñ7€Mä�8‰8Ü
�‰�œtÓ$¤b§g¡g´:¸mÓ3LÔTV×T^ÑT^×TbÑTbÔ&cÐeróð r+   c                  ó0   ‡ — e Zd Zdˆ fd„	Zˆ fd„Zd„ Zˆ xZS )ÚTFIdeficsRMSNormc                ó@   •— t        ‰| �  di |¤Ž || _        || _        y)z?
        TFIdeficsRMSNorm is equivalent to T5LayerNorm
        Nr*   )r{   r|   Úhidden_sizeÚvariance_epsilon)r€   rÞ   Úepsr[   r‚   s       €r,   r|   zTFIdeficsRMSNorm.__init__û  s%   ø€ ô 	‰ÑÑ"˜6Ò"Ø&ˆÔØ #ˆÕr+   c                ó”   •— | j                   ry d| _         | j                  d| j                  gd¬«      | _        t        ‰| �  |«       y )NTr¨   Úones)rz   r>   Úinitializer)r·   r¸   rÞ   r¨   r{   r»   ©r€   r½   r‚   s     €r,   r»   zTFIdeficsRMSNorm.build  s@   ø€ Ø�:Š:ØØˆŒ
Ø—o‘o¨8¸D×<LÑ<LÐ;MÐ[a�oÓbˆŒä‰‰�kÕ"r+   c                ó   — t         j                  j                  t         j                  j                  t        j                  |t         j
                  «      «      dd¬«      }|t         j                  j                  || j                  z   «      z  }| j                  j                  t         j                  t         j                  fv r*t        j                  || j                  j                  «      }| j                  |z  S )Nr3   T)rJ   Úkeepdims)r:   rV   Úreduce_meanÚsquarerX   rÖ   Úrsqrtrß   r¨   rS   Úfloat16Úbfloat16)r€   r"   Úvariances      r,   r„   zTFIdeficsRMSNorm.call  s§   € Ü—7‘7×&Ñ&¤r§w¡w§~¡~´b·g±g¸mÌRÏZÉZÓ6XÓ'YÐ`bÐmqÐ&ÓrˆØ%¬¯©¯©°hÀ×AVÑAVÑ6VÓ(WÑWˆð �;‰;×Ñ¤§¡¬R¯[©[Ð 9Ñ9ÜŸG™G M°4·;±;×3DÑ3DÓEˆMà�{‰{˜]Ñ*Ð*r+   )g�íµ ÷Æ°>©r%   r&   r'   r|   r»   r„   r–   r—   s   @r,   rÜ   rÜ   ú  s   ø„ õ$ô#ö+r+   rÜ   c                  ó.   ‡ — e Zd Zdˆ fd„	Zd„ Zdd„Zˆ xZS )ÚTFIdeficsEmbeddingc           
     ó  •— t        ‰| �  di |¤Ž || _        || _        || _        t        j                  d| j                  t        j                  d| j                  dt
        j                  ¬«      | j                  z  z  z  «      | _	        y )NrÓ   r   é   )ÚstartÚlimitÚdeltarS   r*   )
r{   r|   ÚdimÚmax_position_embeddingsÚbaser:   Úconstantr=   rÖ   Úinv_freq)r€   rõ   rö   r÷   r[   r‚   s        €r,   r|   zTFIdeficsEmbedding.__init__  sp   ø€ Ü‰ÑÑ"˜6Ò"àˆŒØ'>ˆÔ$ØˆŒ	ÜŸ™Ø�4—9‘9¤§¡°¸¿¹ÈÔRT×R\ÑR\Ô!]Ð`d×`hÑ`hÑ!hÑiÑjó
ˆ�r+   c                ó   — t        j                  || j                  j                  ¬«      }t        j                  d|| j                  «      }t        j
                  ||fd¬«      }t        j                  |«      t        j                  |«      fS )NrR   z
i, j -> ijr3   rI   )r:   r=   rù   rS   ÚeinsumrK   ÚcosÚsin)r€   Úseq_lenrf   ÚfreqsÚembs        r,   Ú_compute_cos_sinz#TFIdeficsEmbedding._compute_cos_sin!  sa   € Ü�H‰H�W D§M¡M×$7Ñ$7Ô8ˆÜ—	‘	˜,¨¨4¯=©=Ó9ˆÜ�i‰i˜ ˜¨RÔ0ˆä�v‰v�c‹{œBŸF™F 3›KÐ'Ð'r+   c                óF   — |€t        |«      d   }| j                  |¬«      S )Nrñ   )rþ   )r   r  )r€   Úxrþ   s      r,   r„   zTFIdeficsEmbedding.call(  s)   € àˆ?Ü  “m AÑ&ˆGØ×$Ñ$¨WÐ$Ó5Ð5r+   )i   i'  rc   )r%   r&   r'   r|   r  r„   r–   r—   s   @r,   rï   rï     s   ø„ õ
ò(÷6r+   rï   c                óš   — | dd| j                   d   dz  …f   }| d| j                   d   dz  d…f   }t        j                  | |fd¬«      S )z*Rotates half the hidden dims of the input..Nr3   rñ   rI   )r>   r:   rK   )r  Úx1Úx2s      r,   Úrotate_halfr  /  sZ   € à	
ˆ3Ð"�!—'‘'˜"‘+ Ñ"Ð"Ð"Ñ	#€BØ	
ˆ3�—‘˜‘˜qÑ Ñ"Ð"Ñ	#€BÜ�9‰9�r�c˜2�Y RÔ(Ð(r+   c                ó
  — t        j                  ||«      }t        j                  ||«      }t        j                  |d«      }t        j                  |d«      }| |z  t        | «      |z  z   }||z  t        |«      |z  z   }||fS )Nr   )r:   r?   rÈ   r  )ÚqÚkrü   rý   rQ   Úq_embedÚk_embeds          r,   Úapply_rotary_pos_embr  6  s{   € Ü
�)‰)�C˜Ó
&€CÜ
�)‰)�C˜Ó
&€CÜ
�.‰.˜˜aÓ
 €CÜ
�.‰.˜˜aÓ
 €CØ�3‰wœ; q›>¨CÑ/Ñ0€GØ�3‰wœ; q›>¨CÑ/Ñ0€GØ�GÐÐr+   c                  ó:   ‡ — e Zd Z	 	 	 	 	 	 dˆ fd„Zd„ Zdd„Zˆ xZS )ÚTFIdeficsMLPc                ó†  •— t        ‰| �  di |¤Ž t        j                  j                  j                  |dd¬«      | _        t        j                  j                  j                  |dd¬«      | _        t        j                  j                  j                  |dd¬«      | _        t        |«      | _
        || _        || _        y )NFÚ	gate_proj©r�   rz   Ú	down_projÚup_projr*   )r{   r|   r:   ri   ra   r_   r  r  r  r   Úact_fnÚintermediate_sizerÞ   )r€   rÞ   r  Ú
hidden_actr[   r‚   s        €r,   r|   zTFIdeficsMLP.__init__A  s™   ø€ ô 	‰ÑÑ"˜6Ò"ÜŸ™Ÿ™×.Ñ.Ð/@È5ÐWbÐ.ÓcˆŒÜŸ™Ÿ™×.Ñ.¨{ÀUÐQ\Ð.Ó]ˆŒÜ—x‘x—‘×,Ñ,Ð->ÈÐU^Ð,Ó_ˆŒÜ'¨
Ó3ˆŒØ!2ˆÔØ&ˆÕr+   c                ó„   — | j                  | j                  | j                  |«      «      | j                  |«      z  «      S rc   )r  r  r  r  )r€   r  s     r,   r„   zTFIdeficsMLP.callP  s0   € Ø�~‰~˜dŸk™k¨$¯.©.¸Ó*;Ó<¸t¿|¹|ÈA»ÑNÓOÐOr+   c                óÎ  — | j                   ry d| _         t        | dd «      �Wt        j                  | j                  j
                  «      5  | j                  j                  | j                  «       d d d «       t        | dd «      �Wt        j                  | j                  j
                  «      5  | j                  j                  | j                  «       d d d «       t        | dd «      �Xt        j                  | j                  j
                  «      5  | j                  j                  | j                  «       d d d «       y y # 1 sw Y   ŒÓxY w# 1 sw Y   Œ{xY w# 1 sw Y   y xY w)NTr  r  r  )r·   r¹   r:   rº   r  rz   r»   rÞ   r  r  r  r¼   s     r,   r»   zTFIdeficsMLP.buildS  s  € Ø�:Š:ØØˆŒ
Ü�4˜ dÓ+Ð7Ü—‘˜tŸ~™~×2Ñ2Ó3ñ 7Ø—‘×$Ñ$ T×%5Ñ%5Ô6÷7ä�4˜ dÓ+Ð7Ü—‘˜tŸ~™~×2Ñ2Ó3ñ =Ø—‘×$Ñ$ T×%;Ñ%;Ô<÷=ä�4˜ DÓ)Ð5Ü—‘˜tŸ|™|×0Ñ0Ó1ñ 5Ø—‘×"Ñ" 4×#3Ñ#3Ô4÷5ð 5ð 6÷7ð 7ú÷=ð =ú÷5ð 5ús$   Á&EÂ/&EÄ&EÅEÅEÅE$)rÞ   r¾   r  r¾   r  r•   rc   ©r%   r&   r'   r|   r„   r»   r–   r—   s   @r,   r  r  @  s/   ø„ ð'àð'ð ð'ð õ	'òP÷5r+   r  c                  óˆ   ‡ — e Zd ZdZ	 	 	 	 d	 	 	 	 	 	 	 	 	 	 	 dˆ fd„Zdd„Z	 	 	 	 	 	 d		 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d
d„Zdd„Zˆ xZS )ÚTFIdeficsAttentionz=Multi-headed attention from 'Attention Is All You Need' paperc                óÎ  •— t        ‰| �  di |¤Ž || _        || _        ||z  | _        || _        || _        d| _        | j                  |z  | j                  k7  rt        d| j                  › d|› d�«      ‚|| _	        t        j                  j                  j                  || j                  z  dd¬«      | _        t        j                  j                  j                  || j                  z  dd¬«      | _        t        j                  j                  j                  || j                  z  dd	¬«      | _        t        j                  j                  j                  |dd
¬«      | _        t%        | j                  d¬«      | _        || _        | j(                  rOt+        | j                  |j,                  d¬«      | _        t+        | j                  |j,                  d¬«      | _        y y )NTz?hidden_size must be divisible by num_heads (got `hidden_size`: z and `num_heads`: z).FÚq_projr  Úk_projÚv_projÚo_projÚ
rotary_emb©rz   Úq_layer_norm©rà   rz   Úk_layer_normr*   )r{   r|   rÞ   Ú	num_headsÚhead_dimÚdropoutr±   Ú	is_causalÚ
ValueErrorÚis_cross_attentionr:   ri   ra   r_   r  r  r   r!  rï   r"  Úqk_layer_normsrÜ   Úrms_norm_epsr$  r&  )	r€   rÞ   r'  r)  r,  r±   r-  r[   r‚   s	           €r,   r|   zTFIdeficsAttention.__init__e  s³  ø€ ô 	‰ÑÑ"˜6Ò"Ø&ˆÔØ"ˆŒØ# yÑ0ˆŒØˆŒØˆŒØˆŒà�M‰M˜IÑ%¨$×*:Ñ*:Ò:ÜØQÐRV×RbÑRbÐQcØ$ Y K¨rð3óð ð
 #5ˆÔä—h‘h—o‘o×+Ñ+Ø˜Ÿ™Ñ%ØØð ,ó 
ˆŒô
 —h‘h—o‘o×+Ñ+Ø˜Ÿ™Ñ%ØØð ,ó 
ˆŒô
 —h‘h—o‘o×+Ñ+Ø˜Ÿ™Ñ%ØØð ,ó 
ˆŒô
 —h‘h—o‘o×+Ñ+ØØØð ,ó 
ˆŒô
 -¨T¯]©]ÀÔNˆŒà,ˆÔØ×ÒÜ 0°·±ÀF×DWÑDWÐ^lÔ mˆDÔÜ 0°·±ÀF×DWÑDWÐ^lÔ mˆDÕð r+   c           	     óŒ   — t        j                  t        j                  |||| j                  | j                  f«      g d¢¬«      S )N©r   rñ   r   r	   ©Úperm)r:   Ú	transposer;   r'  r(  )r€   Útensorrþ   rÍ   s       r,   Ú_shapezTFIdeficsAttention._shapeš  s0   € Ü�|‰|œBŸJ™J v°°W¸d¿n¹nÈdÏmÉmÐ/\Ó]ÒdpÔqÐqr+   c           
     ón  — | j                   xs |d u}t        |«      \  }	}
}| j                  | j                  |«      |
|	«      }|sE| j                  | j	                  |«      |
|	«      }| j                  | j                  |«      |
|	«      }nSt        |«      \  }}}| j                  | j	                  |«      ||	«      }| j                  | j                  |«      ||	«      }t        |«      d   }|�|t        |d   «      d   z  }|s_t        j                  |«      rt        j                  ||
«      }nt        ||
«      }| j                  ||«      \  }}t        |||||«      \  }}|�8t        j                  |d   |gd¬«      }t        j                  |d   |gd¬«      }|r||fnd }| j                  r"| j                  |«      }| j                  |«      }t        j                   j#                  t        j$                  |«      |	d|
|gd|	d|
|g› dt        j$                  |«      › �¬«       t'        ||||| j(                  xr |d u xr |
dkD  ¬	«      }t        j                   j#                  t        j$                  |«      |	| j*                  |
| j,                  gd|	| j*                  |
| j,                  g› dt        j$                  |«      › �¬«       t        j.                  t        j0                  |g d
¢¬«      |	|
| j2                  f«      }| j5                  |«      }d }|rt6        j9                  d«       |||fS )Néþÿÿÿr   rñ   rI   r   z$Attention weights should be of size z	, but is )Úmessage)Ú	attn_maskr*  r0  r1  z^attn_weights are not extracted in scaled_dot_product_attention. The model returns None instead)r,  r   r5  r  r  r   r:   Ú	is_tensorÚ
reduce_maxÚmaxr"  r  rK   r-  r$  r&  Ú	debuggingÚassert_equalr>   r   r*  r'  r(  r;   r3  rÞ   r!  ÚloggerÚwarning_once)r€   r"   Úkey_value_statesr9   rQ   Úpast_key_valueÚoutput_attentionsrU   r,  rÍ   Úq_lenÚ_Úquery_statesÚ
key_statesÚvalue_statesÚkv_lenÚ
kv_seq_lenrþ   rü   rý   Úattn_outputÚattn_weightss                         r,   r„   zTFIdeficsAttention.call�  s-  € ð "×4Ñ4ÒTÐ8HÐPTÐ8TÐä" =Ó1‰ˆˆU�Aà—{‘{ 4§;¡;¨}Ó#=¸uÀcÓJˆÙ!ØŸ™ T§[¡[°Ó%?ÀÈÓLˆJØŸ;™; t§{¡{°=Ó'AÀ5È#ÓN‰Lä%Ð&6Ó7‰LˆAˆv�qØŸ™ T§[¡[Ð1AÓ%BÀFÈCÓPˆJØŸ;™; t§{¡{Ð3CÓ'DÀfÈcÓRˆLä 
Ó+¨BÑ/ˆ
ØÐ%Øœ* ^°AÑ%6Ó7¸Ñ;Ñ;ˆJÙ!ä�|‰|˜JÔ'ÜŸ-™-¨
°EÓ:‘ä˜j¨%Ó0�Ø—‘ |°WÓ=‰HˆC�Ü';¸LÈ*ÐVYÐ[^Ð`lÓ'mÑ$ˆL˜*ð Ð%äŸ™ N°1Ñ$5°zÐ#BÈÔKˆJÜŸ9™9 n°QÑ&7¸Ð%FÈQÔOˆLá7@˜* lÑ3Àdˆà×ÒØ×,Ñ,¨\Ó:ˆLØ×*Ñ*¨:Ó6ˆJä
�‰×!Ñ!Ü�H‰H�^Ó$Ø�!�U˜JÐ'Ø:¸CÀÀEÈ:Ð;VÐ:WÐW`Ôac×aiÑaiÐjxÓayÐ`zÐ{ð 	"ô 	
ô 3ØØØØ$à—n‘nÒM¨¸4Ð)?ÒMÀEÈAÁIô
ˆô 	�‰×!Ñ!Ü�H‰H�[Ó!Ø�$—.‘. %¨¯©Ð7Ø:¸CÀÇÁÐQVÐX\×XeÑXeÐ;fÐ:gÐgpÔqs×qyÑqyð  {Fó  rGð  qHð  Ið 	"ô 	
ô —j‘j¤§¡¨kÂÔ!MÐPSÐUZÐ\`×\lÑ\lÐOmÓnˆà—k‘k +Ó.ˆàˆÙÜ×ÑØpôð ˜L¨.Ð8Ð8r+   c                ó:  — | j                   ry d| _         | j                  rMt        | j                  j                  d«      s| j
                  n| j                  j                  j                  }n| j
                  }t        | dd «      �dt        j                  | j                  j                  «      5  | j                  j                  | j                  | j                  z  «       d d d «       t        | dd «      �Wt        j                  | j                  j                  «      5  | j                  j                  | j
                  «       d d d «       t        | dd «      �Mt        j                  | j                   j                  «      5  | j                   j                  |«       d d d «       t        | dd «      �Mt        j                  | j"                  j                  «      5  | j"                  j                  |«       d d d «       t        | dd «      �Nt        j                  | j$                  j                  «      5  | j$                  j                  d «       d d d «       y y # 1 sw Y   �Œ~xY w# 1 sw Y   �Œ'xY w# 1 sw Y   ŒÙxY w# 1 sw Y   Œ‹xY w# 1 sw Y   y xY w)NTÚ	embed_dimr!  r  r  r   r"  )r·   r,  rk   r±   Úvision_configrÞ   rN  r¹   r:   rº   r!  rz   r»   r'  r(  r  r  r   r"  )r€   r½   Úkv_input_dims      r,   r»   zTFIdeficsAttention.buildî  sÿ  € Ø�:Š:ØØˆŒ
Ø×"Ò"ô ˜tŸ{™{×8Ñ8¸+ÔFð × Ò à—[‘[×.Ñ.×8Ñ8ñ ð  ×+Ñ+ˆLÜ�4˜ 4Ó(Ð4Ü—‘˜tŸ{™{×/Ñ/Ó0ñ BØ—‘×!Ñ! $§.¡.°4·=±=Ñ"@ÔA÷Bä�4˜ 4Ó(Ð4Ü—‘˜tŸ{™{×/Ñ/Ó0ñ 4Ø—‘×!Ñ! $×"2Ñ"2Ô3÷4ä�4˜ 4Ó(Ð4Ü—‘˜tŸ{™{×/Ñ/Ó0ñ 0Ø—‘×!Ñ! ,Ô/÷0ä�4˜ 4Ó(Ð4Ü—‘˜tŸ{™{×/Ñ/Ó0ñ 0Ø—‘×!Ñ! ,Ô/÷0ä�4˜ tÓ,Ð8Ü—‘˜tŸ™×3Ñ3Ó4ñ ,Ø—‘×%Ñ% dÔ+÷,ð ,ð 9÷Bñ Bú÷4ñ 4ú÷0ð 0ú÷0ð 0ú÷,ð ,ús<   Â03IÄ!&I,ÆI9ÇJÈ9JÉI)É,I6É9JÊJÊJ)rÃ   FNF)rÞ   r¾   r'  r¾   r)  Úfloatr,  r¿   r±   r   r-  r¿   )r4  rÀ   rþ   r¾   rÍ   r¾   )NNNNFF)r"   rÀ   rA  r   r9   r   rQ   r   rB  r!   rC  r¿   rU   r¿   r’   zATuple[tf.Tensor, Optional[tf.Tensor], Optional[Tuple[tf.Tensor]]]rc   )	r%   r&   r'   r(   r|   r5  r„   r»   r–   r—   s   @r,   r  r  b  sØ   ø„ ÙGð Ø#(Ø $Ø$ð3nàð3nð ð3nð ð	3nð
 !ð3nð ð3nð õ3nójrð 15Ø.2Ø,0Ø59Ø"'ØðO9à ðO9ð .ðO9ð ,ð	O9ð
 *ðO9ð 3ðO9ð  ðO9ð ðO9ð 
KóO9÷b,r+   r  c                  óX   ‡ — e Zd Zdˆ fd„Z	 	 	 	 	 	 d	 	 	 	 	 	 	 	 	 	 	 	 	 dd„Zdd„Zˆ xZS )ÚTFIdeficsDecoderLayerc                óÎ  •— t        ‰| �  di |¤Ž |j                  | _        t        | j                  |j                  |j
                  |d¬«      | _        t        | j                  |j                  |j                  d¬«      | _
        t        |j                  |j                  d¬«      | _        t        |j                  |j                  d¬«      | _        |j
                  | _        y )	NÚ	self_attn)rÞ   r'  r)  r±   rz   Úmlp©rÞ   r  r  rz   Úinput_layernormr%  Úpost_attention_layernormr*   )r{   r|   rÞ   r  Únum_attention_headsr)  rU  r  r  r  rV  rÜ   r.  rX  rY  ©r€   r±   r[   r‚   s      €r,   r|   zTFIdeficsDecoderLayer.__init__  sÆ   ø€ Ü‰ÑÑ"˜6Ò"Ø!×-Ñ-ˆÔÜ+Ø×(Ñ(Ø×0Ñ0Ø—N‘NØØô
ˆŒô  Ø×(Ñ(Ø$×6Ñ6Ø×(Ñ(Øô	
ˆŒô  0°×0BÑ0BÈ×H[ÑH[ÐbsÔtˆÔÜ(8Ø×Ñ F×$7Ñ$7Ð>Xô)
ˆÔ%ð —~‘~ˆ�r+   c                ó�  — |}| j                  |«      }| j                  ||||||¬«      \  }}	}
t        j                  j	                  || j                  ¬«      }||z   }|}| j                  |«      }| j                  |«      }t        j                  j	                  || j                  ¬«      }||z   }|f}|r||	fz  }|r||
fz  }|S )aE  
        Args:
            hidden_states (`tf.Tensor`): input to the layer of shape `(batch, seq_len, embed_dim)`
            attention_mask (`tf.Tensor`, *optional*): attention mask of size
                `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values.
            output_attentions (`bool`, *optional*):
                Whether or not to return the attentions tensors of all attention layers. See `attentions` under
                returned tensors for more detail.
            use_cache (`bool`, *optional*):
                If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding
                (see `past_key_values`).
            past_key_value (`Tuple(tf.Tensor)`, *optional*): cached past key and value projection states
        )r"   r9   rQ   rB  rC  rU   ©Úrate)rX  rU  r:   r©   r)  rY  rV  )r€   r"   r9   rQ   rB  rC  rU   ÚtrainingÚresidualÚself_attn_weightsÚpresent_key_valuerM   s               r,   r„   zTFIdeficsDecoderLayer.call"  sé   € ð0 !ˆà×,Ñ,¨]Ó;ˆð ?C¿n¹nØ'Ø)Ø%Ø)Ø/Øð ?Mó ?
Ñ;ˆÐ(Ð*;ô Ÿ™Ÿ™ m¸$¿,¹,˜ÓGˆØ  =Ñ0ˆð !ˆØ×5Ñ5°mÓDˆØŸ™ Ó/ˆÜŸ™Ÿ™ m¸$¿,¹,˜ÓGˆØ  =Ñ0ˆà Ð"ˆáØÐ)Ð+Ñ+ˆGáØÐ)Ð+Ñ+ˆGàˆr+   c                ó`  — | j                   ry d| _         t        | dd «      �Mt        j                  | j                  j
                  «      5  | j                  j                  d «       d d d «       t        | dd «      �Mt        j                  | j                  j
                  «      5  | j                  j                  d «       d d d «       t        | dd «      �Mt        j                  | j                  j
                  «      5  | j                  j                  d «       d d d «       t        | dd «      �Nt        j                  | j                  j
                  «      5  | j                  j                  d «       d d d «       y y # 1 sw Y   �ŒxY w# 1 sw Y   ŒÌxY w# 1 sw Y   Œ~xY w# 1 sw Y   y xY w)NTrU  rV  rX  rY  )
r·   r¹   r:   rº   rU  rz   r»   rV  rX  rY  r¼   s     r,   r»   zTFIdeficsDecoderLayer.build[  sT  € Ø�:Š:ØØˆŒ
Ü�4˜ dÓ+Ð7Ü—‘˜tŸ~™~×2Ñ2Ó3ñ +Ø—‘×$Ñ$ TÔ*÷+ä�4˜ Ó%Ð1Ü—‘˜tŸx™xŸ}™}Ó-ñ %Ø—‘—‘˜tÔ$÷%ä�4Ð*¨DÓ1Ð=Ü—‘˜t×3Ñ3×8Ñ8Ó9ñ 1Ø×$Ñ$×*Ñ*¨4Ô0÷1ä�4Ð3°TÓ:ÐFÜ—‘˜t×<Ñ<×AÑAÓBñ :Ø×-Ñ-×3Ñ3°DÔ9÷:ð :ð G÷+ñ +ú÷%ð %ú÷1ð 1ú÷:ð :ús0   ÁE?Â%FÃ?FÅF$Å?F	ÆFÆF!Æ$F-©r±   r   )NNNFFF)r"   rÀ   r9   r   rQ   r   rB  r!   rC  r‘   rU   r‘   r’   ú7Tuple[tf.Tensor, Optional[Tuple[tf.Tensor, tf.Tensor]]]rc   r  r—   s   @r,   rS  rS    sl   ø„ õ&ð2 /3Ø,0Ø59Ø,1Ø$)Øð7à ð7ð ,ð7ð *ð	7ð
 3ð7ð *ð7ð "ð7ð 
Aó7÷r:r+   rS  c                  ód   ‡ — e Zd Zdˆ fd„Zˆ fd„Z	 	 	 	 	 	 	 d	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dd„Zˆ xZS )Ú!TFIdeficsGatedCrossAttentionLayerc           	     óð  •— t        ‰| �  d	i |¤Ž |j                  | _        t        | j                  |j                  d|j
                  ||j                  d¬«      | _        t        | j                  |j                  |j                  d¬«      | _        t        |j                  |j                  d¬«      | _        t        |j                  |j                  d¬«      | _        |j
                  | _        t"        j$                  j&                  j(                  | _        t"        j$                  j&                  j(                  | _        |j.                  | _        |j0                  | _        |j2                  | _        y )
NTÚ
cross_attn)rÞ   r'  r,  r)  r±   r-  rz   rV  rW  rX  r%  rY  r*   )r{   r|   rÞ   r  rZ  r)  r-  ri  r  r  r  rV  rÜ   r.  rX  rY  r±   r:   ri   ÚactivationsÚtanhÚact_cross_attnÚ	act_denseÚalpha_initializerÚ
alpha_typeÚalphas_initializer_ranger[  s      €r,   r|   z*TFIdeficsGatedCrossAttentionLayer.__init__n  s*  ø€ Ü‰ÑÑ"˜6Ò"Ø!×-Ñ-ˆÔÜ,Ø×(Ñ(Ø×0Ñ0Ø#Ø—N‘NØØ!×0Ñ0Øô
ˆŒô  Ø×(Ñ(Ø$×6Ñ6Ø×(Ñ(Øô	
ˆŒô  0°×0BÑ0BÈ×H[ÑH[ÐbsÔtˆÔÜ(8Ø×Ñ F×$7Ñ$7Ð>Xô)
ˆÔ%ð —n‘nˆŒä Ÿh™h×2Ñ2×7Ñ7ˆÔÜŸ™×-Ñ-×2Ñ2ˆŒà!'×!9Ñ!9ˆÔØ ×+Ñ+ˆŒØ(.×(GÑ(GˆÕ%r+   c                óê	  •— | j                   ry d| _         | j                  dk(  r½| j                  dk(  rP| j                  dd| j                  fddd¬«      | _        | j                  dd| j                  fddd¬«      | _        �nÙ| j                  dk(  r6| j                  d	ddd¬«      | _        | j                  d	ddd¬«      | _        �n”t        d
| j                  › d�«      ‚| j                  dk(  r½| j                  dk(  rP| j                  dd| j                  fddd¬«      | _        | j                  dd| j                  fddd¬«      | _        �n| j                  dk(  r6| j                  d	ddd¬«      | _        | j                  d	ddd¬«      | _        �nÈt        d
| j                  › d�«      ‚| j                  dv �r‡| j                  dk(  rµ| j                  dd| j                  ft        j                  j                  j                  d| j                  ¬«      dd¬«      | _        | j                  dd| j                  ft        j                  j                  j                  d| j                  ¬«      dd¬«      | _        nÜ| j                  dk(  r›| j                  d	t        j                  j                  j                  d| j                  ¬«      dd¬«      | _        | j                  d	t        j                  j                  j                  d| j                  ¬«      dd¬«      | _        n2t        d
| j                  › d�«      ‚t        d| j                  › d�«      ‚t        | d«      rt        | d«      st        d«      ‚t        j                  | j                   j"                  «      5  | j                   j%                  d «       d d d «       t        j                  | j&                  j"                  «      5  | j&                  j%                  d «       d d d «       t        j                  | j(                  j"                  «      5  | j(                  j%                  d «       d d d «       t        j                  | j*                  j"                  «      5  | j*                  j%                  d «       d d d «       t,        ‰| �I  |«       y # 1 sw Y   �ŒxY w# 1 sw Y   ŒÀxY w# 1 sw Y   ŒxY w# 1 sw Y   Œ>xY w)NTrˆ   Úvectorr   Úalpha_cross_attn)r>   rã   rl   rz   Úalpha_denserQ  )r   z Unknown value for `alpha_type` (ú)râ   >   ÚnormalÚrandomÚgaussianrÃ   )ÚmeanÚstddevro  zAlpha initialization scheme z not yet implemented!z+Alpha parameters not initialized correctly!)r·   rn  ro  r¸   rÞ   rs  rt  r+  r:   ri   ÚinitializersÚRandomNormalrp  ÚNotImplementedErrorrk   rº   ri  rz   r»   rV  rX  rY  r{   rä   s     €r,   r»   z'TFIdeficsGatedCrossAttentionLayer.build�  sƒ  ø€ Ø�:Š:ØØˆŒ
Ø×!Ñ! WÒ,Ø�‰ (Ò*Ø(,¯©Ø˜a ×!1Ñ!1Ð2ÀÐSWÐ^pð )8ó )�Ô%ð $(§?¡?Ø˜a ×!1Ñ!1Ð2ÀÐSWÐ^kð $3ó $�Ö ð —‘ GÒ+Ø(,¯©Ø¨G¸tÐJ\ð )8ó )�Ô%ð $(§?¡?¸È7Ð^bÐiv ?Ó#w�Ö ä Ð#CÀDÇOÁOÐCTÐTUÐ!VÓWÐWà×#Ñ# vÒ-Ø�‰ (Ò*Ø(,¯©Ø˜a ×!1Ñ!1Ð2ÀÐRVÐ]oð )8ó )�Ô%ð $(§?¡?Ø˜a ×!1Ñ!1Ð2ÀÐRVÐ]jð $3ó $�Ö ð —‘ GÒ+Ø(,¯©Ø¨F¸dÐI[ð )8ó )�Ô%ð $(§?¡?¸È6Ð]aÐhu ?Ó#v�Ö ä Ð#CÀDÇOÁOÐCTÐTUÐ!VÓWÐWà×#Ñ#Ð'GÒGØ�‰ (Ò*Ø(,¯©Ø˜a ×!1Ñ!1Ð2Ü "§¡× 5Ñ 5× BÑ BÈÐTX×TqÑTqÐ BÓ rØ"Ø+ð	 )8ó )�Ô%ð $(§?¡?Ø˜a ×!1Ñ!1Ð2Ü "§¡× 5Ñ 5× BÑ BÈÐTX×TqÑTqÐ BÓ rØ"Ø&ð	 $3ó $�Õ ð —‘ GÒ+Ø(,¯©ØÜ "§¡× 5Ñ 5× BÑ BÈÐTX×TqÑTqÐ BÓ rØ"Ø%ð	 )8ó )�Ô%ð $(§?¡?ØÜ "§¡× 5Ñ 5× BÑ BÈÐTX×TqÑTqÐ BÓ rØ"Ø&ð	 $3ó $�Õ ô !Ð#CÀDÇOÁOÐCTÐTUÐ!VÓWÐWô &Ð(DÀT×E[ÑE[ÐD\Ð\qÐ&rÓsÐsä˜Ð0Ô1´g¸dÀMÔ6RÜÐJÓKÐKÜ�]‰]˜4Ÿ?™?×/Ñ/Ó0ñ 	(Ø�O‰O×!Ñ! $Ô'÷	(ä�]‰]˜4Ÿ8™8Ÿ=™=Ó)ñ 	!Ø�H‰H�N‰N˜4Ô ÷	!ä�]‰]˜4×/Ñ/×4Ñ4Ó5ñ 	-Ø× Ñ ×&Ñ& tÔ,÷	-ä�]‰]˜4×8Ñ8×=Ñ=Ó>ñ 	6Ø×)Ñ)×/Ñ/°Ô5÷	6ä‰‰�kÕ"÷	(ñ 	(ú÷	!ð 	!ú÷	-ð 	-ú÷	6ð 	6ús0   Î)SÏ6SÑSÒS)ÓSÓSÓS&Ó)S2c	                óf  — |€t        d«      ‚|€t        d«      ‚|�t        d«      ‚|}	| j                  |«      }| j                  ||||¬«      \  }}
}t        j
                  j                  || j                  ¬«      }t	        j                  |dk(  |j                  ¬«      }t	        j                  |d«      }t	        j                  t	        j                  |t	        j                  |«      «      d	k(  t	        j                  |«      |«      }|	| j                  | j                   «      |z  z   }|}	| j#                  |«      }| j%                  |«      }t        j
                  j                  || j                  ¬«      }|	| j'                  | j(                  «      |z  z   }|f}|r||
fz  }|r||fz  }|S )
a«  
        Args:
            hidden_states (`tf.Tensor`): input to the layer of shape `(batch, seq_len, embed_dim)`
            attention_mask (`tf.Tensor`, *optional*): attention mask of size
                `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values.
            output_attentions (`bool`, *optional*):
                Whether or not to return the attentions tensors of all attention layers. See `attentions` under
                returned tensors for more detail.
            use_cache (`bool`, *optional*):
                If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding
                (see `past_key_values`).
            past_key_value (`Tuple(tf.Tensor)`, *optional*): cached past key and value projection states
            no_images (`bool`, *optional*, defaults to `False`): If `True` the vision part is ignored
        zt`image_hidden_states` is required for Idefics cross attention module which are visual features to be conditioned on.z‹`cross_attention_gate` is required for Idefics cross attention module to zero-out the cross-attention hidden_states attending to no images.zMPast key value states are not implemented for Idefics cross attention module.)r"   rA  r9   rC  r]  r   rR   r3   r   )r+  r}  rX  ri  r:   r©   r)  r±   rX   rS   rÈ   rZ   rÊ   r>   Ú
zeros_likerl  rs  rY  rV  rm  rt  )r€   r"   r9   r$   r7   Úcross_attention_gaterC  rU   rB  r`  ra  rb  rÏ   rM   s                 r,   r„   z&TFIdeficsGatedCrossAttentionLayer.callÞ  s¹  € ð2 Ð&Üð#óð ð
  Ð'Üð ^óð ð Ð%Ü%Ð&uÓvÐvà ˆà×,Ñ,¨]Ó;ˆð ?C¿o¹oØ'Ø0Ø/Ø/ð	 ?Nó ?
Ñ;ˆÐ(Ð*;ô Ÿ™Ÿ™ m¸$¿+¹+˜ÓFˆÜ�w‰wÐ+¨qÑ0¸×8KÑ8KÔLˆä�~‰~˜d BÓ'ˆÜŸ™Ü�O‰O˜D¤"§(¡(¨=Ó"9Ó:¸aÑ?ÄÇÁÈ}ÓA]Ð_ló
ˆð
 ! 4×#6Ñ#6°t×7LÑ7LÓ#MÐP]Ñ#]Ñ]ˆà ˆØ×5Ñ5°mÓDˆØŸ™ Ó/ˆÜŸ™Ÿ™ m¸$¿+¹+˜ÓFˆØ  4§>¡>°$×2BÑ2BÓ#CÀmÑ#SÑSˆà Ð"ˆáØÐ)Ð+Ñ+ˆGáØÐ)Ð+Ñ+ˆGàˆr+   rd  )NNNNFFN)r"   rÀ   r9   r   r$   r   r7   r   r€  r   rC  r‘   rU   r‘   rB  r!   r’   re  rí   r—   s   @r,   rg  rg  m  s�   ø„ õHô>O#ðh /3Ø37Ø48Ø48Ø,1Ø$)Ø59ðKà ðKð ,ðKð 1ð	Kð
 2ðKð 2ðKð *ðKð "ðKð 3ðKð 
A÷Kr+   rg  ag  
    This model inherits from [`TFPreTrainedModel`]. Check the superclass documentation for the generic methods the
    library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
    etc.)

    This model is also a TensorFlow [tf.keras.layers.Layer](https://www.tensorflow.org/api_docs/python/tf/keras/layers/Layer) subclass.
    Use it as a regular TensorFlow Layer and refer to the TensorFlow documentation for all matter related to general usage
    and behavior.

    Parameters:
        config ([`IdeficsConfig`]):
            Model configuration class with all the parameters of the model. Initializing with a config file does not
            load the weights associated with the model, only the configuration. Check out the
            [`~TFPreTrainedModel.from_pretrained`] method to load the model weights.
zSThe bare LLaMA Model outputting raw hidden-states without any specific head on top.c                  ó    — e Zd ZeZdZdZddgZy)ÚTFIdeficsPreTrainedModelrn   TrS  rg  N)r%   r&   r'   r   Úconfig_classÚbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesr*   r+   r,   r‚  r‚  =  s!   „ ð
 !€LØÐØ&*Ð#Ø0Ð2UÐVÑr+   r‚  a   
    Args:
        input_ids (`tf.Tensor` of shape `(batch_size, sequence_length)`):
            Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide
            it.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are input IDs?](../glossary#input-ids)
        attention_mask (`tf.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
            Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:

            - 1 for tokens that are **not masked**,
            - 0 for tokens that are **masked**.

            [What are attention masks?](../glossary#attention-mask)

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            If `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see
            `past_key_values`).

            If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`]
            and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more
            information on the default strategy.

            - 1 indicates the head is **not masked**,
            - 0 indicates the head is **masked**.
        position_ids (`tf.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
            Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
            config.n_positions - 1]`. [What are position IDs?](../glossary#position-ids)
        past_key_values (`tuple(tuple(tf.Tensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
            Tuple of `tuple(tf.Tensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape
            `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape
            `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`.

            Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention
            blocks) that can be used (see `past_key_values` input) to speed up sequential decoding.

            If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that
            don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all
            `decoder_input_ids` of shape `(batch_size, sequence_length)`.
        inputs_embeds (`tf.Tensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
            Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
            is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
            model's internal embedding lookup matrix.
        use_cache (`bool`, *optional*):
            If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see
            `past_key_values`).
        output_attentions (`bool`, *optional*):
            Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
            tensors for more detail.
        output_hidden_states (`bool`, *optional*):
            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
            more detail.
        return_dict (`bool`, *optional*):
            Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
c                  óæ   ‡ — e Zd ZdZeZddˆ fd„Zdd„Zg fd„Zg fd„Z	d„ Z
d„ Zd„ Ze ee«      	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dd	„«       «       Zdd
„Zˆ xZS )ÚTFIdeficsMainLayerzŸ
    Transformer decoder consisting of `config.num_hidden_layers` layers. Each layer is a [`IdeficsDecoderLayer`]

    Args:
        config: IdeficsConfig
    c           	     ó  •— t        ‰| �  di |¤Ž || _        |j                  | _        |j
                  | _        t        |j
                  |j                  |j                  |j                  d¬«      | _
        |j                  j                  | _        |j                  | _        t        |j                  d¬«      | _        |j                  r_|j                   }t#        ||j                  j$                  |j&                  |j(                  |j*                  |j,                  d¬«      | _        t1        |j2                  «      D �cg c]  }t5        |d|› �¬«      ‘Œ c}| _        |j8                  | _        |j2                  | j8                  z  }t1        |«      D �cg c]  }t;        |d|› �¬«      ‘Œ c}| _        d| _        tA        |j                  |jB                  d	¬
«      | _"        d| _        | jG                  |«       y c c}w c c}w )NÚembed_tokens)r}   r~   r�   r   rz   Úvision_modelr#  Úperceiver_resamplerzlayers.zgated_cross_attn_layers.FÚnormr%  r*   )$r{   r|   r±   Úpad_token_idÚpadding_idxÚ
vocab_sizeru   Úadditional_vocab_sizerÞ   Úfreeze_text_layersrŠ  rO  Ú
image_sizer   r‹  Úuse_resamplerÚperceiver_configr   rN  Úresampler_depthÚresampler_n_headsÚresampler_head_dimÚresampler_n_latentsrŒ  r=   Únum_hidden_layersrS  Údecoder_layersÚcross_layer_intervalrg  Úgated_cross_attn_layersÚgradient_checkpointingrÜ   r.  r�  Úfreeze_relevant_params)r€   r±   Úadd_pooling_yearr[   r•  ÚiÚnum_cross_layersr‚   s          €r,   r|   zTFIdeficsMainLayer.__init__•  sÖ  ø€ Ü‰ÑÑ"˜6Ò"ØˆŒØ!×.Ñ.ˆÔØ ×+Ñ+ˆŒä7Ø!×,Ñ,Ø&,×&BÑ&BØ ×,Ñ,Ø#×6Ñ6Øô
ˆÔð !×.Ñ.×9Ñ9ˆŒØ#×1Ñ1ˆÔÜ6°v×7KÑ7KÐR`ÔaˆÔð ×ÒØ%×6Ñ6ÐÜ'BØØ×$Ñ$×.Ñ.Ø ×0Ñ0Ø ×2Ñ2Ø ×3Ñ3Ø ×4Ñ4Ø*ô(ˆDÔ$ô HMÈV×MeÑMeÓGfö
ØBCÔ! &°¸¸¨}Ö=ò
ˆÔð %+×$?Ñ$?ˆÔ!Ø!×3Ñ3°t×7PÑ7PÑPÐô Ð+Ó,ö(
àô .¨fÐ=UÐVWÐUXÐ;YÖZò(
ˆÔ$ð ',ˆÔ#ä$ V×%7Ñ%7¸V×=PÑ=PÐW]Ô^ˆŒ	à&+ˆÔ#Ø×#Ñ# FÕ+ùò
ùò(
s   ÅG=ÆHc                óÊ   — |€| j                   }|j                  r| j                  |j                  «       |j                  r"t	        | j
                  |j                  ¬«       y y ©N)ro   )r±   r’  Úfreeze_text_module_exceptionsÚfreeze_vision_layersrs   r‹  Úfreeze_vision_module_exceptions)r€   r±   s     r,   rŸ  z)TFIdeficsMainLayer.freeze_relevant_paramsÅ  sQ   € Øˆ>Ø—[‘[ˆFà×$Ò$Ø×#Ñ# F×$HÑ$HÔIà×&Ò&Ü˜×*Ñ*¸f×>dÑ>dÖeð 'r+   c                óX   — | j                   | j                  fD ]  }t        ||¬«       Œ y r¤  )r›  r�  rs   )r€   ro   Úmodules      r,   r’  z%TFIdeficsMainLayer.freeze_text_layersÏ  s-   € Ø×*Ñ*¨D¯I©IÐ6ò 	FˆFÜ˜Ð3DÖEñ	Fr+   c                ó2   — t        | j                  |¬«       y r¤  )rs   r‹  )r€   ro   s     r,   r¦  z'TFIdeficsMainLayer.freeze_vision_layersÓ  s   € Ü�T×&Ñ&Ð:KÖLr+   c                ó   — | j                   S rc   ©rŠ  r�   s    r,   Úget_input_embeddingsz'TFIdeficsMainLayer.get_input_embeddingsÖ  s   € Ø× Ñ Ð r+   c                ó   — || _         y rc   r¬  ©r€   rÕ   s     r,   Úset_input_embeddingsz'TFIdeficsMainLayer.set_input_embeddingsÙ  s
   € Ø!ˆÕr+   c                ó†   — d }t        ||j                  |¬«      }|�$t        ||j                  |d   ¬«      }|€|n||z   }|S )N)rÌ   r3   )rÎ   )rÑ   rS   rÚ   )r€   r9   r½   Úinputs_embedsrÌ   Úcombined_attention_maskÚexpanded_attn_masks          r,   Ú_prepare_decoder_attention_maskz2TFIdeficsMainLayer._prepare_decoder_attention_maskÜ  sg   € ð #'Ðä"3ØØ×ÑØ#9ô#
Ðð Ð%ä!-¨n¸m×>QÑ>QÐ[fÐgiÑ[jÔ!kÐà&=Ð&EÑ"ÐK]Ð`wÑKwð $ð 'Ð&r+   c                ó  — |�|n| j                   j                  }|�|n| j                   j                  }|
�|
n| j                   j                  }
|�|n| j                   j                  }|�|�t        d«      ‚|�t        |«      \  }}n|�t        |«      \  }}}nt        d«      ‚|}d}|�t        |d   d   «      d   }||z   }|�d|€bt        j                  j                  t        j                  |t        j                  ¬«      d¬«      dz
  }t        j                  |dk(  d|«      }nA|€?t        j                  |||z   t        j                  ¬«      }t        j                  |d«      }d	}t        t!        |d u «      t!        |d u «      t!        |d u «      f«      dk7  rt        d
«      ‚|��t        j"                  t        j                  |t        j                  ¬«      «      dk(  }t        j                  || j$                  ¬«      }t'        |j(                  «      dk(  rt        |«      d   }t        |«      d   }nRt'        |j(                  «      dk(  r:t        |«      d d \  }}t        j*                  |||z  g|j(                  dd  ¢«      }| j-                  ||¬«      j.                  }nO|�Mt        |«      \  }}}}t        j                  || j$                  ¬«      }t        j*                  |||z  ||f«      }| j                   j0                  r8|€#| j3                  «      }t        |«      dd \  }}nt        |«      \  }}}}|}n|€t        «      dd \  }}nt        d«      ‚t        j*                  |||z  |f«      }|�Bt'        |j(                  «      dk(  r*|	€(t        j4                  ||dft        j                  ¬«      }	t        |	«      d   }t        j                  |	d«      }	t        j6                  |	|¬«      }	t        j*                  |	||||z  f«      }	|�Ft        |«      \  }}}||f}|	€%t        j8                  |t        j                  ¬«      }	t;        |	«      }	nd }	t        j<                  t        j                  t        j>                  |	dk(  d¬«      | j$                  ¬«      d¬«      }|€| jA                  |«      }|€'t        j8                  ||ft        jB                  ¬«      }| jE                  |||f||«      }|}| jF                  r|r|
rtH        jK                  d«       d	}
|rdnd } |rdnd }!|
rdnd }"tM        | jN                  «      D ]Â  \  }#}$|r| |fz  } |�||#   nd }%d„ }&| jF                  rT|rRd }%|
rtH        jK                  d«       d	}
t        jP                  |&|$||||%||	||
||#| jR                  | jT                  «      }'n) |&|$||||%||	|||
|#| jR                  | jT                  ¬«      }'|'d   }|
r|"|'|rdnd   fz  }"|sŒº|!|'d   fz  }!ŒÄ | jW                  |«      }|r| |fz  } |
r|"nd }(t        j*                  |||||f«      }|stY        d„ ||(| |!|fD «       «      S t[        ||(| |!|¬«      S )NzTYou cannot specify both decoder_input_ids and decoder_inputs_embeds at the same timezEYou have to specify either decoder_input_ids or decoder_inputs_embedsr   rñ   rR   r3   rI   r   Fz_Exactly 1 of pixel_values, image_encoder_embeddings or perceiver_embeddings has to be not-None.é   é   )r4   rT   r	   zBIf `perceiver_embeddings` are passed, use_resampler should be True)ÚrepeatszZ`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...r*   c           
     ól   — |
|z  dk(  r||
|z     } ||||||||	d ¬«      }|d   } | ||||||	¬«      }|S )Nr   )r9   r$   r7   r€  rC  rU   rB  )r9   rQ   rB  rC  rU   r*   )Ú
main_blockr"   r9   rQ   rB  r$   r7   r€  rC  rU   Ú	layer_idxrœ  r�  ÚxblockrM   Úlayer_outputss                   r,   Úvblockz'TFIdeficsMainLayer.call.<locals>.vblock‹  ss   € ð  Ð3Ñ3°qÒ8Ø4°YÐBVÑ5VÑW�FÙ$Ø%Ø'5Ø,?Ø-AØ-AØ*;Ø"+Ø'+ô	�Gð %,¨A¡J�Má *Ø!Ø#1Ø!-Ø#1Ø&7Ø'ô!�ð %Ð$r+   )r9   rQ   rB  r$   r7   r€  rC  rU   r¼  rœ  r�  c              3  ó$   K  — | ]  }|�|–— Œ
 y ­wrc   r*   )re   Úvs     r,   rh   z*TFIdeficsMainLayer.call.<locals>.<genexpr>ð  s   è ø€ ò àØ�=ô ñùs   ‚)r   r    r"   r#   r$   ).r±   rC  Úoutput_hidden_statesrU   Úuse_return_dictr+  r   r:   rV   rW   rX   Úint32rZ   r=   rÈ   Úsumr¾   Ú
reduce_sumrS   Úlenr>   r;   r‹  r   r”  rŒ  rˆ   r<   râ   r   ÚsqueezeÚ
reduce_anyrŠ  r¿   rµ  rž  r?  r@  Ú	enumerater›  Úrecompute_gradrœ  r�  r�  Útupler   ))r€   rA   r9   rQ   r    r²  r4   r5   r6   r7   rU   rC  rÂ  rT   Úreturn_dictr_  Ú
batch_sizeÚ
seq_lengthrE  Úseq_length_with_pastrÌ   Ú	no_imagesÚ
num_imagesr$   Úimage_seq_lenÚimage_hidden_sizeÚtext_seq_lenÚimage_batch_sizeÚimage_sequence_lengthÚimage_hidden_shaper€  r"   Úall_hidden_statesÚall_self_attnsÚnext_decoder_cacheÚidxÚdecoder_layerrB  r¿  r¾  Ú
next_caches)                                            r,   r„   zTFIdeficsMainLayer.callð  s;  € ð( 2CÐ1NÑ-ÐTX×T_ÑT_×TqÑTqÐà$8Ð$DÑ È$Ï+É+×JjÑJjð 	ð "+Ð!6‘I¸D¿K¹K×<QÑ<Qˆ	à%0Ð%<‘kÀ$Ç+Á+×B]ÑB]ˆð Ð  ]Ð%>ÜÐsÓtÐtØÐ"Ü%/°	Ó%:Ñ"ˆJ™
ØÐ&Ü(2°=Ó(AÑ%ˆJ˜
¡AäÐdÓeÐeà)ÐØ!"ÐàÐ&Ü%/°ÀÑ0BÀ1Ñ0EÓ%FÀqÑ%IÐ"Ø#7Ð:PÑ#PÐ àÐ%¨,Ð*>äŸ7™7Ÿ>™>¬"¯'©'°.ÌÏÉÔ*QÐXZ˜>Ó[Ð^_Ñ_ˆLÜŸ8™8 N°aÑ$7¸¸LÓI‰LØÐ!ÜŸ8™8Ð$:¸JÐI_Ñ<_Ôgi×goÑgoÔpˆLÜŸ>™>¨,¸Ó:ˆLàˆ	ä”�\ TÐ)Ó*¬CÐ0HÈDÐ0PÓ,QÔSVÐWkÐosÐWsÓStÐuÓvØòô Øqóð ð Ñ%ÜŸ™¤b§g¡g¨lÄ"Ç(Á(Ô&KÓLÐPQÑQˆIÜŸ7™7 <°t·z±zÔBˆLô �<×%Ñ%Ó&¨!Ò+Ü'¨Ó5°aÑ8�
Ü'¨Ó5°aÑ8‘
ä�\×'Ñ'Ó(¨AÒ-Ü)3°LÓ)AÀ"À1Ð)EÑ&�
˜JÜ!Ÿz™z¨,¸ÀjÑ9PÐ8jÐS_×SeÑSeÐfgÐfhÐSiÐ8jÓk�ð #'×"3Ñ"3Ø)ÐD\ð #4ó #çÑñ  ð &Ð1ÜGQÐRjÓGkÑDˆJ˜
 MÐ3DÜ"$§'¡'Ð*BÈ$Ï*É*Ô"UÐÜ"$§*¡*Ø# j°:Ñ&=¸}ÐN_Ð%`ó#Ðð �;‰;×$Ò$Ø#Ð+Ø'+×'?Ñ'?Ð@SÓ'TÐ$Ü3=Ð>RÓ3SÐTUÐVWÐ3XÑ0�Ñ0äKUÐVjÓKkÑH�
˜J¨Ð7HØ"6ÑØ!Ð)Ü/9Ð:MÓ/NÈqÐQRÐ/SÑ,ˆMÑ,äÐaÓbÐbä Ÿj™jØ *¨j¸=Ñ.HÐJ[Ð!\ó
Ðð Ð#¬¨L×,>Ñ,>Ó(?À1Ò(DÐI]ÐIeÜ#%§8¡8¨Z¸ÀQÐ,GÌrÏxÉxÔ#XÐ ä!Ð"6Ó7¸Ñ:ˆÜ!Ÿ~™~Ð.BÀBÓGÐÜ!Ÿy™yÐ)=À}ÔUÐÜ!Ÿz™zÐ*>ÀÈ\Ð[eÐhuÑ[uÐ@vÓwÐàÐ*Ü9CÐDWÓ9XÑ6ÐÐ3°QØ"2Ð4IÐ!JÐØ#Ð+Ü')§w¡wÐ/AÌÏÉÔ'RÐ$Ü#8Ð9MÓ#NÑ à#'Ð ä!Ÿz™zÜ�G‰G”B—M‘MÐ"6¸!Ñ";À"ÔEÈTÏZÉZÔXÐ_`ô 
Ðð Ð Ø ×-Ñ-¨iÓ8ˆMàÐ!ÜŸW™W jÐ2FÐ%GÌrÏwÉwÔWˆNØ×=Ñ=Ø˜Z¨Ð4°mÐE[ó
ˆð &ˆà×&Ò&©8ÙÜ×#Ñ#Øpôð "�	ñ #7™B¸DÐÙ0™°dˆÙ#,™R°$Ðä"+¨D×,?Ñ,?Ó"@ò ^	6ÑˆC�Ù#Ø! mÐ%5Ñ5Ð!à5DÐ5P˜_¨SÒ1ÐVZˆNò'%ðR ×*Ò*©xØ!%�ÙÜ×'Ñ'Øtôð !&�Iä "× 1Ñ 1ØØ!Ø!Ø"Ø Ø"Ø'Ø(Ø%ØØØØ×-Ñ-Ø×0Ñ0ó!‘ñ" !'Ø!Ø!Ø#1Ø!-Ø#1Ø(;Ø)=Ø)=Ø&7Ø'Ø!Ø)-×)BÑ)BØ,0×,HÑ,Hô!�ð  *¨!Ñ,ˆMáØ" }Ñ:K±QÐQRÑ'SÐ&UÑUÐ"â Ø =°Ñ#3Ð"5Ñ5‘ð}^	6ð@ Ÿ	™	 -Ó0ˆñ  Ø -Ð!1Ñ1Ðá+4Ñ'¸$ˆ
Ü Ÿj™jØ *¨j¸-ÐIZÐ![ó
Ðñ Üñ à'¨Ð5FÈÐXkÐlôó ð ô
 0Ø+Ø&Ø+Ø%Ø 3ô
ð 	
r+   c                óð  — | j                   ry d| _         t        | dd «      �Mt        j                  | j                  j
                  «      5  | j                  j                  d «       d d d «       t        | dd «      �Mt        j                  | j                  j
                  «      5  | j                  j                  d «       d d d «       t        | dd «      �Mt        j                  | j                  j
                  «      5  | j                  j                  d «       d d d «       t        | dd «      �Mt        j                  | j                  j
                  «      5  | j                  j                  d «       d d d «       t        | dd «      �J| j                  D ];  }t        j                  |j
                  «      5  |j                  d «       d d d «       Œ= t        | dd «      �K| j                  D ];  }t        j                  |j
                  «      5  |j                  d «       d d d «       Œ= y y # 1 sw Y   �ŒÈxY w# 1 sw Y   �Œ{xY w# 1 sw Y   �Œ.xY w# 1 sw Y   ŒàxY w# 1 sw Y   ŒÓxY w# 1 sw Y   ŒˆxY w)NTrŠ  r‹  r�  rŒ  r›  r�  )r·   r¹   r:   rº   rŠ  rz   r»   r‹  r�  rŒ  r›  r�  )r€   r½   rg   s      r,   r»   zTFIdeficsMainLayer.buildý  s
  € Ø�:Š:ØØˆŒ
Ü�4˜¨Ó.Ð:Ü—‘˜t×0Ñ0×5Ñ5Ó6ñ .Ø×!Ñ!×'Ñ'¨Ô-÷.ä�4˜¨Ó.Ð:Ü—‘˜t×0Ñ0×5Ñ5Ó6ñ .Ø×!Ñ!×'Ñ'¨Ô-÷.ä�4˜ Ó&Ð2Ü—‘˜tŸy™yŸ~™~Ó.ñ &Ø—	‘	—‘ Ô%÷&ä�4Ð.°Ó5ÐAÜ—‘˜t×7Ñ7×<Ñ<Ó=ñ 5Ø×(Ñ(×.Ñ.¨tÔ4÷5ä�4Ð)¨4Ó0Ð<Ø×,Ñ,ò &�Ü—]‘] 5§:¡:Ó.ñ &Ø—K‘K Ô%÷&ð &ð&ô �4Ð2°DÓ9ÐEØ×5Ñ5ò &�Ü—]‘] 5§:¡:Ó.ñ &Ø—K‘K Ô%÷&ð &ñ&ð F÷.ñ .ú÷.ñ .ú÷&ñ &ú÷5ð 5ú÷&ð &ú÷&ð &úsH   ÁH-Â%H:Ã?IÅIÆ8I ÈI,È-H7È:IÉIÉIÉ I)	É,I5	)T)r±   r   r   r¿   rc   ©NNNNNNNNNNNNFNN© rA   úTFModelInputType | Noner9   r   rQ   r   r    r1   r²  r   r4   r   r5   r   r6   r   r7   r   rU   r‘   rC  r‘   rÂ  r‘   rT   r‘   rÍ  r‘   r_  r‘   r’   z9Union[TFIdeficsBaseModelOutputWithPast, Tuple[tf.Tensor]])r%   r&   r'   r(   r   rƒ  r|   rŸ  r’  r¦  r­  r°  rµ  r   r   ÚLLAMA_INPUTS_DOCSTRINGr„   r»   r–   r—   s   @r,   rˆ  rˆ  †  sI  ø„ ñð !€Lö.,ó`fð 46ó Fð 68ó Mò!ò"ò'ð( Ù*Ð+AÓBð .2Ø.2Ø,0Ø59Ø-1Ø,0Ø8<Ø48Ø48Ø$(Ø,0Ø/3Ø38Ø&*Ø#'ð!I
à*ðI
ð ,ðI
ð *ð	I
ð
 3ðI
ð +ðI
ð *ðI
ð #6ðI
ð 2ðI
ð 2ðI
ð "ðI
ð *ðI
ð -ðI
ð #1ðI
ð $ðI
ð  !ð!I
ð" 
Cò#I
ó Có ðI
÷V&r+   rˆ  c                  óŽ   ‡ — e Zd Zdˆ fd„Z	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dd„Zdd„Zˆ xZS )ÚTFIdeficsModelc                óP   •— t        ‰| �  |g|¢­i |¤Ž t        |d¬«      | _        y )Nrn   r#  )r{   r|   rˆ  rn   )r€   r±   r«   r[   r‚   s       €r,   r|   zTFIdeficsModel.__init__  s(   ø€ Ü‰Ñ˜Ð3 &Ò3¨FÒ3ä'¨°WÔ=ˆ�
r+   c                óF   — | j                  |||||||||	|
|||||¬«      }|S )N©rA   r9   rQ   r    r²  r4   r5   r6   r7   rU   rC  rÂ  rT   rÍ  r_  ©rn   )r€   rA   r9   rQ   r    r²  r4   r5   r6   r7   rU   rC  rÂ  rT   rÍ  r_  rM   s                    r,   r„   zTFIdeficsModel.call  sK   € ð$ —*‘*ØØ)Ø%Ø+Ø'Ø%Ø%=Ø!5Ø!5ØØ/Ø!5Ø%=Ø#Øð ó 
ˆð" ˆr+   c                óú   — | j                   ry d| _         t        | dd «      �Nt        j                  | j                  j
                  «      5  | j                  j                  d «       d d d «       y y # 1 sw Y   y xY w)NTrn   )r·   r¹   r:   rº   rn   rz   r»   r¼   s     r,   r»   zTFIdeficsModel.buildB  sg   € Ø�:Š:ØØˆŒ
Ü�4˜ $Ó'Ð3Ü—‘˜tŸz™zŸ™Ó/ñ 'Ø—
‘
× Ñ  Ô&÷'ð 'ð 4÷'ð 'ús   ÁA1Á1A:rd  rà  rá  rc   r  r—   s   @r,   rå  rå    sá   ø„ õ>ð .2Ø.2Ø,0Ø59Ø-1Ø,0Ø8<Ø48Ø48Ø$(Ø,0Ø/3Ø38Ø&*Ø#'ð!#à*ð#ð ,ð#ð *ð	#ð
 3ð#ð +ð#ð *ð#ð #6ð#ð 2ð#ð 2ð#ð "ð#ð *ð#ð -ð#ð #1ð#ð $ð#ð  !ð!#ð" 
Có##÷J'r+   rå  c                  ó>  ‡ — e Zd ZdgZddgZeZdˆ fd„	Zd„ Zd„ Z	d„ Z
d„ Zd„ Zd	„ Zd
„ Ze ee«       eee¬«      	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dd„«       «       «       Zdd„Zed„ «       Zed„ «       Zed„ «       Zdd„Zˆ xZS )ÚTFIdeficsForVisionText2Textzlm_head.weightzmodel.embed_tokens.weightc                óÆ   •— t        ‰| �  |fi |¤Ž t        |d¬«      | _        t	        |j
                  |j                  |j                  d|j                  d¬«      | _	        y )Nrn   r#  FÚlm_head)r¡   r   rz   )
r{   r|   rˆ  rn   r™   rÞ   r�  r‘  Úfreeze_lm_headrî  )r€   r±   r‹  r[   r‚   s       €r,   r|   z$TFIdeficsForVisionText2Text.__init__P  sY   ø€ Ü‰Ñ˜Ñ* 6Ò*Ü'¨°WÔ=ˆŒ
Ü/Ø×ÑØ×ÑØ×(Ñ(ØØ#×2Ñ2Øô
ˆ�r+   c                ó.   — | j                   j                  S rc   ©rn   rŠ  r�   s    r,   r­  z0TFIdeficsForVisionText2Text.get_input_embeddings\  s   € Ø�z‰z×&Ñ&Ð&r+   c                ó&   — || j                   _        y rc   rñ  r¯  s     r,   r°  z0TFIdeficsForVisionText2Text.set_input_embeddings_  s   € Ø"'ˆ�
‰
Õr+   c                ó   — | j                   S rc   ©rî  r�   s    r,   Úget_output_embeddingsz1TFIdeficsForVisionText2Text.get_output_embeddingsb  s   € Ø�|‰|Ðr+   c                ó   — || _         y rc   rô  )r€   Únew_embeddingss     r,   Úset_output_embeddingsz1TFIdeficsForVisionText2Text.set_output_embeddingse  s	   € Ø%ˆ�r+   c                ó   — || _         y rc   ré  )r€   Údecoders     r,   Úset_decoderz'TFIdeficsForVisionText2Text.set_decoderh  s	   € Øˆ�
r+   c                ó   — | j                   S rc   ré  r�   s    r,   Úget_decoderz'TFIdeficsForVisionText2Text.get_decoderk  s   € Ø�z‰zÐr+   c                óÞ  — | j                  «       }| j                  «       }t        | j                  dd«      r`|j                  |_        |j
                  dkD  r@|j                  |j
                  k(  sJ ‚|j                  j                  |j                  _        t        |d«      rJt        |d«      r=|j                  |_        t        |d«      rt        |d«      r|j
                  |_        yyyyy)	z©
        Overwrite `transformers.modeling_utils.PreTrainedModel.tie_weights` to handle the case of
        IdeficsDecoupledLinear and IdeficsDecoupledEmbedding.
        Útie_word_embeddingsTr   r    r}   rž   r~   N)rõ  r­  r¹   r±   r¨   r~   rž   ry   r›   rk   r}   r    )r€   Úoutput_embeddingsÚinput_embeddingss      r,   Útie_weightsz'TFIdeficsForVisionText2Text.tie_weightsn  sé   € ð
 !×6Ñ6Ó8ÐØ×4Ñ4Ó6Ðä�4—;‘;Ð 5°tÔ<Ø'7×'>Ñ'>ÐÔ$Ø×9Ñ9¸AÒ=Ø(×@Ñ@ÐDT×DnÑDnÒnÐnÐnØ9I×9^Ñ9^×9eÑ9eÐ!×/Ñ/Ô6äÐ$ nÔ5¼'ÐBRÐTdÔ:eØ-=×-LÑ-LÐÔ*ÜÐ(Ð*CÔDÌØ Ð"=ôJð =M×<fÑ<fÐ!Õ9ðJÐDð ;fÐ5r+   )Úoutput_typerƒ  c                óÈ  — |�|n| j                   j                  }|�|n| j                   j                  }|�|n| j                   j                  }| j	                  |||||||||	||||||¬«      }|d   }| j                  |«      }d}|
��|�+|ddd…f   }|ddd…dd…f   |dk7     }|
ddd…f   |dk7     }n|ddd…dd…f   }|
ddd…f   }| j                  t        j                  |dg«      t        j                  |d|j                  d   g«      ¬«      }|s|f|dd z   }|�|f|z   S |S t        |||j                  |j                  |j                  |j                  ¬«      S )	a   
            labels (`tf.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
                Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
                config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
                (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

        Returns:

        Example:

        ```python
        >> from transformers import AutoTokenizer, TFIdeficsForVisionText2Text

        >> model = TFIdeficsForVisionText2Text.from_pretrained("HuggingFaceM4/idefics-9b")
        >> tokenizer = AutoTokenizer.from_pretrained("HuggingFaceM4/idefics-9b")

        >> prompt = "Hey, are you consciours? Can you talk to me?"
        >> inputs = tokenizer(prompt, return_tensors="tf")

        >> # Generate
        >> generate_ids = model.generate(inputs.input_ids, max_length=30)
        >> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "Hey, are you consciours? Can you talk to me?\nI'm not consciours, but I can talk to you."
        ```Nrè  r   .r   r3   )Úlabelsr0   )r/   r0   r    r"   r#   r$   )r±   rC  rÂ  rÃ  rn   rî  Úhf_compute_lossr:   r;   r>   r.   r    r"   r#   r$   )r€   rA   r9   rQ   r    r²  r4   r5   r6   r7   r  rU   rC  rÂ  rT   rÍ  r_  rM   r"   r0   r/   Úshift_attention_maskÚshift_logitsÚshift_labelsr¬   s                            r,   r„   z TFIdeficsForVisionText2Text.callƒ  sÖ  € ð^ 2CÐ1NÑ-ÐTX×T_ÑT_×TqÑTqÐà$8Ð$DÑ È$Ï+É+×JjÑJjð 	ð &1Ð%<‘kÀ$Ç+Á+×B]ÑB]ˆð —*‘*ØØ)Ø%Ø+Ø'Ø%Ø%=Ø!5Ø!5ØØ/Ø!5Ø%=Ø#Øð ó 
ˆð$   ™
ˆØ—‘˜mÓ,ˆàˆØÐàÐ)Ø'5°c¸1¹2°gÑ'>Ð$Ø% c¨3¨B¨3² kÑ2Ð3GÈ1Ñ3LÑM�Ø% c¨1©2 g™Ð/CÀqÑ/HÑI‘à% c¨3¨B¨3² kÑ2�Ø% c¨1©2 g™�à×'Ñ'Ü—z‘z ,°°Ó5¼b¿j¹jÈÐXZÐ\h×\nÑ\nÐoqÑ\rÐWsÓ>tð (ó ˆDñ Ø�Y ¨¨ Ñ,ˆFØ'+Ð'7�D�7˜VÑ#ÐC¸VÐCä.ØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)Ø '× ;Ñ ;ô
ð 	
r+   c                óÎ   — |j                  dd «      }|�&| j                  j                  r||d<   n||d<   d |d<   t        |fd|i|¤Ž}dg}|D ]  }|j                  |d «       Œ |S )Nr$   r6   r5   r4   Úpastr8   )Úpopr±   r”  r\   )r€   rA   r  r[   r$   r«   Úunwanted_kwargsÚkwargs           r,   r\   z9TFIdeficsForVisionText2Text.prepare_inputs_for_generationê  s…   € Ø$Ÿj™jÐ)>ÀÓEÐØÐ*Ø�{‰{×(Ò(Ø1D�Ð-Ò.à5H�Ð1Ñ2Ø%)ˆF�>Ñ"Ü.¨yÑN¸tÐNÀvÑNˆØ+Ð,ˆØ$ò 	$ˆEØ�J‰J�u˜dÕ#ð	$àˆr+   c                 ó   — t        | i |¤ŽS rc   )rG   )ÚargsrE   s     r,   Ú_expand_inputs_for_generationz9TFIdeficsForVisionText2Text._expand_inputs_for_generationø  s   € ô
 ,¨TÐB°\ÑBÐBr+   c                ó   — t        | |«      S rc   )rO   )rM   rE   rC   s      r,   Ú#_update_model_kwargs_for_generationz?TFIdeficsForVisionText2Text._update_model_kwargs_for_generationÿ  s   € ä1°'¸<ÓHÐHr+   c                óJ   ‡— d}| D ]  }|t        ˆfd„|D «       «      fz  }Œ |S )Nr*   c              3  óJ   •K  — | ]  }t        j                  |‰«      –— Œ y ­wrc   )r:   r?   )re   Ú
past_stateÚbeam_idxs     €r,   rh   z=TFIdeficsForVisionText2Text._reorder_cache.<locals>.<genexpr>  s   øè ø€ Ò$bÈ¤R§Y¡Y¨z¸8×%DÑ$bùs   ƒ #)rÌ  )r  r  Úreordered_pastÚ
layer_pasts    `  r,   Ú_reorder_cachez*TFIdeficsForVisionText2Text._reorder_cache  s8   ø€ àˆØò 	eˆJØœuÓ$bÐWaÔ$bÓbÐdÑd‰Nð	eàÐr+   c                óÆ  — | j                   ry d| _         t        | dd «      �Mt        j                  | j                  j
                  «      5  | j                  j                  d «       d d d «       t        | dd «      �Nt        j                  | j                  j
                  «      5  | j                  j                  d «       d d d «       y y # 1 sw Y   ŒexY w# 1 sw Y   y xY w)NTrn   rî  )r·   r¹   r:   rº   rn   rz   r»   rî  r¼   s     r,   r»   z!TFIdeficsForVisionText2Text.build
  s³   € Ø�:Š:ØØˆŒ
Ü�4˜ $Ó'Ð3Ü—‘˜tŸz™zŸ™Ó/ñ 'Ø—
‘
× Ñ  Ô&÷'ä�4˜ DÓ)Ð5Ü—‘˜tŸ|™|×0Ñ0Ó1ñ )Ø—‘×"Ñ" 4Ô(÷)ð )ð 6÷'ð 'ú÷)ð )ús   ÁCÂ%CÃCÃC rc   )NNNNNNNNNNNNNFNF) rA   râ  r9   r   rQ   r   r    r1   r²  r   r4   r   r5   r   r6   r   r7   r   r  r   rU   r‘   rC  r‘   rÂ  r‘   rT   r‘   rÍ  r‘   r’   z8Union[TFIdeficsCausalLMOutputWithPast, Tuple[tf.Tensor]])r%   r&   r'   Ú_keys_to_ignore_on_load_missingÚ_tied_weights_keysr   rƒ  r|   r­  r°  rõ  rø  rû  rý  r  r   r   rã  r   r.   Ú_CONFIG_FOR_DOCr„   r\   Ústaticmethodr  r  r  r»   r–   r—   s   @r,   rì  rì  K  s©  ø„ Ø'8Ð&9Ð#Ø5Ð7GÐHÐØ €Lõ

ò'ò(òò&òòògð* Ù*Ð+AÓBÙÐ+JÐYhÔið .2Ø.2Ø,0Ø59Ø-1Ø,0Ø8<Ø48Ø48Ø&*Ø$(Ø,0Ø/3Ø38Ø&*Øð#b
à*ðb
ð ,ðb
ð *ð	b
ð
 3ðb
ð +ðb
ð *ðb
ð #6ðb
ð 2ðb
ð 2ðb
ð $ðb
ð "ðb
ð *ðb
ð -ðb
ð #1ðb
ð  $ð!b
ð$ 
Bò%b
ó jó Có ðb
óHð ñCó ðCð ñIó ðIð ñó ð÷	)r+   rì  )rì  rå  r‚  )r   FNNrc   )r   )Gr(   Ú
__future__r   Údataclassesr   Útypingr   r   r   r   Ú
tensorflowr:   Ú r
   Úactivations_tfr   Úmodeling_tf_outputsr   Úmodeling_tf_utilsr   r   r   r   r   Útf_utilsr   r   Úutilsr   r   r   r   Úconfiguration_ideficsr   Úperceiver_tfr   Ú	vision_tfr   Ú
get_loggerr%   r?  r  r   r.   rG   rO   r\   rs   ri   ra   r`   ru   ÚLayerr™   rÑ   rÚ   rÜ   rï   r  r  r  r  rS  rg  ÚLLAMA_START_DOCSTRINGr‚  rã  rˆ  rå  rì  Ú__all__r*   r+   r,   ú<module>r1     s[  ðñ( å "å !ß /Ó /ã å !Ý /Ý .÷õ ÷ L÷ó õ 1Ý 5Ý 1ð 
ˆ×	Ñ	˜HÓ	%€à!€ð ô); {ó );ó ð);ðX ô&; kó &;ó ð&;ðV ØØØó$#òNó:$ðN +-ó ô$f
 "§(¡(§/¡/×";Ñ";ô f
ôRV;˜rŸx™xŸ™×4Ñ4ô V;óró6ô"+�r—x‘x—‘×,Ñ,ô +ô86˜Ÿ™Ÿ™×.Ñ.ô 6ò2)òô5�2—8‘8—?‘?×(Ñ(ô 5ôDf,˜Ÿ™Ÿ™×.Ñ.ô f,ôR_:˜BŸH™HŸO™O×1Ñ1ô _:ôD|¨¯©¯©×(=Ñ(=ô |ð~Ð ñ" ØYØóôWÐ0ó Wó	ðWð;Ð ñ| ØYØóð ôI&˜Ÿ™Ÿ™×.Ñ.ó I&ó ó	ð
I&ôX1'Ð-ô 1'ôhH)Ð"3Ð5Qô H)òV X�r+   