Ë
    S^(háá  ã                   óÚ  — d Z ddlZddlmZmZmZmZmZmZ ddl	Z	ddl	m
Z
mZ ddlmZmZ ddlmZ ddlmZ dd	lmZ dd
lmZmZmZmZ ddlmZ ddlmZmZmZm Z m!Z!m"Z" ddl#m$Z$  e!jJ                  e&«      Z'dZ(dZ)	 dZ*dZ+dZ,d„ Z-d3d„Z.dde	j^                  fd„Z0 G d„ de«      Z1d„ Z2d„ Z3d„ Z4d4d„Z5 G d„ dejl                  «      Z7 G d„ d ejl                  «      Z8 G d!„ d"ejl                  «      Z9 G d#„ d$ejl                  «      Z:d%„ Z; G d&„ d'ejl                  «      Z<d(„ Z=d)„ Z> ed*e*«       G d+„ d,e1«      «       Z? ed-e*«       G d.„ d/e1e«      «       Z@ G d0„ d1ej‚                  «      ZBg d2¢ZCy)5z`PyTorch Fairseq model, ported from https://github.com/pytorch/fairseq/tree/master/examples/wmt19é    N)ÚAnyÚDictÚListÚOptionalÚTupleÚUnion)ÚTensorÚnn)ÚCrossEntropyLossÚ	LayerNormé   )ÚACT2FN)ÚGenerationMixin)Úis_deepspeed_zero3_enabled)ÚBaseModelOutputÚ)BaseModelOutputWithPastAndCrossAttentionsÚSeq2SeqLMOutputÚSeq2SeqModelOutput)ÚPreTrainedModel)Úadd_code_sample_docstringsÚadd_end_docstringsÚadd_start_docstringsÚ%add_start_docstrings_to_model_forwardÚloggingÚreplace_return_docstringsé   )Ú
FSMTConfigzfacebook/wmt19-ru-enr   a?  

    This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
    library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
    etc.)

    This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
    Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
    and behavior.

    Parameters:
        config ([`FSMTConfig`]): Model configuration class with all the parameters of the model.
            Initializing with a config file does not load the weights associated with the model, only the
            configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.

uŠ  
    Translation example::

    ```python
    >>> from transformers import AutoTokenizer, FSMTForConditionalGeneration

    >>> mname = "facebook/wmt19-ru-en"
    >>> model = FSMTForConditionalGeneration.from_pretrained(mname)
    >>> tokenizer = AutoTokenizer.from_pretrained(mname)

    >>> src_text = "ÐœÐ°ÑˆÐ¸Ð½Ð½Ð¾Ðµ Ð¾Ð±ÑƒÑ‡ÐµÐ½Ð¸Ðµ - Ñ�Ñ‚Ð¾ Ð·Ð´Ð¾Ñ€Ð¾Ð²Ð¾, Ð½Ðµ Ñ‚Ð°Ðº Ð»Ð¸?"
    >>> input_ids = tokenizer(src_text, return_tensors="pt").input_ids
    >>> outputs = model.generate(input_ids, num_beams=5, num_return_sequences=3)
    >>> tokenizer.decode(outputs[0], skip_special_tokens=True)
    "Machine learning is great, isn't it?"
    ```

a¡  
    Args:
        input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
            Indices of input sequence tokens in the vocabulary.

            Indices can be obtained using [`FSTMTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are input IDs?](../glossary#input-ids)
        attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
            Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:

            - 1 for tokens that are **not masked**,
            - 0 for tokens that are **masked**.

            [What are attention masks?](../glossary#attention-mask)
        decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
            Indices of decoder input sequence tokens in the vocabulary.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are decoder input IDs?](../glossary#decoder-input-ids)

            FSMT uses the `eos_token_id` as the starting token for `decoder_input_ids` generation. If `past_key_values`
            is used, optionally only the last `decoder_input_ids` have to be input (see `past_key_values`).
        decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
            Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also
            be used by default.
        head_mask (`torch.Tensor` of shape `(encoder_layers, encoder_attention_heads)`, *optional*):
            Mask to nullify selected heads of the attention modules in the encoder. Mask values selected in `[0, 1]`:

            - 1 indicates the head is **not masked**,
            - 0 indicates the head is **masked**.

        decoder_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*):
            Mask to nullify selected heads of the attention modules in the decoder. Mask values selected in `[0, 1]`:

            - 1 indicates the head is **not masked**,
            - 0 indicates the head is **masked**.

        cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*):
            Mask to nullify selected heads of the cross-attention modules in the decoder. Mask values selected in `[0,
            1]`:

            - 1 indicates the head is **not masked**,
            - 0 indicates the head is **masked**.

        encoder_outputs (`Tuple(torch.FloatTensor)`, *optional*):
            Tuple consists of (`last_hidden_state`, *optional*: `hidden_states`, *optional*: `attentions`)
            `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)` is a sequence of hidden-states at
            the output of the last layer of the encoder. Used in the cross-attention of the decoder.
        past_key_values (`Tuple(torch.FloatTensor)` of length `config.n_layers` with each tuple having 4 tensors of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`):
            Contains precomputed key and value hidden-states of the attention blocks. Can be used to speed up decoding.
            If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that
            don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all
            `decoder_input_ids` of shape `(batch_size, sequence_length)`.
        inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
            Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
            is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
            model's internal embedding lookup matrix.
        decoder_inputs_embeds (`torch.FloatTensor` of shape `(batch_size, target_sequence_length, hidden_size)`, *optional*):
            Optionally, instead of passing `decoder_input_ids` you can choose to directly pass an embedded
            representation. If `past_key_values` is used, optionally only the last `decoder_inputs_embeds` have to be
            input (see `past_key_values`). This is useful if you want more control over how to convert
            `decoder_input_ids` indices into associated vectors than the model's internal embedding lookup matrix.

            If `decoder_input_ids` and `decoder_inputs_embeds` are both unset, `decoder_inputs_embeds` takes the value
            of `inputs_embeds`.
        use_cache (`bool`, *optional*, defaults to `True`):
            If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see
            `past_key_values`).
        output_attentions (`bool`, *optional*):
            Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
            tensors for more detail.
        output_hidden_states (`bool`, *optional*):
            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
            more detail.
        return_dict (`bool`, *optional*):
            Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
c                 óN   — | j                  «       dk(  sJ ‚| j                  d«      S )z+Turns 1->0, 0->1, False->True, True-> Falseé   r   )ÚdimÚeq)Úattention_masks    úd/var/www/skyplay_api_hub/venv/lib/python3.12/site-packages/transformers/models/fsmt/modeling_fsmt.pyÚinvert_maskr$   ,  s*   € à×ÑÓ 1Ò$Ð$Ð$Ø×Ñ˜QÓÐó    c                 óê   — | j                   d   }t        j                  || j                  ¬«      }|j	                  ||«      }|j                  d«      }|r||z   }||k\  }| j                  |dk(  d«      S )Nr   ©Údeviceéÿÿÿÿ)ÚshapeÚtorchÚaranger(   ÚexpandÚ	unsqueezeÚmasked_fill)ÚxÚdiagonalÚlr,   Úmasks        r#   Ú	triu_onnxr4   2  sn   € Ø	�‰�‰
€AÜ�\‰\˜! A§H¡HÔ-€FØ�=‰=˜˜AÓ€DØ×Ñ˜bÓ!€FÙØ˜(Ñ"ˆØ�6‰>€DØ�=‰=˜ ™ AÓ&Ð&r%   c           	      ó$  — | j                   }|€t        ||«      }|j                  «       \  }}|€t        ||«      }nt	        |«      }t        t        t        j                  |||¬«      «      d«      j                  |j                  ¬«      }|||fS )z÷
    Prepare masks that ignore padding tokens in the decoder and a causal mask for the decoder if none are provided.
    This mimics the default behavior in fairseq. To override it pass in masks. Note: this is not called during
    generation
    ©Údtyper   r'   )Úpad_token_idÚshift_tokens_rightÚsizeÚmake_padding_maskr$   r4   Úfill_with_neg_infr+   ÚzerosÚtor(   )	ÚconfigÚ	input_idsÚdecoder_input_idsÚdecoder_padding_maskÚcausal_mask_dtyper8   ÚbszÚtgt_lenÚcausal_masks	            r#   Ú_prepare_fsmt_decoder_inputsrG   =  sŸ   € ð ×&Ñ&€LØÐ Ü.¨y¸,ÓGÐØ$×)Ñ)Ó+�L€CˆØÐ#Ü0Ð1BÀLÓQÑä*Ð+?Ó@ÐÜÔ-¬e¯k©k¸'À7ÐRcÔ.dÓeÐghÓi×lÑlØ ×'Ñ'ð mó €Kð Ð2°KÐ?Ð?r%   c                   ó*   — e Zd ZeZdZd„ Zed„ «       Zy)ÚPretrainedFSMTModelÚmodelc                 óø  — | j                   j                  }t        |t        j                  «      rY|j
                  j                  j                  d|¬«       |j                  �%|j                  j                  j                  «        y y t        |t        «      ra |j                  g |j
                  j                  ¢|j                  ‘­Ž }t        j                  |d¬«      }|j                  «        ||_        y t        |t        j                   «      rf|j
                  j                  j                  d|¬«       |j                  �2|j
                  j                  |j                     j                  «        y y y )Nç        )ÚmeanÚstdF)Úrequires_grad)r?   Úinit_stdÚ
isinstancer
   ÚLinearÚweightÚdataÚnormal_ÚbiasÚzero_ÚSinusoidalPositionalEmbeddingÚget_embeddingr*   Úpadding_idxÚ	ParameterÚdetach_Ú	Embedding)ÚselfÚmodulerN   rS   s       r#   Ú_init_weightsz!PretrainedFSMTModel._init_weights[  s  € Ø�k‰k×"Ñ"ˆÜ�fœbŸi™iÔ(Ø�M‰M×Ñ×&Ñ&¨C°SÐ&Ô9Ø�{‰{Ð&Ø—‘× Ñ ×&Ñ&Õ(ð 'ä˜Ô =Ô>Ø)�V×)Ñ)ÐS¨6¯=©=×+>Ñ+>ÐSÀ×@RÑ@RÒSˆFÜ—\‘\ &¸Ô>ˆFØ�N‰NÔØ"ˆF�MÜ˜¤§¡Ô-Ø�M‰M×Ñ×&Ñ&¨C°SÐ&Ô9Ø×!Ñ!Ð-Ø—‘×"Ñ" 6×#5Ñ#5Ñ6×<Ñ<Õ>ð .ð .r%   c                 ó®   — | j                   j                  }t        j                  g d¢dddd|gg| j                  ¬«      }|j                  |«      |dœ}|S )N)r   é   é
   é   r   r   é   é   r   r'   )r"   r@   )r?   r8   r+   Útensorr(   Úne)r^   Ú	pad_tokenr@   Údummy_inputss       r#   rj   z PretrainedFSMTModel.dummy_inputsk  sW   € à—K‘K×,Ñ,ˆ	Ü—L‘LÒ"2°Q¸¸2¸qÀ)Ð4LÐ!MÐVZ×VaÑVaÔbˆ	à'Ÿl™l¨9Ó5Ø"ñ
ˆð Ðr%   N)	Ú__name__Ú
__module__Ú__qualname__r   Úconfig_classÚbase_model_prefixr`   Úpropertyrj   © r%   r#   rI   rI   W  s&   „ Ø€LØÐò?ð  ñó ñr%   rI   c                 ó²   — | j                   j                  \  }}t        j                  ||d¬«      }| j                   j                  |j                   _        |S )NF©rV   )rS   r*   r
   rR   rT   )ÚembÚ
vocab_sizeÚemb_sizeÚ	lin_layers       r#   Ú_make_linear_from_embrx   v  sD   € ØŸ:™:×+Ñ+Ñ€J�Ü—	‘	˜* h°UÔ;€IØŸJ™JŸO™O€I×ÑÔØÐr%   c                 ó0   — | |k7  rt        d| › d|› �«      ‚y )Nzshape mismatch: z != )ÚAssertionError)Úshape_1Úshape2s     r#   Ú_check_shapesr}   ~  s(   € Ø�&ÒÜÐ/°¨y¸¸V¸HÐEÓFÐFð r%   c                 ó,  — | j                  | dk(  |«       | j                  «       }| j                  |«      j                  d¬«      dz
  j	                  d«      }| j                  d|«      j                  «       |dd…df<   | dd…dd…f   |dd…dd…f<   |S )zXShift input ids one token to the right, and wrap the last non pad token (usually <eos>).iœÿÿÿr   ©r    r)   Nr   )Úmasked_fill_Úclonerh   Úsumr.   ÚgatherÚsqueeze)r@   r8   Úprev_output_tokensÚindex_of_eoss       r#   r9   r9   ƒ  s›   € ð ×Ñ˜9¨Ñ,¨lÔ;à"Ÿ™Ó*ÐØ—L‘L Ó.×2Ñ2°qÐ2Ó9¸AÑ=×HÑHÈÓL€LØ(×/Ñ/°°<Ó@×HÑHÓJÐ’q˜!�tÑØ )ª!¨S¨b¨S¨&Ñ 1Ð’q˜!™"�uÑØÐr%   c                 óL   — | j                  |«      }|j                  «       sd}|S )zTrue for pad tokensN)r!   Úany)r@   rZ   Úpadding_masks      r#   r;   r;   �  s'   € à—<‘< Ó,€LØ×ÑÔØˆØÐr%   c                   ó,   ‡ — e Zd Zdefˆ fd„Zdd„Zˆ xZS )ÚEncoderLayerr?   c                 ó>  •— t         ‰| �  «        |j                  | _        t	        | j                  |j
                  |j                  ¬«      | _        t        | j                  «      | _	        |j                  | _
        t        |j                     | _        |j                  | _        t        j                   | j                  |j"                  «      | _        t        j                   |j"                  | j                  «      | _        t        | j                  «      | _        y )N)Údropout)ÚsuperÚ__init__Úd_modelÚ	embed_dimÚ	AttentionÚencoder_attention_headsÚattention_dropoutÚ	self_attnr   Úself_attn_layer_normr�   r   Úactivation_functionÚactivation_fnÚactivation_dropoutr
   rR   Úencoder_ffn_dimÚfc1Úfc2Úfinal_layer_norm©r^   r?   Ú	__class__s     €r#   r�   zEncoderLayer.__init__œ  sÀ   ø€ Ü‰ÑÔØŸ™ˆŒÜ" 4§>¡>°6×3QÑ3QÐ[a×[sÑ[sÔtˆŒÜ$-¨d¯n©nÓ$=ˆÔ!Ø—~‘~ˆŒÜ# F×$>Ñ$>Ñ?ˆÔØ"(×";Ñ";ˆÔÜ—9‘9˜TŸ^™^¨V×-CÑ-CÓDˆŒÜ—9‘9˜V×3Ñ3°T·^±^ÓDˆŒÜ )¨$¯.©.Ó 9ˆÕr%   c                 óB  — |}| j                  |||||¬«      \  }}t        j                  j                  || j                  | j                  ¬«      }||z   }| j                  |«      }|}| j                  | j                  |«      «      }t        j                  j                  || j                  | j                  ¬«      }| j                  |«      }t        j                  j                  || j                  | j                  ¬«      }||z   }| j                  |«      }||fS )al  
        Args:
            x (`torch.Tensor`): input to the layer of shape *(seq_len, batch, embed_dim)*
            encoder_padding_mask (`torch.ByteTensor`): binary ByteTensor of shape
                *(batch, src_len)* where padding elements are indicated by `1`.
            for t_tgt, t_src is excluded (or masked out), =0 means it is
            included in attention
            layer_head_mask (`torch.FloatTensor`): mask for attention heads in a given layer of size
                *(config.encoder_attention_heads,)*.

        Returns:
            encoded output of shape *(seq_len, batch, embed_dim)*
        )ÚqueryÚkeyÚkey_padding_maskÚlayer_head_maskÚoutput_attentions©ÚpÚtraining)r•   r
   Ú
functionalr�   r¨   r–   r˜   r›   r™   rœ   r�   )r^   r0   Úencoder_padding_maskr¤   r¥   ÚresidualÚattn_weightss          r#   ÚforwardzEncoderLayer.forward¨  s  € ð ˆØŸ.™.ØØØ1Ø+Ø/ð )ó 
‰ˆˆ<ô �M‰M×!Ñ! ! t§|¡|¸d¿m¹mÐ!ÓLˆØ�q‰LˆØ×%Ñ% aÓ(ˆàˆØ×Ñ˜tŸx™x¨›{Ó+ˆÜ�M‰M×!Ñ! ! t×'>Ñ'>ÈÏÉÐ!ÓWˆØ�H‰H�Q‹KˆÜ�M‰M×!Ñ! ! t§|¡|¸d¿m¹mÐ!ÓLˆØ�q‰LˆØ×!Ñ! !Ó$ˆØ�,ˆÐr%   )F©rk   rl   rm   r   r�   r­   Ú__classcell__©rŸ   s   @r#   r‹   r‹   ›  s   ø„ ð
:˜zõ 
:÷!r%   r‹   c                   ó¼   ‡ — e Zd ZdZdefˆ fd„Z	 	 	 	 	 	 ddej                  deej                     deej                     deej                     de	d	e	d
e	fd„Z
ˆ xZS )ÚFSMTEncoderz¢
    Transformer encoder consisting of *config.encoder_layers* self attention layers. Each layer is a [`EncoderLayer`].

    Args:
        config: FSMTConfig
    r?   c                 óü  •— t         ‰| �  «        |j                  | _        |j                  | _        |j
                  | _        || _        |j                  }|j                  rt        j                  |«      nd| _        t        |j                  | j
                  z   dz   || j
                  «      | _        t        j                   t#        |j$                  «      D �cg c]  }t'        |«      ‘Œ c}«      | _        y c c}w )Nç      ð?r   )rŽ   r�   r�   Úencoder_layerdropÚ	layerdroprZ   Úembed_tokensÚembedding_dimÚscale_embeddingÚmathÚsqrtÚembed_scalerX   Úmax_position_embeddingsÚembed_positionsr
   Ú
ModuleListÚrangeÚencoder_layersr‹   Úlayers)r^   r?   r·   r‘   Ú_rŸ   s        €r#   r�   zFSMTEncoder.__init__Ô  sÂ   ø€ Ü‰ÑÔØ—~‘~ˆŒØ×1Ñ1ˆŒØ'×3Ñ3ˆÔØ(ˆÔØ ×.Ñ.ˆ	Ø39×3IÒ3Iœ4Ÿ9™9 YÔ/ÈsˆÔÜ<Ø×*Ñ*¨T×-=Ñ-=Ñ=ÀÑAÀ9Èd×N^ÑN^ó 
ˆÔô —m‘mÄ5È×I^ÑI^ÓC_Ö$`¸a¤\°&Õ%9Ò$`Óaˆ�ùÒ$`s   ÃC9r@   r"   Úinputs_embedsÚ	head_maskr¥   Úoutput_hidden_statesÚreturn_dictc                 ó¬  — |�t        |«      }|�|�t        d«      ‚|�0| j                  |«      | j                  z  }| j	                  |«      }nw|�j|| j                  z  }|dd…dd…df   j                  |dd…dd…df   j                  d«      | j                  j                  «      }	| j	                  |	«      }nt        d«      ‚||z   }
t        j                  j                  |
| j                  | j                  ¬«      }
|
j                  dd«      }
|rdnd}|rdnd}|�[|j                  «       d   t        | j                  «      k(  s2J dt        | j                  «      › d	|j                  «       d   › d
�«       ‚t!        | j                  «      D ]ƒ  \  }}|r*|
j                  dd«      }
||
fz  }|
j                  dd«      }
t#        j$                  g «      }| j                  r|| j&                  k  rd}n ||
||�||   nd|¬«      \  }
}|sŒ~||fz   }Œ… |
j                  dd«      }
|r||
fz  }|st)        d„ |
||fD «       «      S t+        |
||¬«      S )aÜ  
        Args:
            input_ids (`torch.LongTensor`): tokens in the source language of shape
                *(batch, src_len)*
            attention_mask (`torch.LongTensor`): indicating which indices are padding tokens
            inputs_embeds (`torch.FloatTensor`):
                embedding vectors of shape *(batch, src_len, embed_dim)*
            head_mask (`torch.Tensor` of shape `(num_layers, num_heads)`, *optional*):
                Mask to nullify selected heads of the attention modules. Mask values selected in `[0, 1]`:

                - 1 indicates the head is **not masked**,
                - 0 indicates the head is **masked**.

        Returns:
            BaseModelOutput or Tuple comprised of:

                - **x** (`torch.Tensor`): the last encoder layer's output of shape *(src_len, batch, embed_dim)*
                - **encoder_states** (`Tuple(torch.FloatTensor)`): all intermediate hidden states of shape *(src_len,
                  batch, embed_dim)*. Only populated if *output_hidden_states:* is True.
                - **all_attentions** (`Tuple(torch.FloatTensor)`): Attention weights for each layer.
                During training might not be of length n_layers because of layer dropout.
        NzDYou cannot specify both input_ids and inputs_embeds at the same timer   z5You have to specify either input_ids or inputs_embedsr¦   r   rq   z&The head_mask should be specified for ú layers, but it is for ú.)r¤   r¥   c              3   ó&   K  — | ]	  }|€Œ|–— Œ y ­w©Nrq   ©Ú.0Úvs     r#   ú	<genexpr>z&FSMTEncoder.forward.<locals>.<genexpr>?  s   è ø€ ÒY˜qÈ1É=œÑYùó   ‚Š©Úlast_hidden_stateÚhidden_statesÚ
attentions)r$   Ú
ValueErrorr·   r¼   r¾   r/   r!   rZ   r
   r©   r�   r¨   Ú	transposer:   ÚlenrÂ   Ú	enumerater+   Úrandr¶   Útupler   )r^   r@   r"   rÄ   rÅ   r¥   rÆ   rÇ   Ú	embed_posÚposition_idsr0   Úencoder_statesÚall_attentionsÚidxÚencoder_layerÚdropout_probabilityÚattns                    r#   r­   zFSMTEncoder.forwardá  s|  € ðB Ð%Ü(¨Ó8ˆNàÐ  ]Ð%>ÜÐcÓdÐdØÐ"Ø ×-Ñ-¨iÓ8¸4×;KÑ;KÑKˆMØ×,Ñ,¨YÓ7‰IØÐ&Ø)¨D×,<Ñ,<Ñ<ˆMð )ªªA¨q¨Ñ1×=Ñ=Øša¢ A˜gÑ&×)Ñ)¨!Ó,¨d×.BÑ.B×.NÑ.NóˆLð ×,Ñ,¨\Ó:‰IäÐTÓUÐUà˜IÑ%ˆÜ�M‰M×!Ñ! ! t§|¡|¸d¿m¹mÐ!ÓLˆð �K‰K˜˜1Óˆá3™¸ˆÙ0™°dˆàÐ Ø—>‘>Ó# AÑ&¬3¨t¯{©{Ó+;Ò<ð Ø8¼¸T¿[¹[Ó9IÐ8JÐJaÐbk×bpÑbpÓbrÐstÑbuÐavÐvwÐxóÐ<ô #,¨D¯K©KÓ"8ò 	:ÑˆC�Ù#Ø—K‘K  1Ó%�Ø 1 $Ñ&�Ø—K‘K  1Ó%�ä"'§*¡*¨R£.ÐØ�}Š}Ð"5¸¿¹Ò"FØ‘á'ØØ"Ø7@Ð7L Y¨s¢^ÐRVØ&7ô	‘��4ò !Ø!/°4°'Ñ!9‘ð%	:ð* �K‰K˜˜1ÓˆáØ˜q˜dÑ"ˆNáÜÑY Q¨¸Ð$GÔYÓYÐYÜ°À.Ð]kÔlÐlr%   )NNNFFT)rk   rl   rm   Ú__doc__r   r�   r+   r	   r   Úboolr­   r¯   r°   s   @r#   r²   r²   Ì  s¢   ø„ ñðb˜zõ bð  26Ø04Ø,0Ø"'Ø%*Ø ñ_mà—<‘<ð_mð ! §¡Ñ.ð_mð   §¡Ñ-ð	_mð
 ˜EŸL™LÑ)ð_mð  ð_mð #ð_mð ÷_mr%   r²   c                   ó:   ‡ — e Zd Zdefˆ fd„Z	 	 	 	 	 	 	 dd„Zˆ xZS )ÚDecoderLayerr?   c                 óÖ  •— t         ‰| �  «        |j                  | _        t	        | j                  |j
                  |j                  ¬«      | _        |j                  | _        t        |j                     | _        |j                  | _        t        | j                  «      | _        t	        | j                  |j
                  |j                  d¬«      | _        t        | j                  «      | _        t#        j$                  | j                  |j&                  «      | _        t#        j$                  |j&                  | j                  «      | _        t        | j                  «      | _        y )N)r‘   Ú	num_headsr�   T)r�   Úencoder_decoder_attention)rŽ   r�   r�   r‘   r’   Údecoder_attention_headsr”   r•   r�   r   r—   r˜   r™   r   r–   Úencoder_attnÚencoder_attn_layer_normr
   rR   Údecoder_ffn_dimr›   rœ   r�   rž   s     €r#   r�   zDecoderLayer.__init__D  s  ø€ Ü‰ÑÔØŸ™ˆŒä"Ø—n‘nØ×4Ñ4Ø×,Ñ,ô
ˆŒð
 —~‘~ˆŒÜ# F×$>Ñ$>Ñ?ˆÔØ"(×";Ñ";ˆÔä$-¨d¯n©nÓ$=ˆÔ!Ü%Ø�N‰NØ×*Ñ*Ø×,Ñ,Ø&*ô	
ˆÔô (1°·±Ó'@ˆÔ$Ü—9‘9˜TŸ^™^¨V×-CÑ-CÓDˆŒÜ—9‘9˜V×3Ñ3°T·^±^ÓDˆŒÜ )¨$¯.©.Ó 9ˆÕr%   c
           	      ó€  — |}
|€i }| j                  |||||||	¬«      \  }}t        j                  j                  || j                  | j                  ¬«      }|
|z   }| j                  |«      }|}
| j                  j                  | j                   j                  k7  sJ ‚| j                  ||||||	¬«      \  }}t        j                  j                  || j                  | j                  ¬«      }|
|z   }| j                  |«      }|}
| j                  | j                  |«      «      }t        j                  j                  || j                  | j                  ¬«      }| j                  |«      }t        j                  j                  || j                  | j                  ¬«      }|
|z   }| j                  |«      }||||fS )N)r¡   r¢   Úlayer_stater£   Ú	attn_maskr¤   r¥   r¦   )r¡   r¢   r£   rð   r¤   r¥   )r•   r
   r©   r�   r¨   r–   rì   Ú	cache_keyrí   r˜   r›   r™   rœ   r�   )r^   r0   Úencoder_hidden_statesÚencoder_attn_maskrð   rF   r¤   Úcross_attn_layer_head_maskrB   r¥   r«   Úself_attn_weightsÚcross_attn_weightss                r#   r­   zDecoderLayer.forward]  s»  € ð ˆàÐØˆKð  $Ÿ~™~ØØØ#Ø1Ø!Ø+Ø/ð  .ó  
ÑˆÐô �M‰M×!Ñ! ! t§|¡|¸d¿m¹mÐ!ÓLˆØ�q‰LˆØ×%Ñ% aÓ(ˆð ˆØ× Ñ ×*Ñ*¨d¯n©n×.FÑ.FÒFÐFÐFØ $× 1Ñ 1ØØ%Ø.Ø#Ø6Ø/ð !2ó !
ÑˆÐô �M‰M×!Ñ! ! t§|¡|¸d¿m¹mÐ!ÓLˆØ�q‰LˆØ×(Ñ(¨Ó+ˆð ˆØ×Ñ˜tŸx™x¨›{Ó+ˆÜ�M‰M×!Ñ! ! t×'>Ñ'>ÈÏÉÐ!ÓWˆØ�H‰H�Q‹KˆÜ�M‰M×!Ñ! ! t§|¡|¸d¿m¹mÐ!ÓLˆØ�q‰LˆØ×!Ñ! !Ó$ˆàØØØð	
ð 	
r%   )NNNNNNFr®   r°   s   @r#   rç   rç   C  s+   ø„ ð:˜zõ :ð: ØØØØ#'Ø!Ø÷;
r%   rç   c                   óf  ‡ — e Zd ZdZdedej                  fˆ fd„Zd„ Z	 	 	 	 	 	 	 	 dde	j                  de	j                  de	j                  d	e	j                  d
e	j                  dee	j                     dee	j                     dee	j                     deee	j                        dedededefd„Zˆ xZS )ÚFSMTDecoderzÈ
    Transformer decoder consisting of *config.decoder_layers* layers. Each layer is a [`DecoderLayer`]

    Args:
        config: FSMTConfig
        embed_tokens (nn.Embedding): output embedding
    r?   r·   c                 óÊ  •— t         ‰| �  «        |j                  | _        |j                  | _        |j
                  | _        |j                  rt        j                  |j                  «      nd| _
        || _        |j                  }t        |j                  | j
                  z   dz   || j
                  «      | _        t!        j"                  t%        |j&                  «      D �cg c]  }t)        |«      ‘Œ c}«      | _        t-        «       r_dd l}|j0                  j3                  | j                  j4                  d ¬«      5  | j                  j4                  j6                  }d d d «       n | j                  j4                  j6                  }t!        j8                  d   |d   d¬«      | _        | j                  j4                  | j:                  _        y c c}w # 1 sw Y   ŒWxY w)Nr´   r   r   )Úmodifier_rankFrs   )rŽ   r�   r�   Údecoder_layerdropr¶   rZ   r¹   rº   r»   r�   r¼   r·   r¸   rX   r½   r¾   r
   r¿   rÀ   Údecoder_layersrç   rÂ   r   Ú	deepspeedÚzeroÚGatheredParametersrS   r*   rR   Úoutput_projection)r^   r?   r·   r‘   rÃ   rþ   Úembed_tokens_weight_shaperŸ   s          €r#   r�   zFSMTDecoder.__init__¤  s…  ø€ Ü‰ÑÔØ—~‘~ˆŒØ×1Ñ1ˆŒØ'×3Ñ3ˆÔØ8>×8NÒ8Nœ4Ÿ9™9 V§^¡^Ô4ÐTWˆÔØ(ˆÔØ ×.Ñ.ˆ	Ü<Ø×*Ñ*¨T×-=Ñ-=Ñ=ÀÑAÀ9Èd×N^ÑN^ó 
ˆÔô —m‘mÄ5È×I^ÑI^ÓC_Ö$`¸a¤\°&Õ%9Ò$`ÓaˆŒä%Ô'Ûà—‘×2Ñ2°4×3DÑ3D×3KÑ3KÐ[_Ð2Ó`ñ KØ,0×,=Ñ,=×,DÑ,D×,JÑ,JÐ)÷Kð Kð )-×(9Ñ(9×(@Ñ(@×(FÑ(FÐ%Ü!#§¡Ð+DÀQÑ+GÐIbÐcdÑIeÐlqÔ!rˆÔØ(,×(9Ñ(9×(@Ñ(@ˆ×ÑÕ%ùò %a÷
Kð Kús   Ã$GÅ!GÇG"c                 óN   — | j                   j                  | j                  _        y rÌ   )r  rS   r·   ©r^   s    r#   Ú_tie_weightszFSMTDecoder._tie_weights»  s   € Ø#'×#9Ñ#9×#@Ñ#@ˆ×ÑÕ r%   r@   ró   rª   rB   Údecoder_causal_maskrÅ   rÄ   Úcross_attn_head_maskÚpast_key_valuesÚ	use_cacher¥   rÆ   rÇ   c                 ób  — |�t        |«      }|�|�t        d«      ‚|�H| j                  |«      }|
r|dd…dd…f   }|dd…dd…f   }| j                  |«      | j                  z  }nw|�j|dd…dd…df   j                  |dd…dd…df   j                  d«      | j                  j                  «      }| j                  |«      }|| j                  z  }nt        d«      ‚||z  }t        j                  j                  || j                  | j                  ¬«      }|j                  dd«      }|j                  dd«      }|rdnd}|rdnd}|rdnd}g }t        ||gd	d
g«      D ]f  \  }}|€Œ	|j                  «       d   t        | j                   «      k(  rŒ3J d|› dt        | j                   «      › d|j                  «       d   › d�«       ‚ t#        | j                   «      D ]¿  \  }}|r*|j                  dd«      }||fz  }|j                  dd«      }| j                  r%t%        j&                  g «      }|| j(                  k  rŒc|	�|	|   nd} ||||||||�||   nd|�||   nd|¬«	      \  }}}}|
r|j+                  |j-                  «       «       |sŒ´||fz  }||fz  }ŒÁ |r*|j                  dd«      }||fz  }|j                  dd«      }|j                  dd«      }|j                  dd«      }| j/                  |«      }|
r|nd}|st1        d„ |||||fD «       «      S t3        |||||¬«      S )a‘  
        Includes several features from "Jointly Learning to Align and Translate with Transformer Models" (Garg et al.,
        EMNLP 2019).

        Args:
            input_ids (`torch.LongTensor` of shape `(batch, tgt_len)`):
                previous decoder outputs for teacher forcing
            encoder_hidden_states: output from the encoder, used for
                encoder-side attention
            encoder_padding_mask: for ignoring pad tokens
            past_key_values (dict or None): dictionary used for storing state during generation
            head_mask (`torch.Tensor` of shape `(num_layers, num_heads)`, *optional*):
                Mask to nullify selected heads of the attention modules. Mask values selected in `[0, 1]`:

                - 1 indicates the head is **not masked**,
                - 0 indicates the head is **masked**.

            cross_attn_head_mask (`torch.Tensor` of shape `(num_layers, num_heads)`, *optional*):
                Mask to nullify selected heads of the cross-attention modules. Mask values selected in `[0, 1]`:

                - 1 indicates the head is **not masked**,
                - 0 indicates the head is **masked**.

        Returns:
            BaseModelOutputWithPast or tuple:

                - the decoder's features of shape *(batch, tgt_len, embed_dim)*
                - the cache
                - hidden states
                - attentions
        NzTYou cannot specify both decoder_input_ids and decoder_inputs_embeds at the same timer)   r   zEYou have to specify either decoder_input_ids or decoder_inputs_embedsr¦   r   rq   rÅ   r  zThe `z` should be specified for rÉ   rÊ   )rô   rB   rð   rF   r¤   rõ   r¥   c              3   ó&   K  — | ]	  }|€Œ|–— Œ y ­wrÌ   rq   rÍ   s     r#   rÐ   z&FSMTDecoder.forward.<locals>.<genexpr>H  s   è ø€ ò ØÐbcÑbo”ñùrÑ   )rÓ   r  rÔ   rÕ   Úcross_attentions)r$   rÖ   r¾   r·   r¼   r/   r!   rZ   r
   r©   r�   r¨   r×   Úzipr:   rØ   rÂ   rÙ   r+   rÚ   r¶   ÚappendÚcopyr  rÛ   r   )r^   r@   ró   rª   rB   r  rÅ   rÄ   r  r  r	  r¥   rÆ   rÇ   Ú	positionsr0   rÝ   Úall_hidden_statesÚall_self_attnsÚall_cross_attnsÚnext_decoder_cacherñ   Ú	mask_namerà   Údecoder_layerrâ   rð   Úlayer_self_attnÚ
layer_pastÚlayer_cross_attnÚ
next_caches                                  r#   r­   zFSMTDecoder.forward¾  s¾  € ð`  Ð+Ü#.Ð/CÓ#DÐ àÐ  ]Ð%>ÜÐsÓtÐtØÐ"à×,Ñ,¨YÓ7ˆIÙØ%¢a¨© fÑ-�	Ø%¢a¨© fÑ-�	Ø×!Ñ! )Ó,¨t×/?Ñ/?Ñ?‰AØÐ&ð )ªªA¨q¨Ñ1×=Ñ=Øša¢ A˜gÑ&×)Ñ)¨!Ó,¨d×.BÑ.B×.NÑ.NóˆLð ×,Ñ,¨\Ó:ˆIØ × 0Ñ 0Ñ0‰AäÐdÓeÐeà	ˆY‰ˆÜ�M‰M×!Ñ! ! t§|¡|¸d¿m¹mÐ!ÓLˆð �K‰K˜˜1ÓˆØ 5× ?Ñ ?ÀÀ1Ó EÐñ #7™B¸DÐÙ0™°dˆÙ 1™"°tˆØÐô %(¨Ð4HÐ(IÈKÐYoÐKpÓ$qò 	Ñ ˆI�yØÑ$Ø —~‘~Ó'¨Ñ*¬s°4·;±;Ó/?Ó@ð Ø˜I˜;Ð&@ÄÀTÇ[Á[ÓAQÐ@Rð SØ!Ÿ™Ó(¨Ñ+Ð,¨Að/óÐ@ð	ô #,¨D¯K©KÓ"8ò 	7ÑˆC�á#Ø—K‘K  1Ó%�Ø! a TÑ)Ð!Ø—K‘K  1Ó%�Ø�}Š}Ü&+§j¡j°£nÐ#Ø&¨¯©Ò7Øà2AÐ2M˜/¨#Ò.ÐSWˆKá?LØØ%Ø"6Ø%9Ø'Ø/Ø3<Ð3H ¨3¢ÈdØI]ÐIiÐ,@ÀÒ,EÐosØ"3ô
@Ñ<ˆAˆ 
Ð,<ñ Ø"×)Ñ)¨*¯/©/Ó*;Ô<â Ø ?Ð"4Ñ4�ØÐ$4Ð#6Ñ6‘ð=	7ñB  Ø—‘˜A˜qÓ!ˆAØ ! Ñ%ÐØ—‘˜A˜qÓ!ˆAð �K‰K˜˜1ÓˆØ 5× ?Ñ ?ÀÀ1Ó EÐà×"Ñ" 1Ó%ˆá+4Ñ'¸$ˆ
áÜñ Ø˜zÐ+<¸nÈoÐ^ôó ð ô 9ØØ&Ø+Ø%Ø,ô
ð 	
r%   )NNNNFFFT)rk   rl   rm   rä   r   r
   r]   r�   r  r+   r	   r   r   ÚFloatTensorrå   r­   r¯   r°   s   @r#   rù   rù   ›  s  ø„ ñðA˜zð A¸¿¹õ Aò.Að -1Ø04Ø7;Ø=AØØ"'Ø%*Ø ñS
à—<‘<ðS
ð  %Ÿ|™|ðS
ð $Ÿl™lð	S
ð
 $Ÿl™lðS
ð #Ÿ\™\ðS
ð ˜EŸL™LÑ)ðS
ð   §¡Ñ-ðS
ð ' u§|¡|Ñ4ðS
ð " $ u×'8Ñ'8Ñ"9Ñ:ðS
ð ðS
ð  ðS
ð #ðS
ð ÷S
r%   rù   c                 óf   — | j                  «       D ]  \  }}|€Œ	|j                  d|«      | |<   Œ | S )Nr   )ÚitemsÚindex_select)Ú
attn_cacheÚ	new_orderÚkÚinput_buffer_ks       r#   Ú_reorder_bufferr#  T  sE   € Ø'×-Ñ-Ó/ò FÑˆˆ>ØÑ%Ø*×7Ñ7¸¸9ÓEˆJ�qŠMðFð Ðr%   c                   ó    ‡ — e Zd ZdZ	 	 	 dˆ fd„	Zd„ Z	 	 	 	 	 ddee   dee   deee	ee   f      dee   dee   d	e
eee   f   fd
„Zd„ Zˆ xZS )r’   z=Multi-headed attention from 'Attention Is All You Need' paperc                 óú  •— t         ‰| �  «        || _        || _        || _        ||z  | _        | j
                  |z  | j                  k(  sJ d«       ‚| j
                  dz  | _        || _        t        j                  |||¬«      | _
        t        j                  |||¬«      | _        t        j                  |||¬«      | _        t        j                  |||¬«      | _        | j                  rd| _        y d| _        y )Nz(embed_dim must be divisible by num_headsg      à¿rs   Úencoder_decoderr^   )rŽ   r�   r‘   ré   r�   Úhead_dimÚscalingrê   r
   rR   Úk_projÚv_projÚq_projÚout_projrò   )r^   r‘   ré   r�   rV   rê   rŸ   s         €r#   r�   zAttention.__init__^  sÑ   ø€ ô 	‰ÑÔØ"ˆŒØ"ˆŒØˆŒØ! YÑ.ˆŒØ�}‰}˜yÑ(¨D¯N©NÒ:ÐfÐ<fÓfÐ:Ø—}‘} dÑ*ˆŒà)BˆÔ&Ü—i‘i 	¨9¸4Ô@ˆŒÜ—i‘i 	¨9¸4Ô@ˆŒÜ—i‘i 	¨9¸4Ô@ˆŒÜŸ	™	 )¨Y¸TÔBˆŒØ.2×.LÒ.LÐ*ˆ�ÐRXˆ�r%   c                 ó’   — |j                  «       j                  ||| j                  z  | j                  «      j	                  dd«      S )Nr   r   )Ú
contiguousÚviewré   r'  r×   )r^   rg   Úseq_lenrD   s       r#   Ú_shapezAttention._shapeu  s<   € Ø× Ñ Ó"×'Ñ'¨°°t·~±~Ñ1EÀtÇ}Á}ÓU×_Ñ_Ð`aÐcdÓeÐer%   r¢   r£   rð   rñ   r¤   Úreturnc                 ó®	  — | j                   }|j                  «       \  }	}
}|| j                  k(  sJ ‚t        |j                  «       «      |	|
|gk(  sJ ‚|�%|j	                  | j
                  i «      }d|v r	|rd}nd}i }| j                  |«      | j                  z  }|r*|€dx}}nE| j                  |«      }| j                  |«      }n"| j                  |«      }| j                  |«      }| j                  ||	|
«      }|�| j                  |d|
«      }|�| j                  |d|
«      }|�| j                  ||||||
«      \  }}}|j                  |
| j                  d| j                  «      |j                  |
| j                  d| j                  «      |s|nddœ|| j
                  <   |€J ‚|j                  d«      }t        j                   ||j#                  dd«      «      }|j                  «       |
| j                  z  |	|fk(  sJ ‚|�A|j                  |
| j                  |	|«      |z   }|j                  |
| j                  z  |	|«      }|�|j%                  «       dk(  rd}|�|j                  «       dd |
|fk(  sJ ‚|�—|j                  |
| j                  |	|«      }|j'                  d«      j'                  d«      }|j)                  |t        j*                  |j,                  «      j.                  «      }|j                  |
| j                  z  |	|«      }t0        j2                  j5                  |d¬«      }|�—|j                  «       | j                  fk(  s&J d	| j                  f› d
|j                  «       › �«       ‚|j                  dddd«      |j                  |
| j                  |	|«      z  }|j                  |
| j                  z  |	|«      }|r?|j                  |
| j                  |	|«      }|j                  |
| j                  z  |	|«      }nd}t0        j2                  j7                  || j6                  | j8                  ¬«      }|€J ‚t        j                   ||«      }|j                  «       |
| j                  z  |	| j                  fk(  sJ ‚|j#                  dd«      j;                  «       j                  |	|
|«      }| j=                  |«      }||fS )z+Input shape: Time(SeqLen) x Batch x ChannelNÚprev_keyr)   )r4  Ú
prev_valueÚprev_key_padding_maskr   r   r   r   z/Head mask for a single layer should be of size z	, but is r¦   )rê   r:   r‘   ÚlistÚgetrò   r+  r(  r)  r*  r1  Ú_use_saved_stater/  ré   r'  r+   Úbmmr×   r    r.   r/   Úfinfor7   Úminr
   r©   Úsoftmaxr�   r¨   r.  r,  )r^   r¡   r¢   r£   rð   rñ   r¤   r¥   Ú	static_kvrE   rD   r‘   Úsaved_stateÚqr!  rÏ   Úsrc_lenr¬   ÚreshapedÚattn_weights_reshapedÚ
attn_probsÚattn_outputs                         r#   r­   zAttention.forwardx  sÇ  € ð ×8Ñ8ˆ	Ø"'§*¡*£,Ñˆ��iØ˜DŸN™NÒ*Ð*Ð*Ü�E—J‘J“LÓ! g¨s°IÐ%>Ò>Ð>Ð>àÐ"Ø%Ÿ/™/¨$¯.©.¸"Ó=ˆKØ˜[Ñ(©Yà‘àˆKØˆKà�K‰K˜Ó §¡Ñ-ˆÙØˆ{Ø��‘Aà—K‘K Ó$�Ø—K‘K Ó$‘à—‘˜EÓ"ˆAØ—‘˜EÓ"ˆAà�K‰K˜˜7 CÓ(ˆØˆ=Ø—‘˜A˜r 3Ó'ˆAØˆ=Ø—‘˜A˜r 3Ó'ˆAàÐ"Ø%)×%:Ñ%:¸1¸aÀÐN^Ð`iÐknÓ%oÑ"ˆAˆqÐ"ð Ÿ™˜s D§N¡N°B¸¿¹ÓFØŸ&™&  d§n¡n°b¸$¿-¹-ÓHÙ=FÑ%5ÈDñ'
ˆ�D—N‘NÑ#ð ˆ}Ðˆ}Ø—&‘&˜“)ˆÜ—y‘y  A§K¡K°°1Ó$5Ó6ˆØ× Ñ Ó" s¨T¯^©^Ñ';¸WÀgÐ&NÒNÐNÐNàÐ Ø'×,Ñ,¨S°$·.±.À'È7ÓSÐV_Ñ_ˆLØ'×,Ñ,¨S°4·>±>Ñ-AÀ7ÈGÓTˆLð Ð'Ð,<×,@Ñ,@Ó,BÀaÒ,GØ#ÐØÐ'Ð+;×+@Ñ+@Ó+BÀ2ÀAÐ+FØØðK
ò ,
ð 	
ð 
ð
 Ð'Ø'×,Ñ,¨S°$·.±.À'È7ÓSˆLØ'×1Ñ1°!Ó4×>Ñ>¸qÓAˆHØ'×3Ñ3°H¼e¿k¹kÈ,×J\ÑJ\Ó>]×>aÑ>aÓbˆLØ'×,Ñ,¨S°4·>±>Ñ-AÀ7ÈGÓTˆLä—}‘}×,Ñ,¨\¸rÐ,ÓBˆàÐ&Ø"×'Ñ'Ó)¨d¯n©nÐ->Ò>ð ØAÀ4Ç>Á>ÐBSÐATÐT]Ð^m×^rÑ^rÓ^tÐ]uÐvóÐ>ð +×/Ñ/°°2°q¸!Ó<¸|×?PÑ?PÐQTÐVZ×VdÑVdÐfmÐovÓ?wÑwˆLØ'×,Ñ,¨S°4·>±>Ñ-AÀ7ÈGÓTˆLáà$0×$5Ñ$5°c¸4¿>¹>È7ÐT[Ó$\Ð!Ø0×5Ñ5°c¸D¿N¹NÑ6JÈGÐU\Ó]‰Là$(Ð!ä—]‘]×*Ñ*ØØ�l‰lØ—]‘]ð +ó 
ˆ
ð ˆ}Ðˆ}Ü—i‘i 
¨AÓ.ˆØ×ÑÓ! c¨D¯N©NÑ&:¸GÀTÇ]Á]Ð%SÒSÐSÐSØ!×+Ñ+¨A¨qÓ1×<Ñ<Ó>×CÑCÀGÈSÐR[Ó\ˆØ—m‘m KÓ0ˆàÐ1Ð1Ð1r%   c                 óæ  — d|v rU|d   }|€J ‚|j                  || j                  z  d| j                  «      }|r|}n|€J ‚t        j                  ||gd¬«      }d|v rU|d   }	|	€J ‚|	j                  || j                  z  d| j                  «      }
|r|
}n|€J ‚t        j                  |
|gd¬«      }|�|€J ‚|j                  dd «      }|�|r|}nt        j                  ||gd¬«      }n|}|||fS )Nr4  r)   r   r   r5  r6  )r/  ré   r'  r+   Úcatr8  )r^   r!  rÏ   r?  r£   r>  rD   Ú	_prev_keyr4  Ú_prev_valuer5  r6  Únew_key_padding_masks                r#   r9  zAttention._use_saved_stateá  s(  € à˜Ñ$Ø# JÑ/ˆIØÐ(Ð(Ð(Ø —~‘~ c¨D¯N©NÑ&:¸BÀÇÁÓNˆHÙØ‘à�}Ð$�}Ü—I‘I˜x¨˜m°Ô3�Ø˜;Ñ&Ø% lÑ3ˆKØÐ*Ð*Ð*Ø$×)Ñ)¨#°·±Ñ*>ÀÀDÇMÁMÓRˆJÙØ‘à�}Ð$�}Ü—I‘I˜z¨1˜o°1Ô5�Øˆ}  Ð.Ð.Ø2=·/±/ÐBYÐ[_Ó2`ÐØ Ð,ÙØ'<Ñ$ä',§y¡yÐ2GÐIYÐ1ZÐ`aÔ'bÑ$à#3Ð Ø�!Ð)Ð)Ð)r%   )rL   TF)NNNNF)rk   rl   rm   rä   r�   r1  r   r	   r   Ústrr   r­   r9  r¯   r°   s   @r#   r’   r’   [  s³   ø„ ÙGð ØØ"'õYò.fð .2Ø=AØ&*Ø,0Øñg2ð �fÑðg2ð # 6Ñ*ð	g2ð
 ˜d 3¨°Ñ(8Ð#8Ñ9Ñ:ðg2ð ˜FÑ#ðg2ð " &Ñ)ðg2ð 
ˆv�x Ñ'Ð'Ñ	(óg2öR*r%   r’   c                 ó¬   — | j                  «       j                  t        j                  | j                  «      j
                  «      j                  | «      S )z:FP16-compatible function that fills a input_ids with -inf.)ÚfloatÚfill_r+   r;  r7   r<  Útype_as©Úts    r#   r<   r<     s5   € à�7‰7‹9�?‰?œ5Ÿ;™; q§w¡wÓ/×3Ñ3Ó4×<Ñ<¸QÓ?Ð?r%   c                 ó   — t        | dd «      S )Nr*   )ÚgetattrrP  s    r#   Ú
_get_shaperT    s   € Ü�1�g˜tÓ$Ð$r%   zRThe bare FSMT Model outputting raw hidden-states without any specific head on top.c            %       óN  ‡ — e Zd ZddgZdefˆ fd„Zd„ Zd„ Zd„ Z e	e
«       eeee¬«      	 	 	 	 	 	 	 	 	 	 	 	 	 	 dd	ej                   d
eej$                     deej                      deej&                     deej$                     deej$                     deej$                     deeej*                        deeej*                        dee   dee   dee   deej*                     deej*                     dee   deeej$                     ef   f d„«       «       Zd„ Zd„ Zd„ Zd„ Zˆ xZS )Ú	FSMTModelúdecoder.embed_tokens.weightú decoder.output_projection.weightr?   c                 óL  •— t         ‰| �  |«       |j                  }t        j                  |j
                  |j                  |«      }t        j                  |j                  |j                  |«      }t        ||«      | _	        t        ||«      | _        | j                  «        y rÌ   )rŽ   r�   r8   r
   r]   Úsrc_vocab_sizer�   Útgt_vocab_sizer²   Úencoderrù   ÚdecoderÚ	post_init)r^   r?   rZ   Úencoder_embed_tokensÚdecoder_embed_tokensrŸ   s        €r#   r�   zFSMTModel.__init__  s�   ø€ Ü‰Ñ˜Ô à×)Ñ)ˆÜ!Ÿ|™|¨F×,AÑ,AÀ6Ç>Á>ÐS^Ó_ÐÜ!Ÿ|™|¨F×,AÑ,AÀ6Ç>Á>ÐS^Ó_Ðä" 6Ð+?Ó@ˆŒÜ" 6Ð+?Ó@ˆŒð 	�‰Õr%   c                 ó   — | j                   S rÌ   )r\  r  s    r#   Úget_encoderzFSMTModel.get_encoder  ó   € Ø�|‰|Ðr%   c                 ó   — | j                   S rÌ   )r]  r  s    r#   Úget_decoderzFSMTModel.get_decoder"  rc  r%   c                 ó  — | j                   j                  ri| j                  | j                  j                  | j                  «       «       | j                  | j                  j                  | j                  «       «       y y rÌ   )r?   Útie_word_embeddingsÚ_tie_or_clone_weightsr]  r·   Úget_input_embeddingsr  r  s    r#   r  zFSMTModel._tie_weights%  sZ   € Ø�;‰;×*Ò*Ø×&Ñ& t§|¡|×'@Ñ'@À$×B[ÑB[ÓB]Ô^Ø×&Ñ& t§|¡|×'EÑ'EÀt×G`ÑG`ÓGbÕcð +r%   )Ú
checkpointÚoutput_typern   r@   r"   rA   Údecoder_attention_maskrÅ   Údecoder_head_maskr  Úencoder_outputsr  r	  r¥   rÆ   rÄ   Údecoder_inputs_embedsrÇ   r2  c                 óv  — |€d}
|�|n| j                   j                  }|�|n| j                   j                  }|
�|
n| j                   j                  }
|�|n| j                   j                  }|
sI|�Gt        | j                   |||| j                  j                  j                  j                  ¬«      \  }}}nd\  }}|€|€t        d«      ‚|€| j                  |||||||¬«      }nI|rGt        |t        «      s7t        |d   t        |«      dkD  r|d   nd t        |«      dkD  r|d   nd ¬	«      }| j                  ||d   |||||||	|
|||¬
«      }|s||z   S t        |j                   |j"                  |j$                  |j&                  |j(                  |j                   |j$                  |j&                  ¬«      S )NF)rA   rB   rC   ©NNzIMake sure that `decoder_input_ids` or `decoder_inputs_embeds` are passed.)r@   r"   rÄ   rÅ   r¥   rÆ   rÇ   r   r   r   rÒ   )	r  rÄ   rÅ   r  r  r	  r¥   rÆ   rÇ   )rÓ   r  Údecoder_hidden_statesÚdecoder_attentionsr  Úencoder_last_hidden_stateró   Úencoder_attentions)r?   r¥   rÆ   r	  Úuse_return_dictrG   r]  r·   rS   r7   rÖ   r\  rQ   r   rØ   r   rÓ   r  rÔ   rÕ   r  )r^   r@   r"   rA   rl  rÅ   rm  r  rn  r  r	  r¥   rÆ   rÄ   ro  rÇ   rB   rF   Údecoder_outputss                      r#   r­   zFSMTModel.forward*  sÿ  € ð0 Ð$ØˆIà1BÐ1NÑ-ÐTX×T_ÑT_×TqÑTqÐà$8Ð$DÑ È$Ï+É+×JjÑJjð 	ð "+Ð!6‘I¸D¿K¹K×<QÑ<Qˆ	Ø%0Ð%<‘kÀ$Ç+Á+×B]ÑB]ˆñ ˜YÐ2ÜC_Ø—‘ØØ"3Ø%;Ø"&§,¡,×";Ñ";×"BÑ"B×"HÑ"HôDÑ@ÐÐ3±[ð 1;Ñ-Ð  +àÐ$Ð)>Ð)FÜÐhÓiÐiàÐ"Ø"Ÿl™lØ#Ø-Ø+Ø#Ø"3Ø%9Ø'ð +ó ‰Oñ ¤¨O¼_Ô!MÜ-Ø"1°!Ñ"4Ü47¸Ó4HÈ1Ò4L˜o¨aÒ0ÐRVÜ14°_Ó1EÈÒ1I˜?¨1Ò-ÈtôˆOð Ÿ,™,ØØ˜AÑØØ Ø +Ø/Ø'Ø!5Ø+ØØ/Ø!5Ø#ð 'ó 
ˆñ  Ø" _Ñ4Ð4ä!Ø-×?Ñ?Ø+×;Ñ;Ø"1×"?Ñ"?Ø.×9Ñ9Ø,×=Ñ=Ø&5×&GÑ&GØ"1×"?Ñ"?Ø.×9Ñ9ô	
ð 		
r%   c                 ó.   — | j                   j                  S rÌ   ©r\  r·   r  s    r#   ri  zFSMTModel.get_input_embeddingsŒ  ó   € Ø�|‰|×(Ñ(Ð(r%   c                 ó&   — || j                   _        y rÌ   ry  ©r^   Úvalues     r#   Úset_input_embeddingszFSMTModel.set_input_embeddings�  ó   € Ø$)ˆ�‰Õ!r%   c                 ó.   — | j                   j                  S rÌ   ©r]  r·   r  s    r#   Úget_output_embeddingszFSMTModel.get_output_embeddings’  rz  r%   c                 ó&   — || j                   _        y rÌ   r�  r|  s     r#   Úset_output_embeddingszFSMTModel.set_output_embeddings•  r  r%   )NNNNNNNNNNNNNN)rk   rl   rm   Ú_tied_weights_keysr   r�   rb  re  r  r   ÚFSMT_INPUTS_DOCSTRINGr   Ú_CHECKPOINT_FOR_DOCr   Ú_CONFIG_FOR_DOCr+   Ú
LongTensorr   r	   Ú
BoolTensorr   r  rå   r   r­   ri  r~  r‚  r„  r¯   r°   s   @r#   rV  rV    sé  ø„ ð
 8Ð9[Ð\Ðð˜zõ òòòdñ
 +Ð+@ÓAÙØ&Ø&Ø$ôð 26Ø8<Ø=AØ,0Ø48Ø7;Ø>BØ>BØ$(Ø,0Ø/3Ø59Ø=AØ&*ñ!Z
à×#Ñ#ðZ
ð ! §¡Ñ.ðZ
ð $ E×$4Ñ$4Ñ5ð	Z
ð
 !)¨×)9Ñ)9Ñ :ðZ
ð ˜EŸL™LÑ)ðZ
ð $ E§L¡LÑ1ðZ
ð ' u§|¡|Ñ4ðZ
ð " %¨×(9Ñ(9Ñ":Ñ;ðZ
ð " %¨×(9Ñ(9Ñ":Ñ;ðZ
ð ˜D‘>ðZ
ð $ D™>ðZ
ð ' t™nðZ
ð   × 1Ñ 1Ñ2ðZ
ð  (¨×(9Ñ(9Ñ:ðZ
ð  ˜d‘^ð!Z
ð" 
ˆu�U—\‘\Ñ"Ð$6Ð6Ñ	7ò#Z
óó BðZ
òx)ò*ò)ö*r%   rV  zLThe FSMT Model with a language modeling head. Can be used for summarization.c            (       ó¬  ‡ — e Zd ZdZddgZdefˆ fd„Z ee«       e	e
e¬«       ee«      	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 ddeej                      deej"                     d	eej                      d
eej$                     deej"                     deej"                     deej"                     deeej(                        deeej(                        deej"                     deej"                     deej                      dee   dee   dee   dee   deeej"                     e
f   f"d„«       «       «       Zdej"                  fd„Zed„ «       Zd„ Zd„ Zd„ Zd„ Zˆ xZS ) ÚFSMTForConditionalGenerationrJ   rW  rX  r?   c                 óh   •— t         ‰| �  |«       t        |«      }|| _        | j	                  «        y rÌ   )rŽ   r�   rV  rJ   r^  )r^   r?   Ú
base_modelrŸ   s      €r#   r�   z%FSMTForConditionalGeneration.__init__   s-   ø€ Ü‰Ñ˜Ô Ü˜vÓ&ˆ
ØˆŒ
ð 	�‰Õr%   )rk  rn   r@   r"   rA   rl  rÅ   rm  r  rn  r  rÄ   ro  Úlabelsr	  r¥   rÆ   rÇ   r2  c                 óú  — |�|n| j                   j                  }|�d}| j                  ||
|||||||||	||||¬«      }|d   }d}|�Ft        «       } ||j	                  d| j                   j
                  «      |j	                  d«      «      }|s|f|dd z   }|�|f|z   S |S t        |||j                  |j                  |j                  |j                  |j                  |j                  |j                  ¬«	      S )aÀ  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

        Returns:

        NF)rÄ   r"   rA   ro  rn  rl  rÅ   rm  r  r  r	  r¥   rÆ   rÇ   r   r)   r   )	ÚlossÚlogitsr  rr  rs  r  rt  ró   ru  )r?   rv  rJ   r   r/  r[  r   r  rr  rs  r  rt  ró   ru  )r^   r@   r"   rA   rl  rÅ   rm  r  rn  r  rÄ   ro  r�  r	  r¥   rÆ   rÇ   ÚoutputsÚ	lm_logitsÚmasked_lm_lossÚloss_fctÚoutputs                         r#   r­   z$FSMTForConditionalGeneration.forward¨  s1  € ð> &1Ð%<‘kÀ$Ç+Á+×B]ÑB]ˆàÐØˆIà—*‘*ØØ'Ø)Ø/Ø"7Ø+Ø#9ØØ/Ø!5Ø+ØØ/Ø!5Ø#ð ó 
ˆð" ˜A‘Jˆ	àˆØÐÜ'Ó)ˆHá% i§n¡n°R¸¿¹×9SÑ9SÓ&TÐV\×VaÑVaÐbdÓVeÓfˆNáØ�\ G¨A¨B KÑ/ˆFØ3AÐ3M�^Ð%¨Ñ.ÐYÐSYÐYäØØØ#×3Ñ3Ø")×"?Ñ"?Ø&×9Ñ9Ø$×5Ñ5Ø&-×&GÑ&GØ")×"?Ñ"?Ø&×9Ñ9ô

ð 
	
r%   c                 óB   — t        || j                  j                  «      S rÌ   )r9   r?   r8   )r^   r�  s     r#   Ú%prepare_decoder_input_ids_from_labelszBFSMTForConditionalGeneration.prepare_decoder_input_ids_from_labelsõ  s   € Ü! &¨$¯+©+×*BÑ*BÓCÐCr%   c           
      ó    — g }| D ]@  }|j                  «       D ��ci c]  \  }}|t        ||«      “Œ }}}|j                  |«       ŒB |S c c}}w rÌ   )r  r#  r  )r  Úbeam_idxÚreordered_pastr  Úattn_keyr  Úlayer_past_news          r#   Ú_reorder_cachez+FSMTForConditionalGeneration._reorder_cacheø  sj   € àˆØ)ò 	2ˆJð ]g×\lÑ\lÓ\n÷ÙDXÀHÈj�œ/¨*°hÓ?Ñ?ðˆNñ ð ×!Ñ! .Õ1ð	2ð Ðùó	s   ›A
c                 ó.   — | j                   j                  S rÌ   )rJ   r\  r  s    r#   rb  z(FSMTForConditionalGeneration.get_encoder  ó   € Ø�z‰z×!Ñ!Ð!r%   c                 ó.   — | j                   j                  S rÌ   )rJ   r]  r  s    r#   re  z(FSMTForConditionalGeneration.get_decoder  r¡  r%   c                 óB   — | j                   j                  j                  S rÌ   ©rJ   r]  r·   r  s    r#   r‚  z2FSMTForConditionalGeneration.get_output_embeddings	  s   € Ø�z‰z×!Ñ!×.Ñ.Ð.r%   c                 ó:   — || j                   j                  _        y rÌ   r¤  r|  s     r#   r„  z2FSMTForConditionalGeneration.set_output_embeddings  s   € Ø*/ˆ�
‰
×ÑÕ'r%   )NNNNNNNNNNNNNNNN) rk   rl   rm   ro   r…  r   r�   r   r†  r   r   rˆ  r   ÚFSMT_GENERATION_EXAMPLEr   r+   r‰  r	   rŠ  r   r  rå   r   r­   r™  ÚstaticmethodrŸ  rb  re  r‚  r„  r¯   r°   s   @r#   rŒ  rŒ  ™  s)  ø„ ð  ÐØ7Ð9[Ð\Ðð˜zõ ñ +Ð+@ÓAÙ¨?ÈÔYÙÐ/Ó0ð 15Ø15Ø8<Ø=AØ,0Ø48Ø7;Ø>BØ>BØ04Ø8<Ø-1Ø$(Ø,0Ø/3Ø&*ñ#H
à˜E×,Ñ,Ñ-ðH
ð ! §¡Ñ.ðH
ð $ E×$4Ñ$4Ñ5ð	H
ð
 !)¨×)9Ñ)9Ñ :ðH
ð ˜EŸL™LÑ)ðH
ð $ E§L¡LÑ1ðH
ð ' u§|¡|Ñ4ðH
ð " %¨×(9Ñ(9Ñ":Ñ;ðH
ð " %¨×(9Ñ(9Ñ":Ñ;ðH
ð   §¡Ñ-ðH
ð  (¨¯©Ñ5ðH
ð ˜×)Ñ)Ñ*ðH
ð ˜D‘>ðH
ð $ D™>ðH
ð  ' t™nð!H
ð" ˜d‘^ð#H
ð$ 
ˆu�U—\‘\Ñ" OÐ3Ñ	4ò%H
ó 1ó Zó BðH
ðTD¸E¿L¹Ló Dð ñó ðò"ò"ò/ö0r%   rŒ  c                   ót   ‡ — e Zd ZdZˆ fd„Zd„ Zed„ «       Zedefd„«       Z		 	 d
de
e   de
e   fˆ fd	„Zˆ xZS )rX   a<  
    This module produces sinusoidal positional embeddings of any length.

    We don't want to save the weight of this embedding since it's not trained (deterministic) and it can be huge.

    Padding symbols are ignored.

    These embeddings get automatically extended in forward if more positions is needed.
    c                 ó(   •— t         ‰| �  |||«       y rÌ   )rŽ   r�   )r^   Únum_positionsr¸   rZ   rŸ   s       €r#   r�   z&SinusoidalPositionalEmbedding.__init__  s   ø€ Ü‰Ñ˜¨°{ÕCr%   c                 ó*  — | j                  |||«      }|j                  | j                  j                  | j                  j                  ¬«      }t        j                  |«      | _        | j                  j                  «        d| j                  _        y )N)r7   r(   F)	rY   r>   rS   r7   r(   r
   r[   r\   rO   )r^   rª  r¸   rZ   rS   s        r#   Úmake_weightz)SinusoidalPositionalEmbedding.make_weight  sh   € Ø×#Ñ# M°=À+ÓNˆà—‘ §¡×!2Ñ!2¸4¿;¹;×;MÑ;M�ÓNˆÜ—l‘l 6Ó*ˆŒØ�‰×ÑÔØ$)ˆ�‰Õ!r%   c                 ó   — |dz  }t        j                  d«      |dz
  z  }t        j                  t        j                  |t        j
                  ¬«      j                  «       | z  «      }t        j                  | t        j
                  ¬«      j                  «       j                  d«      |j                  d«      z  }t        j                  t        j                  |«      t        j                  |«      gd¬«      j                  | d«      }|dz  dk(  r-t        j                  |t        j                  | d«      gd¬«      }|�	d||dd…f<   |S )	zÊ
        Build sinusoidal embeddings.

        This matches the implementation in tensor2tensor, but differs slightly from the description in Section 3.5 of
        "Attention Is All You Need".
        r   i'  r   r6   r   r   r)   N)rº   Úlogr+   Úexpr,   Úint64rM  r.   rG  ÚsinÚcosr/  r=   )Únum_embeddingsr¸   rZ   Úhalf_dimrt   s        r#   rY   z+SinusoidalPositionalEmbedding.get_embedding&  s  € ð ! AÑ%ˆÜ�h‰h�u‹o ¨A¡Ñ.ˆÜ�i‰iœŸ™ X´U·[±[ÔA×GÑGÓIÈSÈDÑPÓQˆÜ�l‰l˜>´·±Ô=×CÑCÓE×OÑOÐPQÓRÐUX×UbÑUbÐcdÓUeÑeˆÜ�i‰iœŸ™ 3›¬¯©°3«Ð8¸aÔ@×EÑEÀnÐVXÓYˆØ˜1Ñ Ò!ä—)‘)˜S¤%§+¡+¨n¸aÓ"@ÐAÀqÔIˆCØÐ"Ø"#ˆC�šQ�ÑØˆ
r%   rZ   c                 ó´   — | j                  |«      j                  «       }t        j                  |d¬«      j	                  |«      |z  j                  «       |z   S )z™
        Replace non-padding symbols with their position numbers.

        Position numbers begin at padding_idx+1. Padding symbols are ignored.
        r   r   )rh   Úintr+   ÚcumsumrO  Úlong)rg   rZ   r3   s      r#   Úmake_positionsz,SinusoidalPositionalEmbedding.make_positions:  sJ   € ð �y‰y˜Ó%×)Ñ)Ó+ˆÜ—‘˜T qÔ)×1Ñ1°$Ó7¸$Ñ>×DÑDÓFÈÑTÐTr%   Úincremental_stateÚtimestepc                 ó,  •— |j                   dd \  }}| j                  dz   |z   }|| j                  j                  d«      kD  r'| j	                  || j
                  | j                  «       | j                  || j                  «      }t        ‰| �!  |«      S )z/Input is expected to be of size [bsz x seqlen].Nr   r   r   )	r*   rZ   rS   r:   r¬  r¸   r¹  rŽ   r­   )	r^   Úinputrº  r»  rD   r0  Úmax_posr  rŸ   s	           €r#   r­   z%SinusoidalPositionalEmbedding.forwardH  s‰   ø€ ð —{‘{ 2 A�‰ˆˆWØ×"Ñ" QÑ&¨Ñ0ˆØ�T—[‘[×%Ñ% aÓ(Ò(à×Ñ˜W d×&8Ñ&8¸$×:JÑ:JÔKØ×'Ñ'¨¨t×/?Ñ/?Ó@ˆ	Ü‰w‰˜yÓ)Ð)r%   rq  )rk   rl   rm   rä   r�   r¬  r§  rY   r¶  r¹  r   r   r	   r­   r¯   r°   s   @r#   rX   rX     st   ø„ ñôDò*ð ñó ðð& ðU¨Cò Uó ðUð  ,0Ø%)ñ	*ð $ C™=ð*ð ˜6Ñ"÷	*ñ *r%   rX   )rŒ  rV  rI   )r   )r   )Drä   rº   Útypingr   r   r   r   r   r   r+   r	   r
   Útorch.nnr   r   Úactivationsr   Ú
generationr   Úintegrations.deepspeedr   Úmodeling_outputsr   r   r   r   Úmodeling_utilsr   Úutilsr   r   r   r   r   r   Úconfiguration_fsmtr   Ú
get_loggerrk   Úloggerr‡  rˆ  ÚFSMT_START_DOCSTRINGr¦  r†  r$   r4   Úfloat32rG   rI   rx   r}   r9   r;   ÚModuler‹   r²   rç   rù   r#  r’   r<   rT  rV  rŒ  r]   rX   Ú__all__rq   r%   r#   ú<module>rÎ     s§  ðñ8 gã ß :× :ã ß ß 0å !Ý )Ý @÷ó õ .÷÷ õ +ð 
ˆ×	Ñ	˜HÓ	%€à,Ð Ø€ðfCðLÐ ð Ð ð&PÐ òf ó'ð ØØ—m‘mó@ô4˜/ô ò>òGò

óô.�2—9‘9ô .ôbtm�"—)‘)ô tmônU
�2—9‘9ô U
ôpv
�"—)‘)ô v
òrôc*�—	‘	ô c*òL@ò%ñ ØXØóôG*Ð#ó G*ó	ðG*ñT ØRÐThóôq0Ð#6¸ó q0óðq0ôhE* B§L¡Lô E*òP O�r%   