Ë
    T^(he×  ã                   ó:  — d Z ddlZddlZddlmZ ddlmZmZmZm	Z	 ddl
ZddlZddlmZ ddlmZmZmZ ddlmZ dd	lmZ dd
lmZmZmZmZmZmZ ddlmZmZm Z  ddl!m"Z"m#Z#m$Z$ ddl%m&Z&m'Z'm(Z(m)Z)m*Z*m+Z+ ddl,m-Z-  e*j\                  e/«      Z0dZ1dZ2d„ Z3d6d„Z4 G d„ dejj                  «      Z6 G d„ dejj                  «      Z7 G d„ de«      Z8e G d„ de&«      «       Z9dZ:dZ; e(de:«       G d„ d e8«      «       Z< G d!„ d"ejj                  «      Z= e(d#e:«       G d$„ d%e8e«      «       Z> e(d&e:«       G d'„ d(e8«      «       Z? e(d)e:«       G d*„ d+e8«      «       Z@ e(d,e:«       G d-„ d.e8«      «       ZA e(d/e:«       G d0„ d1e8«      «       ZB e(d2e:«       G d3„ d4e8«      «       ZCg d5¢ZDy)7z
PyTorch XLM model.
é    N)Ú	dataclass)ÚDictÚOptionalÚTupleÚUnion)Únn)ÚBCEWithLogitsLossÚCrossEntropyLossÚMSELossé   )Úgelu)ÚGenerationMixin)ÚBaseModelOutputÚMaskedLMOutputÚMultipleChoiceModelOutputÚQuestionAnsweringModelOutputÚSequenceClassifierOutputÚTokenClassifierOutput)ÚPreTrainedModelÚSequenceSummaryÚ	SQuADHead)Úapply_chunking_to_forwardÚ find_pruneable_heads_and_indicesÚprune_linear_layer)ÚModelOutputÚadd_code_sample_docstringsÚadd_start_docstringsÚ%add_start_docstrings_to_model_forwardÚloggingÚreplace_return_docstringsé   )Ú	XLMConfigzFacebookAI/xlm-mlm-en-2048r"   c                 ó  — t        j                  t        | «      D ��cg c];  }t        |«      D �cg c]$  }|t        j                  dd|dz  z  |z  «      z  ‘Œ& c}‘Œ= c}}«      }d|_        t        j                  t        j                  |d d …dd d…f   «      «      |d d …dd d…f<   t        j                  t        j                  |d d …dd d…f   «      «      |d d …dd d…f<   |j                  «        y c c}w c c}}w )Ni'  é   Fr   r!   )
ÚnpÚarrayÚrangeÚpowerÚrequires_gradÚtorchÚFloatTensorÚsinÚcosÚdetach_)Ún_posÚdimÚoutÚposÚjÚposition_encs         úb/var/www/skyplay_api_hub/venv/lib/python3.12/site-packages/transformers/models/xlm/modeling_xlm.pyÚcreate_sinusoidal_embeddingsr6   :   sØ   € Ü—8‘8ÔhmÐnsÓht×uÐadÔQVÐWZÓQ[Ö\ÈA˜c¤B§H¡H¨U°A¸¸a¹±LÀ3Ñ4FÓ$GÓGÔ\ÓuÓv€LØ€CÔÜ×$Ñ$¤R§V¡V¨Lº¸A¸D¸q¸D¸Ñ,AÓ%BÓC€CŠˆ1ˆ4ˆaˆ4ˆ�LÜ×$Ñ$¤R§V¡V¨Lº¸A¸D¸q¸D¸Ñ,AÓ%BÓC€CŠˆ1ˆ4ˆaˆ4ˆ�LØ‡K�K…Mùò	 ]ùÓus   žC;
°)C6ÁC;
Ã6C;
c                 ó°  — t        j                  | t         j                  |j                  ¬«      }|�|}n/|j	                  «       j                  «       | k  sJ ‚||dd…df   k  }|j                  d«      }|r'|dddd…f   j                  || d«      |ddd…df   k  }n|}|j                  «       || fk(  sJ ‚|du s|j                  «       || | fk(  sJ ‚||fS )zH
    Generate hidden states mask, and optionally an attention mask.
    ©ÚdtypeÚdeviceNr   r!   F)r*   ÚarangeÚlongr:   ÚmaxÚitemÚsizeÚrepeat)ÚslenÚlengthsÚcausalÚpadding_maskÚalenÚmaskÚbsÚ	attn_masks           r5   Ú	get_masksrI   B   så   € ô �<‰<˜¤E§J¡J°w·~±~ÔF€DØÐØ‰à�{‰{‹}×!Ñ!Ó# tÒ+Ð+Ð+Ø�gša ˜gÑ&Ñ&ˆð 
�‰�a‹€BÙØ˜˜t¢Q˜Ñ'×.Ñ.¨r°4¸Ó;¸tÀDÊ!ÈTÀMÑ?RÑR‰	àˆ	ð �9‰9‹;˜2˜t˜*Ò$Ð$Ð$Ø�U‰?˜iŸn™nÓ.°2°t¸TÐ2BÒBÐBÐBà�ˆ?Ðó    c                   óN   ‡ — e Zd Z ej                  «       Zˆ fd„Zd„ Zdd„Zˆ xZ	S )ÚMultiHeadAttentionc                 óÎ  •— t         ‰| �  «        t        t        j                  «      | _        || _        || _        |j                  | _	        | j                  | j                  z  dk(  sJ ‚t        j                  ||«      | _        t        j                  ||«      | _        t        j                  ||«      | _        t        j                  ||«      | _        t!        «       | _        y )Nr   )ÚsuperÚ__init__ÚnextrL   ÚNEW_IDÚlayer_idr0   Ún_headsÚattention_dropoutÚdropoutr   ÚLinearÚq_linÚk_linÚv_linÚout_linÚsetÚpruned_heads)ÚselfrS   r0   ÚconfigÚ	__class__s       €r5   rO   zMultiHeadAttention.__init__^   s©   ø€ Ü‰ÑÔÜÔ/×6Ñ6Ó7ˆŒØˆŒØˆŒØ×/Ñ/ˆŒØ�x‰x˜$Ÿ,™,Ñ&¨!Ò+Ð+Ð+ä—Y‘Y˜s CÓ(ˆŒ
Ü—Y‘Y˜s CÓ(ˆŒ
Ü—Y‘Y˜s CÓ(ˆŒ
Ü—y‘y  cÓ*ˆŒÜ›EˆÕrJ   c                 ó  — | j                   | j                  z  }t        |«      dk(  ry t        || j                  || j                  «      \  }}t        | j                  |«      | _        t        | j                  |«      | _        t        | j                  |«      | _        t        | j                  |d¬«      | _	        | j                  t        |«      z
  | _        || j                  z  | _         | j                  j                  |«      | _        y )Nr   r!   ©r0   )r0   rS   Úlenr   r\   r   rW   rX   rY   rZ   Úunion)r]   ÚheadsÚattention_head_sizeÚindexs       r5   Úprune_headszMultiHeadAttention.prune_headsl   sÊ   € Ø"Ÿh™h¨$¯,©,Ñ6ÐÜˆu‹:˜Š?ØÜ7¸¸t¿|¹|ÐM`Ðbf×bsÑbsÓt‰ˆˆuä'¨¯
©
°EÓ:ˆŒ
Ü'¨¯
©
°EÓ:ˆŒ
Ü'¨¯
©
°EÓ:ˆŒ
Ü)¨$¯,©,¸À1ÔEˆŒà—|‘|¤c¨%£jÑ0ˆŒØ&¨¯©Ñ5ˆŒØ ×-Ñ-×3Ñ3°EÓ:ˆÕrJ   c                 ó¦  ‡ ‡‡— |j                  «       \  Š}}|€|€|n|d   |z   }	n|j                  d«      }	‰ j                  }
‰ j                  |
z  Š|j                  «       dk(  r‰d||	fn‰dd|	f}ˆˆˆ fd„}ˆˆˆ fd„} |‰ j                  |«      «      }|€/ |‰ j	                  |«      «      } |‰ j                  |«      «      }nB|�‰ j                  |vr2|x}} |‰ j	                  |«      «      } |‰ j                  |«      «      }|�x‰ j                  |v rY|€E|‰ j                     \  }}t        j                  |gd¬«      }t        j                  |gd¬«      }n|‰ j                     \  }}f|‰ j                  <   |t        j                  ‰«      z  }t        j                  |j                  dd«      «      }|dk(  j                  |«      j                  |«      }|j                  |t        j                   |j"                  «      j$                  «       t&        j(                  j+                  |j-                  «       d	¬«      j/                  |«      }t&        j(                  j1                  |‰ j0                  ‰ j2                  ¬
«      }|�||z  }t        j                  |«      } ||«      }‰ j5                  |«      f}|r||fz   }|S )zd
        Self-attention (if kv is None) or attention over source sentence (provided by kv).
        rA   r!   r   c                 ó`   •— | j                  ‰d‰j                  ‰«      j                  dd«      S )Ú
projectionéÿÿÿÿr!   r$   )ÚviewrS   Ú	transpose©ÚxrG   Údim_per_headr]   s    €€€r5   Úshapez)MultiHeadAttention.forward.<locals>.shape‹   s)   ø€ à—6‘6˜"˜b $§,¡,°Ó=×GÑGÈÈ1ÓMÐMrJ   c                 ó€   •— | j                  dd«      j                  «       j                  ‰d‰j                  ‰z  «      S )zcompute contextr!   r$   rk   )rm   Ú
contiguousrl   rS   rn   s    €€€r5   Úunshapez+MultiHeadAttention.forward.<locals>.unshape�   s5   ø€ à—;‘;˜q !Ó$×/Ñ/Ó1×6Ñ6°r¸2¸t¿|¹|ÈlÑ?ZÓ[Ð[rJ   r$   ra   r   rk   ©ÚpÚtraining)r?   rS   r0   rW   rX   rY   rR   r*   ÚcatÚmathÚsqrtÚmatmulrm   rl   Ú	expand_asÚmasked_fill_Úfinfor9   Úminr   Ú
functionalÚsoftmaxÚfloatÚtype_asrU   rw   rZ   )r]   ÚinputrF   ÚkvÚcacheÚ	head_maskÚoutput_attentionsÚqlenr0   ÚklenrS   Úmask_reshaperq   rt   ÚqÚkÚvÚk_Úv_ÚscoresÚweightsÚcontextÚoutputsrG   rp   s   `                      @@r5   ÚforwardzMultiHeadAttention.forward{   s„  ú€ ð Ÿ
™
›‰ˆˆD�#Øˆ:Ø ˜=‘4¨e°F©m¸dÑ.B‰Dà—7‘7˜1“:ˆDà—,‘,ˆØ—x‘x 7Ñ*ˆØ.2¯h©h«j¸Aªo˜˜A˜t TÑ*ÀBÈÈ1ÈdÐCSˆö	Nö	\ñ �$—*‘*˜UÓ#Ó$ˆØˆ:Ù�d—j‘j Ó'Ó(ˆAÙ�d—j‘j Ó'Ó(‰AØˆ]˜dŸm™m°5Ñ8ØˆJˆA�Ù�d—j‘j “mÓ$ˆAÙ�d—j‘j “mÓ$ˆAàÐØ�}‰} Ñ%Ø�:Ø" 4§=¡=Ñ1‘F�B˜ÜŸ	™	 2 q '¨qÔ1�AÜŸ	™	 2 q '¨qÔ1‘Aà  §¡Ñ/‘D�A�qØ$% q 6ˆE�$—-‘-Ñ à”—	‘	˜,Ó'Ñ'ˆÜ—‘˜a §¡¨Q°Ó!2Ó3ˆØ˜‘	×Ñ Ó-×7Ñ7¸Ó?ˆØ×Ñ˜D¤%§+¡+¨f¯l©lÓ";×"?Ñ"?Ô@ä—-‘-×'Ñ'¨¯©«¸BÐ'Ó?×GÑGÈÓOˆÜ—-‘-×'Ñ'¨°4·<±<È$Ï-É-Ð'ÓXˆð Ð Ø 	Ñ)ˆGä—,‘,˜w¨Ó*ˆÙ˜'Ó"ˆà—<‘< Ó(Ð*ˆÙØ  
Ñ*ˆGØˆrJ   )NNNF)
Ú__name__Ú
__module__Ú__qualname__Ú	itertoolsÚcountrQ   rO   rg   r•   Ú__classcell__©r_   s   @r5   rL   rL   [   s    ø„ ØˆY�_‰_Ó€Fô"ò;÷=rJ   rL   c                   ó*   ‡ — e Zd Zˆ fd„Zd„ Zd„ Zˆ xZS )ÚTransformerFFNc                 óB  •— t         ‰| �  «        |j                  | _        t        j                  ||«      | _        t        j                  ||«      | _        |j                  rt        nt        j                  j                  | _        |j                  | _        d| _        y ©Nr!   )rN   rO   rU   r   rV   Úlin1Úlin2Úgelu_activationr   r€   ÚreluÚactÚchunk_size_feed_forwardÚseq_len_dim)r]   Úin_dimÚ
dim_hiddenÚout_dimr^   r_   s        €r5   rO   zTransformerFFN.__init__¼   sn   ø€ Ü‰ÑÔØ—~‘~ˆŒÜ—I‘I˜f jÓ1ˆŒ	Ü—I‘I˜j¨'Ó2ˆŒ	Ø!×1Ò1•4´r·}±}×7IÑ7IˆŒØ'-×'EÑ'EˆÔ$ØˆÕrJ   c                 óZ   — t        | j                  | j                  | j                  |«      S ©N)r   Úff_chunkr¦   r§   )r]   r„   s     r5   r•   zTransformerFFN.forwardÅ   s%   € Ü(¨¯©¸×8TÑ8TÐVZ×VfÑVfÐhmÓnÐnrJ   c                 óØ   — | j                  |«      }| j                  |«      }| j                  |«      }t        j                  j                  || j
                  | j                  ¬«      }|S )Nru   )r¡   r¥   r¢   r   r€   rU   rw   )r]   r„   ro   s      r5   r­   zTransformerFFN.ff_chunkÈ   sR   € Ø�I‰I�eÓˆØ�H‰H�Q‹KˆØ�I‰I�a‹LˆÜ�M‰M×!Ñ! ! t§|¡|¸d¿m¹mÐ!ÓLˆØˆrJ   )r–   r—   r˜   rO   r•   r­   r›   rœ   s   @r5   rž   rž   »   s   ø„ ôòoörJ   rž   c                   óD   ‡ — e Zd ZdZeZdZdZˆ fd„Ze	d„ «       Z
d„ Zˆ xZS )ÚXLMPreTrainedModelz†
    An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained
    models.
    NÚtransformerc                 ó$   •— t        ‰| �  |i |¤Ž y r¬   )rN   rO   )r]   ÚinputsÚkwargsr_   s      €r5   rO   zXLMPreTrainedModel.__init__Ú   s   ø€ Ü‰Ñ˜&Ð+ FÓ+rJ   c                 ó&  — t        j                  g d¢g d¢g d¢g«      }t        j                  g d¢g d¢g d¢g«      }| j                  j                  r8| j                  j                  dkD  rt        j                  g d¢g d¢g d¢g«      }nd }|||dœS )	N)é   é   r   r   r!   )r!   r$   r   r   r   )r   r   r   é   é   )r!   r!   r   r   r!   )r!   r!   r!   r   r   )r!   r   r   r!   r!   r!   )Ú	input_idsÚattention_maskÚlangs)r*   Útensorr^   Úuse_lang_embÚn_langs)r]   Úinputs_listÚ
attns_listÚ
langs_lists       r5   Údummy_inputszXLMPreTrainedModel.dummy_inputsÝ   sr   € ä—l‘l¢O²_ÂoÐ#VÓWˆÜ—\‘\¢?²OÂ_Ð"UÓVˆ
Ø�;‰;×#Ò#¨¯©×(;Ñ(;¸aÒ(?ÜŸ™¢ºÊÐ&YÓZ‰JàˆJØ(¸JÐQ[Ñ\Ð\rJ   c                 ó„  — t        |t        j                  «      rŸ| j                  �V| j                  j                  �@t        j
                  j                  |j                  d| j                  j                  ¬«       |j                  �1|j                  j                  |j                     j                  «        t        |t        j                  «      r˜| j                  �Œ| j                  j                  �vt        j
                  j                  |j                  d| j                  j                  ¬«       |j                  �*t        j
                  j                  |j                  d«       t        |t        j                  «      rI|j                  j                  j                  «        |j                  j                  j!                  d«       t        |t"        «      rb| j                  j$                  rKt'        | j                  j(                  | j                  j*                  |j,                  j                  ¬«       yyy)zInitialize the weights.Nr   )ÚmeanÚstdg        g      ð?)r1   )Ú
isinstancer   Ú	Embeddingr^   Úembed_init_stdÚinitÚnormal_ÚweightÚpadding_idxÚdataÚzero_rV   Úinit_stdÚbiasÚ	constant_Ú	LayerNormÚfill_ÚXLMModelÚsinusoidal_embeddingsr6   Úmax_position_embeddingsÚemb_dimÚposition_embeddings)r]   Úmodules     r5   Ú_init_weightsz XLMPreTrainedModel._init_weightsç   sq  € ä�fœbŸl™lÔ+Ø�{‰{Ð&¨4¯;©;×+EÑ+EÐ+QÜ—‘—‘ §¡°A¸4¿;¹;×;UÑ;U�ÔVØ×!Ñ!Ð-Ø—‘×"Ñ" 6×#5Ñ#5Ñ6×<Ñ<Ô>Ü�fœbŸi™iÔ(Ø�{‰{Ð&¨4¯;©;×+?Ñ+?Ð+KÜ—‘—‘ §¡°A¸4¿;¹;×;OÑ;O�ÔPØ—;‘;Ð*Ü—G‘G×%Ñ% f§k¡k°3Ô7Ü�fœbŸl™lÔ+Ø�K‰K×Ñ×"Ñ"Ô$Ø�M‰M×Ñ×$Ñ$ SÔ)Ü�fœhÔ'¨D¯K©K×,MÒ,MÜ(Ø—‘×3Ñ3°T·[±[×5HÑ5HÈf×NhÑNh×NoÑNoöð -NÐ'rJ   )r–   r—   r˜   Ú__doc__r"   Úconfig_classÚload_tf_weightsÚbase_model_prefixrO   ÚpropertyrÃ   rÛ   r›   rœ   s   @r5   r°   r°   Ð   s:   ø„ ñð
 €LØ€OØ%Ðô,ð ñ]ó ð]örJ   r°   c                   óf  — e Zd ZU dZdZeej                     ed<   dZ	eej                     ed<   dZ
eej                     ed<   dZeej                     ed<   dZeej                     ed<   dZeej                     ed<   dZeeej                  d	f      ed
<   dZeeej                  d	f      ed<   y)ÚXLMForQuestionAnsweringOutputa9
  
    Base class for outputs of question answering models using a `SquadHead`.

    Args:
        loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned if both `start_positions` and `end_positions` are provided):
            Classification loss as the sum of start token, end token (and is_impossible if provided) classification
            losses.
        start_top_log_probs (`torch.FloatTensor` of shape `(batch_size, config.start_n_top)`, *optional*, returned if `start_positions` or `end_positions` is not provided):
            Log probabilities for the top config.start_n_top start token possibilities (beam-search).
        start_top_index (`torch.LongTensor` of shape `(batch_size, config.start_n_top)`, *optional*, returned if `start_positions` or `end_positions` is not provided):
            Indices for the top config.start_n_top start token possibilities (beam-search).
        end_top_log_probs (`torch.FloatTensor` of shape `(batch_size, config.start_n_top * config.end_n_top)`, *optional*, returned if `start_positions` or `end_positions` is not provided):
            Log probabilities for the top `config.start_n_top * config.end_n_top` end token possibilities
            (beam-search).
        end_top_index (`torch.LongTensor` of shape `(batch_size, config.start_n_top * config.end_n_top)`, *optional*, returned if `start_positions` or `end_positions` is not provided):
            Indices for the top `config.start_n_top * config.end_n_top` end token possibilities (beam-search).
        cls_logits (`torch.FloatTensor` of shape `(batch_size,)`, *optional*, returned if `start_positions` or `end_positions` is not provided):
            Log probabilities for the `is_impossible` label of the answers.
        hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
            Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
            shape `(batch_size, sequence_length, hidden_size)`.

            Hidden-states of the model at the output of each layer plus the initial embedding outputs.
        attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
            Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
            sequence_length)`.

            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
            heads.
    NÚlossÚstart_top_log_probsÚstart_top_indexÚend_top_log_probsÚend_top_indexÚ
cls_logits.Úhidden_statesÚ
attentions)r–   r—   r˜   rÜ   rã   r   r*   r+   Ú__annotations__rä   rå   Ú
LongTensorræ   rç   rè   ré   r   rê   © rJ   r5   râ   râ   ü   sÇ   … ñð> )-€Dˆ(�5×$Ñ$Ñ
%Ó,Ø7;Ð˜ %×"3Ñ"3Ñ4Ó;Ø26€O�X˜e×.Ñ.Ñ/Ó6Ø59Ð�x × 1Ñ 1Ñ2Ó9Ø04€M�8˜E×,Ñ,Ñ-Ó4Ø.2€J�˜×*Ñ*Ñ+Ó2Ø=A€M�8˜E %×"3Ñ"3°SÐ"8Ñ9Ñ:ÓAØ:>€J�˜˜u×0Ñ0°#Ð5Ñ6Ñ7Ô>rJ   râ   a=  

    This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
    library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
    etc.)

    This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
    Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
    and behavior.

    Parameters:
        config ([`XLMConfig`]): Model configuration class with all the parameters of the model.
            Initializing with a config file does not load the weights associated with the model, only the
            configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
a8  
    Args:
        input_ids (`torch.LongTensor` of shape `({0})`):
            Indices of input sequence tokens in the vocabulary.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are input IDs?](../glossary#input-ids)
        attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*):
            Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:

            - 1 for tokens that are **not masked**,
            - 0 for tokens that are **masked**.

            [What are attention masks?](../glossary#attention-mask)
        langs (`torch.LongTensor` of shape `({0})`, *optional*):
            A parallel sequence of tokens to be used to indicate the language of each token in the input. Indices are
            languages ids which can be obtained from the language names by using two conversion mappings provided in
            the configuration of the model (only provided for multilingual models). More precisely, the *language name
            to language id* mapping is in `model.config.lang2id` (which is a dictionary string to int) and the
            *language id to language name* mapping is in `model.config.id2lang` (dictionary int to string).

            See usage examples detailed in the [multilingual documentation](../multilingual).
        token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*):
            Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
            1]`:

            - 0 corresponds to a *sentence A* token,
            - 1 corresponds to a *sentence B* token.

            [What are token type IDs?](../glossary#token-type-ids)
        position_ids (`torch.LongTensor` of shape `({0})`, *optional*):
            Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
            config.max_position_embeddings - 1]`.

            [What are position IDs?](../glossary#position-ids)
        lengths (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Length of each sentence that can be used to avoid performing attention on padding token indices. You can
            also use *attention_mask* for the same result (see above), kept here for compatibility. Indices selected in
            `[0, ..., input_ids.size(-1)]`.
        cache (`Dict[str, torch.FloatTensor]`, *optional*):
            Dictionary string to `torch.FloatTensor` that contains precomputed hidden states (key and values in the
            attention blocks) as computed by the model (see `cache` output below). Can be used to speed up sequential
            decoding.

            The dictionary object will be modified in-place during the forward pass to add newly computed
            hidden-states.
        head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*):
            Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`:

            - 1 indicates the head is **not masked**,
            - 0 indicates the head is **masked**.

        inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*):
            Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
            is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
            model's internal embedding lookup matrix.
        output_attentions (`bool`, *optional*):
            Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
            tensors for more detail.
        output_hidden_states (`bool`, *optional*):
            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
            more detail.
        return_dict (`bool`, *optional*):
            Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
z]The bare XLM Model transformer outputting raw hidden-states without any specific head on top.c                   óæ  ‡ — e Zd Zˆ fd„Zd„ Zd„ Zd„ Z eej                  d«      «       e
eee¬«      	 	 	 	 	 	 	 	 	 	 	 	 ddeej                      deej                      d	eej                      d
eej                      deej                      deej                      deeeej                   f      deej                      deej                      dee   dee   dee   deeef   fd„«       «       Zˆ xZS )rÕ   c           
      óF	  •— t         ‰| �  |«       |j                  | _        |j                   | _        | j                  rt	        d«      ‚|j
                  | _        |j                  | _        |j                  | _        |j                  | _        |j                  | _	        |j                  | _
        |j                  | _        | j                  dz  | _        |j                  | _        |j                  | _        |j                   | _        |j"                  | _        | j                  | j                  z  dk(  sJ d«       ‚t%        j&                  |j(                  | j                  «      | _        |j                  dkD  r;|j                  r/t%        j&                  | j                  | j                  «      | _        t%        j&                  | j                  | j                  | j                  ¬«      | _        t%        j0                  | j                  |j2                  ¬«      | _        t%        j6                  «       | _        t%        j6                  «       | _        t%        j6                  «       | _        t%        j6                  «       | _        tA        | j                  «      D �]  }| j8                  jC                  tE        | j                  | j                  |¬«      «       | j:                  jC                  t%        j0                  | j                  |j2                  ¬«      «       | j<                  jC                  tG        | j                  | j                  | j                  |¬«      «       | j>                  jC                  t%        j0                  | j                  |j2                  ¬«      «       �Œ tI        |d	«      rœ|jJ                  jM                  «       jO                  «       }i |_%        |D ]h  \  }}| j8                  tQ        |«         j                  |j                  k(  sŒ6| jS                  tQ        |«      tU        tW        tP        |«      «      i«       Œj | jY                  «        | j[                  d
t]        j^                  |j(                  «      ja                  d«      d¬«       y )Nz,Currently XLM can only be used as an encoderr¸   r   z-transformer dim must be a multiple of n_headsr!   )rÍ   )Úeps)r^   r\   Úposition_ids)r!   rk   F)Ú
persistent)1rN   rO   Ú
is_encoderÚ
is_decoderÚNotImplementedErrorrC   r¿   r¾   Ún_wordsÚ	eos_indexÚ	pad_indexrØ   r0   Ú
hidden_dimrS   Ún_layersrU   rT   r   rÈ   r×   rÙ   Úlang_embeddingsÚ
embeddingsrÓ   Úlayer_norm_epsÚlayer_norm_embÚ
ModuleListrê   Úlayer_norm1ÚffnsÚlayer_norm2r'   ÚappendrL   rž   Úhasattrr\   ÚcopyÚitemsÚintrg   ÚlistÚmapÚ	post_initÚregister_bufferr*   r;   Úexpand)r]   r^   Ú_r\   Úlayerrd   r_   s         €r5   rO   zXLMModel.__init__�  sG  ø€ Ü‰Ñ˜Ô ð !×+Ñ+ˆŒØ$×/Ñ/Ð/ˆŒØ�?Š?Ü%Ð&TÓUÐUà—m‘mˆŒð —~‘~ˆŒØ"×/Ñ/ˆÔØ—~‘~ˆŒØ×)Ñ)ˆŒØ×)Ñ)ˆŒð —>‘>ˆŒØŸ(™( Q™,ˆŒØ—~‘~ˆŒØŸ™ˆŒØ—~‘~ˆŒØ!'×!9Ñ!9ˆÔØ�x‰x˜$Ÿ,™,Ñ&¨!Ò+Ð\Ð-\Ó\Ð+ô $&§<¡<°×0NÑ0NÐPT×PXÑPXÓ#YˆÔ Ø�>‰>˜AÒ &×"5Ò"5Ü#%§<¡<°·±¸d¿h¹hÓ#GˆDÔ ÜŸ,™, t§|¡|°T·X±XÈ4Ï>É>ÔZˆŒÜ Ÿl™l¨4¯8©8¸×9NÑ9NÔOˆÔô Ÿ-™-›/ˆŒÜŸ=™=›?ˆÔÜ—M‘M“OˆŒ	ÜŸ=™=›?ˆÔô
 �t—}‘}Ó%ó 	WˆAØ�O‰O×"Ñ"Ô#5°d·l±lÀDÇHÁHÐU[Ô#\Ô]Ø×Ñ×#Ñ#¤B§L¡L°·±¸v×?TÑ?TÔ$UÔVð �I‰I×Ñœ^¨D¯H©H°d·o±oÀtÇxÁxÐX^Ô_Ô`Ø×Ñ×#Ñ#¤B§L¡L°·±¸v×?TÑ?TÔ$UÖVð	Wô �6˜>Ô*Ø!×.Ñ.×3Ñ3Ó5×;Ñ;Ó=ˆLØ"$ˆFÔØ ,ò J‘��uØ—?‘?¤3 u£:Ñ.×6Ñ6¸&¿.¹.ÓHØ×$Ñ$¤c¨%£j´$´s¼3À³Ó2GÐ%HÕIðJð
 	�‰ÔØ×ÑØœEŸL™L¨×)GÑ)GÓH×OÑOÐPWÓXÐejð 	õ 	
rJ   c                 ó   — | j                   S r¬   ©rü   ©r]   s    r5   Úget_input_embeddingszXLMModel.get_input_embeddingsÇ  s   € Ø�‰ÐrJ   c                 ó   — || _         y r¬   r  ©r]   Únew_embeddingss     r5   Úset_input_embeddingszXLMModel.set_input_embeddingsÊ  s	   € Ø(ˆ�rJ   c                 óp   — |j                  «       D ]#  \  }}| j                  |   j                  |«       Œ% y)z�
        Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base
        class PreTrainedModel
        N)r  rê   rg   )r]   Úheads_to_pruner  rd   s       r5   Ú_prune_headszXLMModel._prune_headsÍ  s7   € ð
 +×0Ñ0Ó2ò 	6‰LˆE�5Ø�O‰O˜EÑ"×.Ñ.¨uÕ5ñ	6rJ   úbatch_size, sequence_length©Ú
checkpointÚoutput_typerÝ   rº   r»   r¼   Útoken_type_idsrñ   rB   r†   r‡   Úinputs_embedsrˆ   Úoutput_hidden_statesÚreturn_dictÚreturnc                 ó`  — |
�|
n| j                   j                  }
|�|n| j                   j                  }|�|n| j                   j                  }|�|j	                  «       \  }}n|	j	                  «       d d \  }}|�|j
                  n|	j
                  }|€K|�.|| j                  k7  j                  d¬«      j                  «       }nt        j                  |g|z  |¬«      }|j	                  d«      |k(  sJ ‚|j                  «       j                  «       |k  sJ ‚t        ||| j                  |¬«      \  }}|€| j                  d d …d |…f   }n|j	                  «       ||fk(  sJ ‚|�|j	                  «       ||fk(  sJ ‚| j!                  || j                   j"                  «      }|�H|�F||d   z
  }|d d …| d …f   }|d d …| d …f   }|�|d d …| d …f   }|d d …| d …f   }|d d …| d …f   }|	€| j%                  |«      }	|	| j'                  |«      j)                  |	«      z   }|�/| j*                  r#| j,                  dkD  r|| j/                  |«      z   }|�|| j%                  |«      z   }| j1                  |«      }t2        j4                  j7                  || j6                  | j8                  ¬«      }||j;                  d«      j=                  |j>                  «      z  }|rd	nd }|
rd	nd }tA        | j"                  «      D ]á  }|r||fz   } | jB                  |   |||||   |
¬
«      }|d   }|
r	||d   fz   }t2        j4                  j7                  || j6                  | j8                  ¬«      }||z   } | jD                  |   |«      }| | jF                  |   |«      z   } | jH                  |   |«      }||j;                  d«      j=                  |j>                  «      z  }Œã |r||fz   }|�|dxx   |j	                  d«      z  cc<   |stK        d„ |||fD «       «      S tM        |||¬«      S )Nrk   r!   ra   )r:   r   )rD   rA   ru   rí   )r†   r‡   rˆ   c              3   ó&   K  — | ]	  }|€Œ|–— Œ y ­wr¬   rí   )Ú.0rŽ   s     r5   ú	<genexpr>z#XLMModel.forward.<locals>.<genexpr>a  s   è ø€ ÒY˜qÈ1É=œÑYùs   ‚Š)Úlast_hidden_stateré   rê   )'r^   rˆ   r   Úuse_return_dictr?   r:   rø   Úsumr<   r*   r½   r=   r>   rI   rC   rñ   Úget_head_maskrú   rü   rÙ   r|   r¾   r¿   rû   rþ   r   r€   rU   rw   Ú	unsqueezeÚtor9   r'   rê   r   r  r  Útupler   )r]   rº   r»   r¼   r  rñ   rB   r†   r‡   r  rˆ   r   r!  r´   rG   rA   r:   rF   rH   Ú_slenr½   ré   rê   ÚiÚattn_outputsÚattns                             r5   r•   zXLMModel.forwardÕ  sm  € ð, 2CÐ1NÑ-ÐTX×T_ÑT_×TqÑTqÐà$8Ð$DÑ È$Ï+É+×JjÑJjð 	ð &1Ð%<‘kÀ$Ç+Á+×B]ÑB]ˆàÐ Ø —~‘~Ó'‰HˆB‘à$×)Ñ)Ó+¨C¨RÐ0‰HˆB�à%.Ð%:�×!Ò!À×@TÑ@Tˆàˆ?ØÐ$Ø$¨¯©Ñ6×;Ñ;ÀÐ;ÓB×GÑGÓI‘äŸ,™,¨ v°¡{¸6ÔB�ð �|‰|˜A‹ "Ò$Ð$Ð$Ø�{‰{‹}×!Ñ!Ó# tÒ+Ð+Ð+ô $ D¨'°4·;±;È^Ô\‰ˆˆið
 ÐØ×,Ñ,ªQ°°°¨XÑ6‰Là×$Ñ$Ó&¨2¨t¨*Ò4Ð4Ð4ð ÐØ—:‘:“< B¨ :Ò-Ð-Ð-ð ×&Ñ& y°$·+±+×2FÑ2FÓGˆ	ð Ð Ð!6Ø˜5 ™=Ñ(ˆEØ!¢! e V¡W *Ñ-ˆIØ'ª¨E¨6©7¨
Ñ3ˆLØÐ Øša % ¡˜jÑ)�Øš˜E˜6™7˜
Ñ#ˆDØ!¢! e V¡W *Ñ-ˆIð Ð Ø ŸO™O¨IÓ6ˆMà ×!9Ñ!9¸,Ó!G×!QÑ!QÐR_Ó!`Ñ`ˆØÐ ×!2Ò!2°t·|±|ÀaÒ7GØ˜d×2Ñ2°5Ó9Ñ9ˆFØÐ%Ø˜dŸo™o¨nÓ=Ñ=ˆFØ×$Ñ$ VÓ,ˆÜ—‘×&Ñ& v°·±ÈÏÉÐ&ÓVˆØ�$—.‘. Ó$×'Ñ'¨¯©Ó5Ñ5ˆñ 3™¸ˆÙ,‘R°$ˆ
Ü�t—}‘}Ó%ò 	:ˆAÙ#Ø -°°	Ñ 9�ð .˜4Ÿ?™?¨1Ñ-ØØØØ# A™,Ø"3ôˆLð   ‘?ˆDÙ Ø'¨<¸©?Ð*<Ñ<�
Ü—=‘=×(Ñ(¨°·±ÈÏÉÐ(ÓVˆDØ˜d‘]ˆFØ(�T×%Ñ% aÑ(¨Ó0ˆFð ˜l˜dŸi™i¨™l¨6Ó2Ñ2ˆFØ(�T×%Ñ% aÑ(¨Ó0ˆFØ�d—n‘n RÓ(×+Ñ+¨F¯L©LÓ9Ñ9‰Fð;	:ñ@  Ø)¨V¨IÑ5ˆMð ÐØ�&‹M˜VŸ[™[¨›^Ñ+‹Mñ
 ÜÑY V¨]¸JÐ$GÔYÓYÐYÜ°À}ÐakÔlÐlrJ   )NNNNNNNNNNNN)r–   r—   r˜   rO   r  r  r  r   ÚXLM_INPUTS_DOCSTRINGÚformatr   Ú_CHECKPOINT_FOR_DOCr   Ú_CONFIG_FOR_DOCr   r*   ÚTensorr   ÚstrÚboolr   r   r•   r›   rœ   s   @r5   rÕ   rÕ   |  s‹  ø„ ô
D
òLò)ò6ñ +Ð+?×+FÑ+FÐGdÓ+eÓfÙØ&Ø#Ø$ôð -1Ø15Ø(,Ø15Ø/3Ø*.Ø37Ø,0Ø04Ø,0Ø/3Ø&*ñGmà˜EŸL™LÑ)ðGmð ! §¡Ñ.ðGmð ˜Ÿ™Ñ%ð	Gmð
 ! §¡Ñ.ðGmð ˜uŸ|™|Ñ,ðGmð ˜%Ÿ,™,Ñ'ðGmð ˜˜S %§,¡,Ð.Ñ/Ñ0ðGmð ˜EŸL™LÑ)ðGmð   §¡Ñ-ðGmð $ D™>ðGmð ' t™nðGmð ˜d‘^ðGmð 
ˆu�oÐ%Ñ	&òGmóó gôGmrJ   rÕ   c                   ó*   ‡ — e Zd ZdZˆ fd„Zdd„Zˆ xZS )ÚXLMPredLayerz?
    Prediction layer (cross_entropy or adaptive_softmax).
    c                 ó†  •— t         ‰| �  «        |j                  | _        |j                  | _        |j                  | _        |j
                  }|j                  du r(t        j                  ||j                  d¬«      | _        y t        j                  ||j                  |j                  |j                  d¬«      | _        y )NFT)rÑ   )Úin_featuresÚ	n_classesÚcutoffsÚ	div_valueÚ	head_bias)rN   rO   Úasmrö   rø   rØ   r   rV   ÚprojÚAdaptiveLogSoftmaxWithLossÚasm_cutoffsÚasm_div_value)r]   r^   r0   r_   s      €r5   rO   zXLMPredLayer.__init__j  s�   ø€ Ü‰ÑÔØ—:‘:ˆŒØ—~‘~ˆŒØ×)Ñ)ˆŒØ�n‰nˆà�:‰:˜ÑÜŸ	™	 # v§~¡~¸DÔAˆD�Iä×5Ñ5ØØ Ÿ.™.Ø×*Ñ*Ø ×.Ñ.ØôˆD�IrJ   c                 óz  — d}| j                   du rl| j                  |«      }|f|z   }|�Qt        j                  j	                  |j                  d| j                  «      |j                  d«      d¬«      }|f|z   }|S | j                  j                  |«      }|f|z   }|�| j                  ||«      \  }}|f|z   }|S )z,Compute the loss, and optionally the scores.rí   Frk   rÅ   )Ú	reduction)rA  rB  r   r€   Úcross_entropyrl   rö   Úlog_prob)r]   ro   Úyr”   r‘   rã   r  s          r5   r•   zXLMPredLayer.forward|  sÇ   € àˆØ�8‰8�uÑØ—Y‘Y˜q“\ˆFØ�i 'Ñ)ˆGØˆ}Ü—}‘}×2Ñ2°6·;±;¸rÀ4Ç<Á<Ó3PÐRS×RXÑRXÐY[ÓR\ÐhnÐ2Óo�Ø˜' GÑ+�ð ˆð —Y‘Y×'Ñ'¨Ó*ˆFØ�i 'Ñ)ˆGØˆ}ØŸ)™) A q›/‘��4Ø˜' GÑ+�àˆrJ   r¬   )r–   r—   r˜   rÜ   rO   r•   r›   rœ   s   @r5   r:  r:  e  s   ø„ ñô÷$rJ   r:  z†
    The XLM Model transformer with a language modeling head on top (linear layer with weights tied to the input
    embeddings).
    c            !       ó  ‡ — e Zd ZdgZˆ fd„Zd„ Zd„ Zd„ Z ee	j                  d«      «       eeeed¬«      	 	 	 	 	 	 	 	 	 	 	 	 	 dd	eej"                     d
eej"                     deej"                     deej"                     deej"                     deej"                     deeeej"                  f      deej"                     deej"                     deej"                     dee   dee   dee   deeef   fd„«       «       Zˆ xZS )ÚXLMWithLMHeadModelzpred_layer.proj.weightc                 ó„   •— t         ‰| �  |«       t        |«      | _        t	        |«      | _        | j                  «        y r¬   )rN   rO   rÕ   r±   r:  Ú
pred_layerr
  ©r]   r^   r_   s     €r5   rO   zXLMWithLMHeadModel.__init__™  s5   ø€ Ü‰Ñ˜Ô Ü# FÓ+ˆÔÜ& vÓ.ˆŒð 	�‰ÕrJ   c                 ó.   — | j                   j                  S r¬   ©rN  rB  r  s    r5   Úget_output_embeddingsz(XLMWithLMHeadModel.get_output_embeddings¡  s   € Ø�‰×#Ñ#Ð#rJ   c                 ó&   — || j                   _        y r¬   rQ  r  s     r5   Úset_output_embeddingsz(XLMWithLMHeadModel.set_output_embeddings¤  s   € Ø-ˆ�‰ÕrJ   c                 óP  — | j                   j                  }| j                   j                  }|j                  d   }t	        j
                  |df|t        j                  |j                  ¬«      }t	        j                  ||gd¬«      }|�t	        j                  ||«      }nd }||dœS )Nr   r!   r8   ra   )rº   r¼   )
r^   Úmask_token_idÚlang_idrq   r*   Úfullr<   r:   rx   Ú	full_like)r]   rº   r´   rV  rW  Úeffective_batch_sizeÚ
mask_tokenr¼   s           r5   Úprepare_inputs_for_generationz0XLMWithLMHeadModel.prepare_inputs_for_generation§  s”   € ð Ÿ™×1Ñ1ˆØ—+‘+×%Ñ%ˆà(Ÿ™¨qÑ1ÐÜ—Z‘ZÐ!5°qÐ 9¸=ÔPU×PZÑPZÐcl×csÑcsÔtˆ
Ü—I‘I˜y¨*Ð5¸1Ô=ˆ	ØÐÜ—O‘O I¨wÓ7‰EàˆEØ&°Ñ7Ð7rJ   r  z
<special1>)r  r  rÝ   rF   rº   r»   r¼   r  rñ   rB   r†   r‡   r  Úlabelsrˆ   r   r!  r"  c                 ó  — |�|n| j                   j                  } | j                  |f||||||||	|||dœ|¤Ž}|d   }| j                  ||
«      }|s||dd z   S t	        |
�|d   nd|
€|d   n|d   |j
                  |j                  ¬«      S )a³  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for language modeling. Note that the labels **are shifted** inside the model, i.e. you can set
            `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100`
            are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]`
        N©r»   r¼   r  rñ   rB   r†   r‡   r  rˆ   r   r!  r   r!   ©rã   Úlogitsré   rê   )r^   r(  r±   rN  r   ré   rê   )r]   rº   r»   r¼   r  rñ   rB   r†   r‡   r  r]  rˆ   r   r!  r´   Útransformer_outputsÚoutputr”   s                     r5   r•   zXLMWithLMHeadModel.forward¶  sÐ   € ð< &1Ð%<‘kÀ$Ç+Á+×B]ÑB]ˆà.˜d×.Ñ.Øð
à)ØØ)Ø%ØØØØ'Ø/Ø!5Ø#ñ
ð ñ
Ðð  % QÑ'ˆØ—/‘/ &¨&Ó1ˆáØÐ0°°Ð4Ñ4Ð4äØ%Ð1�˜’°tØ!' �7˜1’:°W¸Q±ZØ-×;Ñ;Ø*×5Ñ5ô	
ð 	
rJ   ©NNNNNNNNNNNNN)r–   r—   r˜   Ú_tied_weights_keysrO   rR  rT  r\  r   r2  r3  r   r4  r   r5  r   r*   r6  r   r7  r8  r   r   r•   r›   rœ   s   @r5   rL  rL  �  s�  ø„ ð 3Ð3Ðôò$ò.ò8ñ +Ð+?×+FÑ+FÐGdÓ+eÓfÙØ&Ø"Ø$Øô	ð -1Ø15Ø(,Ø15Ø/3Ø*.Ø37Ø,0Ø04Ø)-Ø,0Ø/3Ø&*ñ4
à˜EŸL™LÑ)ð4
ð ! §¡Ñ.ð4
ð ˜Ÿ™Ñ%ð	4
ð
 ! §¡Ñ.ð4
ð ˜uŸ|™|Ñ,ð4
ð ˜%Ÿ,™,Ñ'ð4
ð ˜˜S %§,¡,Ð.Ñ/Ñ0ð4
ð ˜EŸL™LÑ)ð4
ð   §¡Ñ-ð4
ð ˜Ÿ™Ñ&ð4
ð $ D™>ð4
ð ' t™nð4
ð ˜d‘^ð4
ð  
ˆu�nÐ$Ñ	%ò!4
óó gô4
rJ   rL  z�
    XLM Model with a sequence classification/regression head on top (a linear layer on top of the pooled output) e.g.
    for GLUE tasks.
    c            !       óô  ‡ — e Zd Zˆ fd„Z eej                  d«      «       eee	e
¬«      	 	 	 	 	 	 	 	 	 	 	 	 	 ddeej                     deej                     deej                     deej                     deej                     d	eej                     d
eeeej                  f      deej                     deej                     deej                     dee   dee   dee   deee	f   fd„«       «       Zˆ xZS )ÚXLMForSequenceClassificationc                 ó´   •— t         ‰| �  |«       |j                  | _        || _        t	        |«      | _        t        |«      | _        | j                  «        y r¬   )	rN   rO   Ú
num_labelsr^   rÕ   r±   r   Úsequence_summaryr
  rO  s     €r5   rO   z%XLMForSequenceClassification.__init__ü  sJ   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒØˆŒä# FÓ+ˆÔÜ /°Ó 7ˆÔð 	�‰ÕrJ   r  r  rº   r»   r¼   r  rñ   rB   r†   r‡   r  r]  rˆ   r   r!  r"  c                 ó$  — |�|n| j                   j                  }| j                  |||||||||	|||¬«      }|d   }| j                  |«      }d}|
��‡| j                   j                  €�| j
                  dk(  rd| j                   _        nl| j
                  dkD  rL|
j                  t        j                  k(  s|
j                  t        j                  k(  rd| j                   _        nd| j                   _        | j                   j                  dk(  rIt        «       }| j
                  dk(  r& ||j                  «       |
j                  «       «      }nŒ |||
«      }n‚| j                   j                  dk(  r=t        «       } ||j                  d| j
                  «      |
j                  d«      «      }n,| j                   j                  dk(  rt        «       } |||
«      }|s|f|dd z   }|�|f|z   S |S t        |||j                   |j"                  ¬	«      S )
a�  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        Nr_  r   r!   Ú
regressionÚsingle_label_classificationÚmulti_label_classificationrk   r`  )r^   r(  r±   rj  Úproblem_typeri  r9   r*   r<   r  r   Úsqueezer
   rl   r	   r   ré   rê   )r]   rº   r»   r¼   r  rñ   rB   r†   r‡   r  r]  rˆ   r   r!  rb  rc  ra  rã   Úloss_fcts                      r5   r•   z$XLMForSequenceClassification.forward  së  € ð8 &1Ð%<‘kÀ$Ç+Á+×B]ÑB]ˆà"×.Ñ.ØØ)ØØ)Ø%ØØØØ'Ø/Ø!5Ø#ð /ó 
Ðð % QÑ'ˆØ×&Ñ& vÓ.ˆàˆØÑØ�{‰{×'Ñ'Ð/Ø—?‘? aÒ'Ø/;�D—K‘KÕ,Ø—_‘_ qÒ(¨f¯l©l¼e¿j¹jÒ.HÈFÏLÉLÔ\a×\eÑ\eÒLeØ/L�D—K‘KÕ,à/K�D—K‘KÔ,à�{‰{×'Ñ'¨<Ò7Ü"›9�Ø—?‘? aÒ'Ù# F§N¡NÓ$4°f·n±nÓ6FÓG‘Dá# F¨FÓ3‘DØ—‘×)Ñ)Ð-JÒJÜ+Ó-�Ù §¡¨B°·±Ó @À&Ç+Á+ÈbÃ/ÓR‘Ø—‘×)Ñ)Ð-IÒIÜ,Ó.�Ù ¨Ó/�áØ�YÐ!4°Q°RÐ!8Ñ8ˆFØ)-Ð)9�T�G˜fÑ$ÐE¸vÐEä'ØØØ-×;Ñ;Ø*×5Ñ5ô	
ð 	
rJ   rd  )r–   r—   r˜   rO   r   r2  r3  r   r4  r   r5  r   r*   r6  r   r7  r8  r   r   r•   r›   rœ   s   @r5   rg  rg  ô  sƒ  ø„ ô	ñ +Ð+?×+FÑ+FÐGdÓ+eÓfÙØ&Ø,Ø$ôð -1Ø15Ø(,Ø15Ø/3Ø*.Ø37Ø,0Ø04Ø)-Ø,0Ø/3Ø&*ñJ
à˜EŸL™LÑ)ðJ
ð ! §¡Ñ.ðJ
ð ˜Ÿ™Ñ%ð	J
ð
 ! §¡Ñ.ðJ
ð ˜uŸ|™|Ñ,ðJ
ð ˜%Ÿ,™,Ñ'ðJ
ð ˜˜S %§,¡,Ð.Ñ/Ñ0ðJ
ð ˜EŸL™LÑ)ðJ
ð   §¡Ñ-ðJ
ð ˜Ÿ™Ñ&ðJ
ð $ D™>ðJ
ð ' t™nðJ
ð ˜d‘^ðJ
ð 
ˆuÐ.Ð.Ñ	/òJ
óó gôJ
rJ   rg  zÜ
    XLM Model with a span classification head on top for extractive question-answering tasks like SQuAD (a linear
    layers on top of the hidden-states output to compute `span start logits` and `span end logits`).
    c            #       ó  ‡ — e Zd Zˆ fd„Z eej                  d«      «       eee	e
¬«      	 	 	 	 	 	 	 	 	 	 	 	 	 	 ddeej                     deej                     deej                     deej                     deej                     d	eej                     d
eeeej                  f      deej                     deej                     deej                     deej                     dee   dee   dee   deee	f   fd„«       «       Zˆ xZS )ÚXLMForQuestionAnsweringSimplec                 óÂ   •— t         ‰| �  |«       t        |«      | _        t	        j
                  |j                  |j                  «      | _        | j                  «        y r¬   )
rN   rO   rÕ   r±   r   rV   Úhidden_sizeri  Ú
qa_outputsr
  rO  s     €r5   rO   z&XLMForQuestionAnsweringSimple.__init__b  sG   ø€ Ü‰Ñ˜Ô ä# FÓ+ˆÔÜŸ)™) F×$6Ñ$6¸×8IÑ8IÓJˆŒð 	�‰ÕrJ   r  r  rº   r»   r¼   r  rñ   rB   r†   r‡   r  Ústart_positionsÚend_positionsrˆ   r   r!  r"  c                 ó.  — |�|n| j                   j                  }| j                  |||||||||	|||¬«      }|d   }| j                  |«      }|j	                  dd¬«      \  }}|j                  d«      j                  «       }|j                  d«      j                  «       }d}|
�·|�µt        |
j                  «       «      dkD  r|
j                  d«      }
t        |j                  «       «      dkD  r|j                  d«      }|j                  d«      }|
j                  d|«      }
|j                  d|«      }t        |¬«      } |||
«      } |||«      }||z   dz  }|s||f|dd z   }|�|f|z   S |S t        ||||j                  |j                  ¬	«      S )
a  
        start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for position (index) of the start of the labelled span for computing the token classification loss.
            Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
            are not taken into account for computing the loss.
        end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for position (index) of the end of the labelled span for computing the token classification loss.
            Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
            are not taken into account for computing the loss.
        Nr_  r   r!   rk   ra   )Úignore_indexr$   )rã   Ústart_logitsÚ
end_logitsré   rê   )r^   r(  r±   rv  Úsplitrp  rs   rb   r?   Úclampr
   r   ré   rê   )r]   rº   r»   r¼   r  rñ   rB   r†   r‡   r  rw  rx  rˆ   r   r!  rb  Úsequence_outputra  r{  r|  Ú
total_lossÚignored_indexrq  Ú
start_lossÚend_lossrc  s                             r5   r•   z%XLMForQuestionAnsweringSimple.forwardk  sÒ  € ðB &1Ð%<‘kÀ$Ç+Á+×B]ÑB]ˆà"×.Ñ.ØØ)ØØ)Ø%ØØØØ'Ø/Ø!5Ø#ð /ó 
Ðð .¨aÑ0ˆà—‘ Ó1ˆØ#)§<¡<°°r <Ó#:Ñ ˆ�jØ#×+Ñ+¨BÓ/×:Ñ:Ó<ˆØ×'Ñ'¨Ó+×6Ñ6Ó8ˆ
àˆ
ØÐ&¨=Ð+Dä�?×'Ñ'Ó)Ó*¨QÒ.Ø"1×"9Ñ"9¸"Ó"=�Ü�=×%Ñ%Ó'Ó(¨1Ò,Ø -× 5Ñ 5°bÓ 9�à(×-Ñ-¨aÓ0ˆMØ-×3Ñ3°A°}ÓEˆOØ)×/Ñ/°°=ÓAˆMä'°]ÔCˆHÙ! ,°Ó@ˆJÙ 
¨MÓ:ˆHØ$ xÑ/°1Ñ4ˆJáØ" JÐ/Ð2EÀaÀbÐ2IÑIˆFØ/9Ð/E�Z�M FÑ*ÐQÈ6ÐQä+ØØ%Ø!Ø-×;Ñ;Ø*×5Ñ5ô
ð 	
rJ   )NNNNNNNNNNNNNN)r–   r—   r˜   rO   r   r2  r3  r   r4  r   r5  r   r*   r6  r   r7  r8  r   r   r•   r›   rœ   s   @r5   rs  rs  Z  sš  ø„ ôñ +Ð+?×+FÑ+FÐGdÓ+eÓfÙØ&Ø0Ø$ôð -1Ø15Ø(,Ø15Ø/3Ø*.Ø37Ø,0Ø04Ø26Ø04Ø,0Ø/3Ø&*ñN
à˜EŸL™LÑ)ðN
ð ! §¡Ñ.ðN
ð ˜Ÿ™Ñ%ð	N
ð
 ! §¡Ñ.ðN
ð ˜uŸ|™|Ñ,ðN
ð ˜%Ÿ,™,Ñ'ðN
ð ˜˜S %§,¡,Ð.Ñ/Ñ0ðN
ð ˜EŸL™LÑ)ðN
ð   §¡Ñ-ðN
ð " %§,¡,Ñ/ðN
ð   §¡Ñ-ðN
ð $ D™>ðN
ð ' t™nðN
ð ˜d‘^ðN
ð  
ˆuÐ2Ð2Ñ	3ò!N
óó gôN
rJ   rs  zè
    XLM Model with a beam-search span classification head on top for extractive question-answering tasks like SQuAD (a
    linear layers on top of the hidden-states output to compute `span start logits` and `span end logits`).
    c            )       ór  ‡ — e Zd Zˆ fd„Z eej                  d«      «       eee	¬«      	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dde
ej                     de
ej                     de
ej                     de
ej                     de
ej                     d	e
ej                     d
e
eeej                  f      de
ej                     de
ej                     de
ej                     de
ej                     de
ej                     de
ej                     de
ej                     de
e   de
e   de
e   deeef   f$d„«       «       Zˆ xZS )ÚXLMForQuestionAnsweringc                 ó„   •— t         ‰| �  |«       t        |«      | _        t	        |«      | _        | j                  «        y r¬   )rN   rO   rÕ   r±   r   rv  r
  rO  s     €r5   rO   z XLMForQuestionAnswering.__init__Ê  s5   ø€ Ü‰Ñ˜Ô ä# FÓ+ˆÔÜ# FÓ+ˆŒð 	�‰ÕrJ   r  )r  rÝ   rº   r»   r¼   r  rñ   rB   r†   r‡   r  rw  rx  Úis_impossibleÚ	cls_indexÚp_maskrˆ   r   r!  r"  c                 ó„  — |�|n| j                   j                  }| j                  |||||||||	|||¬«      }|d   }| j                  ||
|||||¬«      }|s||dd z   S t	        |j
                  |j                  |j                  |j                  |j                  |j                  |j                  |j                  ¬«      S )aN  
        start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for position (index) of the start of the labelled span for computing the token classification loss.
            Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
            are not taken into account for computing the loss.
        end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for position (index) of the end of the labelled span for computing the token classification loss.
            Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
            are not taken into account for computing the loss.
        is_impossible (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels whether a question has an answer or no answer (SQuAD 2.0)
        cls_index (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for position (index) of the classification token to use as input for computing plausibility of the
            answer.
        p_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Optional mask of tokens which can't be in answers (e.g. [CLS], [PAD], ...). 1.0 means token should be
            masked. 0.0 mean token is not masked.

        Returns:

        Example:

        ```python
        >>> from transformers import AutoTokenizer, XLMForQuestionAnswering
        >>> import torch

        >>> tokenizer = AutoTokenizer.from_pretrained("FacebookAI/xlm-mlm-en-2048")
        >>> model = XLMForQuestionAnswering.from_pretrained("FacebookAI/xlm-mlm-en-2048")

        >>> input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True)).unsqueeze(
        ...     0
        ... )  # Batch size 1
        >>> start_positions = torch.tensor([1])
        >>> end_positions = torch.tensor([3])

        >>> outputs = model(input_ids, start_positions=start_positions, end_positions=end_positions)
        >>> loss = outputs.loss
        ```Nr_  r   )rw  rx  rˆ  r‡  r‰  r!  r!   )rã   rä   rå   ræ   rç   rè   ré   rê   )r^   r(  r±   rv  râ   rã   rä   rå   ræ   rç   rè   ré   rê   )r]   rº   r»   r¼   r  rñ   rB   r†   r‡   r  rw  rx  r‡  rˆ  r‰  rˆ   r   r!  rb  rc  r”   s                        r5   r•   zXLMForQuestionAnswering.forwardÓ  sô   € ðx &1Ð%<‘kÀ$Ç+Á+×B]ÑB]ˆà"×.Ñ.ØØ)ØØ)Ø%ØØØØ'Ø/Ø!5Ø#ð /ó 
Ðð % QÑ'ˆà—/‘/ØØ+Ø'ØØ'ØØ#ð "ó 
ˆñ ØÐ0°°Ð4Ñ4Ð4ä,Ø—‘Ø '× ;Ñ ;Ø#×3Ñ3Ø%×7Ñ7Ø!×/Ñ/Ø×)Ñ)Ø-×;Ñ;Ø*×5Ñ5ô	
ð 		
rJ   )NNNNNNNNNNNNNNNNN)r–   r—   r˜   rO   r   r2  r3  r    râ   r5  r   r*   r6  r   r7  r8  r   r   r•   r›   rœ   s   @r5   r…  r…  Â  sÛ  ø„ ôñ +Ð+?×+FÑ+FÐGdÓ+eÓfÙÐ+HÐWfÔgð -1Ø15Ø(,Ø15Ø/3Ø*.Ø37Ø,0Ø04Ø26Ø04Ø04Ø,0Ø)-Ø,0Ø/3Ø&*ñ%c
à˜EŸL™LÑ)ðc
ð ! §¡Ñ.ðc
ð ˜Ÿ™Ñ%ð	c
ð
 ! §¡Ñ.ðc
ð ˜uŸ|™|Ñ,ðc
ð ˜%Ÿ,™,Ñ'ðc
ð ˜˜S %§,¡,Ð.Ñ/Ñ0ðc
ð ˜EŸL™LÑ)ðc
ð   §¡Ñ-ðc
ð " %§,¡,Ñ/ðc
ð   §¡Ñ-ðc
ð   §¡Ñ-ðc
ð ˜EŸL™LÑ)ðc
ð ˜Ÿ™Ñ&ðc
ð  $ D™>ð!c
ð" ' t™nð#c
ð$ ˜d‘^ð%c
ð& 
ˆuÐ3Ð3Ñ	4ò'c
ó hó gôc
rJ   r…  z¢
    XLM Model with a token classification head on top (a linear layer on top of the hidden-states output) e.g. for
    Named-Entity-Recognition (NER) tasks.
    c            !       óô  ‡ — e Zd Zˆ fd„Z eej                  d«      «       eee	e
¬«      	 	 	 	 	 	 	 	 	 	 	 	 	 ddeej                     deej                     deej                     deej                     deej                     d	eej                     d
eeeej                  f      deej                     deej                     deej                     dee   dee   dee   deee	f   fd„«       «       Zˆ xZS )ÚXLMForTokenClassificationc                 ó,  •— t         ‰| �  |«       |j                  | _        t        |«      | _        t        j                  |j                  «      | _        t        j                  |j                  |j                  «      | _
        | j                  «        y r¬   )rN   rO   ri  rÕ   r±   r   ÚDropoutrU   rV   ru  Ú
classifierr
  rO  s     €r5   rO   z"XLMForTokenClassification.__init__C  sh   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒä# FÓ+ˆÔÜ—z‘z &§.¡.Ó1ˆŒÜŸ)™) F×$6Ñ$6¸×8IÑ8IÓJˆŒð 	�‰ÕrJ   r  r  rº   r»   r¼   r  rñ   rB   r†   r‡   r  r]  rˆ   r   r!  r"  c                 ó®  — |�|n| j                   j                  }| j                  |||||||||	|||¬«      }|d   }| j                  |«      }| j	                  |«      }d}|
�<t        «       } ||j                  d| j                  «      |
j                  d«      «      }|s|f|dd z   }|�|f|z   S |S t        |||j                  |j                  ¬«      S )zÛ
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`.
        Nr_  r   rk   r!   r`  )r^   r(  r±   rU   r�  r
   rl   ri  r   ré   rê   )r]   rº   r»   r¼   r  rñ   rB   r†   r‡   r  r]  rˆ   r   r!  r”   r  ra  rã   rq  rc  s                       r5   r•   z!XLMForTokenClassification.forwardN  s  € ð4 &1Ð%<‘kÀ$Ç+Á+×B]ÑB]ˆà×"Ñ"ØØ)ØØ)Ø%ØØØØ'Ø/Ø!5Ø#ð #ó 
ˆð " !™*ˆàŸ,™, Ó7ˆØ—‘ Ó1ˆàˆØÐÜ'Ó)ˆHÙ˜FŸK™K¨¨D¯O©OÓ<¸f¿k¹kÈ"»oÓNˆDáØ�Y ¨¨ Ñ,ˆFØ)-Ð)9�T�G˜fÑ$ÐE¸vÐEä$ØØØ!×/Ñ/Ø×)Ñ)ô	
ð 	
rJ   rd  )r–   r—   r˜   rO   r   r2  r3  r   r4  r   r5  r   r*   r6  r   r7  r8  r   r   r•   r›   rœ   s   @r5   rŒ  rŒ  ;  ss  ø„ ô	ñ +Ð+?×+FÑ+FÐGdÓ+eÓfÙØ&Ø)Ø$ôð -1Ø15Ø(,Ø15Ø/3Ø*.Ø37Ø,0Ø04Ø)-Ø,0Ø/3Ø&*ñ8
à˜EŸL™LÑ)ð8
ð ! §¡Ñ.ð8
ð ˜Ÿ™Ñ%ð	8
ð
 ! §¡Ñ.ð8
ð ˜uŸ|™|Ñ,ð8
ð ˜%Ÿ,™,Ñ'ð8
ð ˜˜S %§,¡,Ð.Ñ/Ñ0ð8
ð ˜EŸL™LÑ)ð8
ð   §¡Ñ-ð8
ð ˜Ÿ™Ñ&ð8
ð $ D™>ð8
ð ' t™nð8
ð ˜d‘^ð8
ð 
ˆuÐ+Ð+Ñ	,ò8
óó gô8
rJ   rŒ  z¤
    XLM Model with a multiple choice classification head on top (a linear layer on top of the pooled output and a
    softmax) e.g. for RocStories/SWAG tasks.
    c            !       óô  ‡ — e Zd Zˆ fd„Z eej                  d«      «       eee	e
¬«      	 	 	 	 	 	 	 	 	 	 	 	 	 ddeej                     deej                     deej                     deej                     deej                     d	eej                     d
eeeej                  f      deej                     deej                     deej                     dee   dee   dee   deee	f   fd„«       «       Zˆ xZS )ÚXLMForMultipleChoicec                 óÖ   •— t        ‰| �  |g|¢­i |¤Ž t        |«      | _        t	        |«      | _        t        j                  |j                  d«      | _	        | j                  «        y r    )rN   rO   rÕ   r±   r   rj  r   rV   ri  Úlogits_projr
  )r]   r^   r³   r´   r_   s       €r5   rO   zXLMForMultipleChoice.__init__—  sY   ø€ Ü‰Ñ˜Ð3 &Ò3¨FÒ3ä# FÓ+ˆÔÜ /°Ó 7ˆÔÜŸ9™9 V×%6Ñ%6¸Ó:ˆÔð 	�‰ÕrJ   z(batch_size, num_choices, sequence_lengthr  rº   r»   r¼   r  rñ   rB   r†   r‡   r  r]  rˆ   r   r!  r"  c                 óÎ  — |�|n| j                   j                  }|�|j                  d   n|	j                  d   }|�!|j                  d|j	                  d«      «      nd}|�!|j                  d|j	                  d«      «      nd}|�!|j                  d|j	                  d«      «      nd}|�!|j                  d|j	                  d«      «      nd}|�!|j                  d|j	                  d«      «      nd}|	�1|	j                  d|	j	                  d«      |	j	                  d«      «      nd}	|�t
        j                  d«       d}| j                  |||||||||	|||¬«      }|d   }| j                  |«      }| j                  |«      }|j                  d|«      }d}|
�t        «       } |||
«      }|s|f|dd z   }|�|f|z   S |S t        |||j                  |j                  ¬«      S )	aJ  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the multiple choice classification loss. Indices should be in `[0, ...,
            num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See
            `input_ids` above)
        Nr!   rk   éþÿÿÿzrThe `lengths` parameter cannot be used with the XLM multiple choice models. Please use the attention mask instead.)rº   r»   r¼   r  rñ   rB   r†   r‡   r  rˆ   r   r!  r   r`  )r^   r(  rq   rl   r?   ÚloggerÚwarningr±   rj  r”  r
   r   ré   rê   )r]   rº   r»   r¼   r  rñ   rB   r†   r‡   r  r]  rˆ   r   r!  Únum_choicesrb  rc  ra  Úreshaped_logitsrã   rq  s                        r5   r•   zXLMForMultipleChoice.forward¡  s-  € ð8 &1Ð%<‘kÀ$Ç+Á+×B]ÑB]ˆØ,5Ð,A�i—o‘o aÒ(À}×GZÑGZÐ[\ÑG]ˆà>GÐ>S�I—N‘N 2 y§~¡~°bÓ'9Ô:ÐY]ˆ	ØM[ÐMg˜×,Ñ,¨R°×1DÑ1DÀRÓ1HÔIÐmqˆØM[ÐMg˜×,Ñ,¨R°×1DÑ1DÀRÓ1HÔIÐmqˆØGSÐG_�|×(Ñ(¨¨\×->Ñ->¸rÓ-BÔCÐeiˆØ27Ð2C�—
‘
˜2˜uŸz™z¨"›~Ô.Èˆð Ð(ð ×Ñ˜r =×#5Ñ#5°bÓ#9¸=×;MÑ;MÈbÓ;QÔRàð 	ð ÐÜ�N‰Nð*ôð ˆGà"×.Ñ.ØØ)ØØ)Ø%ØØØØ'Ø/Ø!5Ø#ð /ó 
Ðð % QÑ'ˆØ×&Ñ& vÓ.ˆØ×!Ñ! &Ó)ˆØ Ÿ+™+ b¨+Ó6ˆàˆØÐÜ'Ó)ˆHÙ˜O¨VÓ4ˆDáØ%Ð'Ð*=¸a¸bÐ*AÑAˆFØ)-Ð)9�T�G˜fÑ$ÐE¸vÐEä(ØØ"Ø-×;Ñ;Ø*×5Ñ5ô	
ð 	
rJ   rd  )r–   r—   r˜   rO   r   r2  r3  r   r4  r   r5  r   r*   r6  r   r7  r8  r   r   r•   r›   rœ   s   @r5   r’  r’  �  sƒ  ø„ ôñ +Ð+?×+FÑ+FÐGqÓ+rÓsÙØ&Ø-Ø$ôð -1Ø15Ø(,Ø15Ø/3Ø*.Ø37Ø,0Ø04Ø)-Ø,0Ø/3Ø&*ñL
à˜EŸL™LÑ)ðL
ð ! §¡Ñ.ðL
ð ˜Ÿ™Ñ%ð	L
ð
 ! §¡Ñ.ðL
ð ˜uŸ|™|Ñ,ðL
ð ˜%Ÿ,™,Ñ'ðL
ð ˜˜S %§,¡,Ð.Ñ/Ñ0ðL
ð ˜EŸL™LÑ)ðL
ð   §¡Ñ-ðL
ð ˜Ÿ™Ñ&ðL
ð $ D™>ðL
ð ' t™nðL
ð ˜d‘^ðL
ð 
ˆuÐ/Ð/Ñ	0òL
óó tôL
rJ   r’  )r’  r…  rs  rg  rŒ  rÕ   r°   rL  r¬   )ErÜ   r™   ry   Údataclassesr   Útypingr   r   r   r   Únumpyr%   r*   r   Útorch.nnr	   r
   r   Úactivationsr   Ú
generationr   Úmodeling_outputsr   r   r   r   r   r   Úmodeling_utilsr   r   r   Úpytorch_utilsr   r   r   Úutilsr   r   r   r   r   r    Úconfiguration_xlmr"   Ú
get_loggerr–   r—  r4  r5  r6   rI   ÚModulerL   rž   r°   râ   ÚXLM_START_DOCSTRINGr2  rÕ   r:  rL  rg  rs  r…  rŒ  r’  Ú__all__rí   rJ   r5   ú<module>rª     sA  ðñó Û Ý !ß /Ó /ã Û Ý ß AÑ Aå Ý )÷÷ ÷ JÑ Iß lÑ l÷÷ õ )ð 
ˆ×	Ñ	˜HÓ	%€à2Ð Ø€òóô2]˜Ÿ™ô ]ô@�R—Y‘Yô ô*)˜ô )ðX ô'? Kó '?ó ð'?ðTÐ ð BÐ ñJ ØcØóôbmÐ!ó bmó	ðbmôJ'�2—9‘9ô 'ñT ðð óô[
Ð+¨_ó [
óð[
ñ| ðð óô\
Ð#5ó \
óð\
ñ~ ðð óô^
Ð$6ó ^
óð^
ñB ðð óôo
Ð0ó o
óðo
ñd ðð óôJ
Ð 2ó J
óðJ
ñZ ðð óô]
Ð-ó ]
óð]
ò@	�rJ   