Ë
    T^(hì¸ ã                   ó,  — d Z ddlZddlZddlZddlmZmZmZmZm	Z	 ddl
Z
ddl
mZ ddlmZ ddlmZ ddlmZmZmZmZ dd	lmZ dd
lmZ ddlmZmZmZmZ ddlmZ ddl m!Z!m"Z"m#Z# ddl$m%Z%m&Z&m'Z'm(Z(m)Z)m*Z*m+Z+m,Z,m-Z- ddl.m/Z/  e)«       rddl0m1Z1 ddl2m3Z3  e,jh                  e5«      Z6dZ7dZ8dYde
jr                  de:de:de:de
jr                  f
d„Z;de
jr                  de:de:de
jr                  fd„Z<dYde
jr                  de:de:de:de
jr                  f
d„Z=de:de
jr                  fd„Z>d e
jr                  de:de
jr                  fd!„Z?d"e
jr                  de:d#e
j€                  de
jr                  fd$„ZAd"e
jr                  d%e:dee
jr                  e
jr                  f   fd&„ZBd"e
jr                  d%e:de
jr                  fd'„ZCd(e
jr                  d)e
jr                  d*e:de
jr                  fd+„ZD G d,„ d-ejŠ                  «      ZF	 dd.lGmHZH eHZFe6j“                  d/«        e!jš                  eF«        G d1„ d2ejŠ                  «      ZN G d3„ d4ejŠ                  «      ZO G d5„ d6ejŠ                  «      ZP G d7„ d8ejŠ                  «      ZQ G d9„ d:ejŠ                  «      ZR G d;„ d<ejŠ                  «      ZS G d=„ d>ejŠ                  «      ZT G d?„ d@ejŠ                  «      ZU G dA„ dBejŠ                  «      ZV G dC„ dDejŠ                  «      ZW G dE„ dFejŠ                  «      ZX G dG„ dHe«      ZY G dI„ dJeY«      ZZdKZ[dLZ\dMZ]dNZ^ e'dOe[«       G dP„ dQeY«      «       Z_ e'dRe[«       G dS„ dTeYe«      «       Z` e'dUe[«       G dV„ dWeY«      «       Zag dX¢Zby# eJ$ r Y �ŒjeK$ r e6j™                  d0«       Y �Œ‚w xY w)ZzPyTorch LongT5 model.é    N)ÚAnyÚListÚOptionalÚTupleÚUnion)Únn)ÚCrossEntropyLossé   )ÚACT2FN)ÚCacheÚDynamicCacheÚEncoderDecoderCacheÚStaticCache)ÚGenerationMixin)ÚAttentionMaskConverter)ÚBaseModelOutputÚ)BaseModelOutputWithPastAndCrossAttentionsÚSeq2SeqLMOutputÚSeq2SeqModelOutput)ÚPreTrainedModel)ÚALL_LAYERNORM_LAYERSÚ find_pruneable_heads_and_indicesÚprune_linear_layer)	ÚDUMMY_INPUTSÚ
DUMMY_MASKÚadd_start_docstringsÚ%add_start_docstrings_to_model_forwardÚis_torch_flex_attn_availableÚis_torch_fx_proxyÚis_torchdynamo_compilingÚloggingÚreplace_return_docstringsé   )ÚLongT5Config)Ú	BlockMask)Úmake_flex_block_causal_maskr$   zgoogle/long-t5-local-baseÚxÚ	block_lenÚdimÚ	pad_valueÚreturnc                 ót  — | j                   |    |z  }t        | j                   «      sCt        | j                   «      }||xx   |z  cc<   t        j                  || j
                  ¬«      S dg| j                  z  }d|f||<   t        |ddd…   d«      }t        j                  j                  | |d|¬«      } | S )	zHPad a tensor so that a sequence length will be a multiple of `block_len`©Údtype©r   r   r   Néÿÿÿÿ© Úconstant©ÚpadÚmodeÚvalue)ÚshapeÚallÚlistÚtorchÚzerosr.   ÚndimÚsumr   Ú
functionalr4   )r'   r(   r)   r*   Úpad_lenÚ	new_shaper4   s          úh/var/www/skyplay_api_hub/venv/lib/python3.12/site-packages/transformers/models/longt5/modeling_longt5.pyÚ_pad_to_multiplerB   B   s¤   € à�w‰w�s‰|ˆm˜iÑ'€Gäˆq�w‰wŒ<Ü˜Ÿ™“Mˆ	Ø�#‹˜'Ñ!‹Ü�{‰{˜9¨A¯G©GÔ4Ð4àˆ(�Q—V‘VÑ
€CØ�7ˆ|€Cˆ�HÜ
ˆc‘$�B�$‰i˜Ó
€CÜ
�‰×Ñ˜! ¨:¸YÐÓG€AØ€Hó    c                 ó>  — | j                   |   |z  dk7  rt        | ||d¬«      } | j                   |   |z  }| j                   d| ||fz   | j                   |dz   d z   }d|v r,t        j                  || j                  | j
                  ¬«      S | j                  |«      S )zÅSplit an input tensor into blocks of a given `block_len` along the given `dim`. If the dimension length
    is not a multiple of `block_len`, it will be padded first with selected `pad_value`.
    r   )r*   Nr#   ©r.   Údevice)r7   rB   r:   Úemptyr.   rF   Úreshape)r'   r(   r)   Ú
num_blocksÚoutput_shapes        rA   Ú_split_into_blocksrK   R   sš   € ð
 	‡w�wˆs�|�iÑ 1Ò$Ü˜Q 	¨3¸!Ô<ˆØ—‘˜‘ Ñ*€JØ—7‘7˜4˜C�= J°	Ð#:Ñ:¸Q¿W¹WÀcÈAÁgÀ[Ð=QÑQ€LàˆLÑÜ�{‰{˜<¨q¯w©w¸q¿x¹xÔHÐHØ�9‰9�\Ó"Ð"rC   Ú	block_dimÚsequence_dimc                 óœ  — | j                   |   }dg| j                  z  }d||<   t        |ddd…   d«      }t        j                  j                  | |d|¬«      } g }t        d«      D ]M  }t        d	d«      g| j                  z  }t        |||z   «      ||<   t        |«      }|j                  | |   «       ŒO t        j                  ||¬
«      S )zšConcatenate three consecutive blocks for each input block for local attentiont.

    For more information, see: https://arxiv.org/pdf/2112.07916.pdf.
    r/   )r#   r#   Nr0   r1   r2   r3   r
   r   ©r)   )r7   r<   r=   r   r>   r4   ÚrangeÚsliceÚtupleÚappendr:   Úcat)	r'   rL   rM   r*   rI   r4   Úblocks_listÚiÚindicess	            rA   Ú_concatenate_3_blocksrX   a   sÐ   € ð
 —‘˜Ñ#€Jàˆ(�Q—V‘VÑ
€CØ€Cˆ	�NÜ
ˆc‘$�B�$‰i˜Ó
€Cä
�‰×Ñ˜! ¨:¸YÐÓG€Aà&(€KÜ�1‹Xò 'ˆô ˜˜D“>Ð" Q§V¡VÑ+ˆÜ" 1 a¨*¡nÓ5ˆ�	ÑÜ˜“.ˆØ×Ñ˜1˜W™:Õ&ð'ô �9‰9�[ lÔ3Ð3rC   c                 ó¨   — t        j                  d| z  t         j                  ¬«      }|| |   }|j                  d«      |j                  d«      z
  }|S )z:Makes 3-blocked relative position ids for local attention.r
   r-   r   r#   )r:   ÚarangeÚint32Ú	unsqueeze)r(   Úposition_idsÚcenter_position_idsÚrelative_position_idss       rA   Ú"_make_3block_relative_position_idsr`   z   sR   € ä—<‘<  I¡´U·[±[ÔA€LØ& y°)°Ð<Ðà(×2Ñ2°1Ó5Ð8K×8UÑ8UÐVWÓ8XÑXÐØ Ð rC   Úlocal_attention_maskc                 óÄ   — t        |«      }t        j                  |«      |k  }|dddd…dd…f   }|j                  | j                  «      }t        j
                  | |«      S )znMask local attention mask to enforce that tokens are not allowed to attend tokens farther than ``local_radius.N)r`   r:   ÚabsÚtorF   Úlogical_and)ra   r(   r_   Úlocality_masks       rA   Ú_mask_local_attention_maskrg   ƒ   s_   € ä>¸yÓIÐÜ—I‘IÐ3Ó4°yÑ@€MØ! $¨ªa²Ð"2Ñ3€MØ!×$Ñ$Ð%9×%@Ñ%@ÓA€MÜ×ÑÐ1°=ÓAÐArC   Úattention_maskrF   c                 ó  — t        | |d¬«      }t        |dd¬«      }|j                  d«      }|j                  d«      }t        j                  ||«      }t        ||«      }|j                  d«      j                  |«      S )z;Prepare attention mask to be applied for a local attention.r#   rO   é   ©rL   rM   r0   éþÿÿÿ)rK   rX   r\   r:   re   rg   rd   )rh   r(   rF   Ú_blocked_attention_maskÚ_3blocked_attention_maskra   s         rA   Ú_get_local_attention_maskro   Œ   s‡   € ô 1°ÀÐPQÔRÐä4Ð5LÐXYÐhiÔjÐà5×?Ñ?ÀÓCÐØ7×AÑAÀ"ÓEÐä ×,Ñ,Ð-DÐF^Ó_ÐÜ5Ð6JÈIÓVÐà×)Ñ)¨!Ó,×/Ñ/°Ó7Ð7rC   Úglobal_block_sizec                 ó¨  ‡‡— | j                   dd \  }Šdt        j                  dt        j                  fˆˆfd„}t        j                  | | j                  ¬«      ‰z  }t        j
                  |d¬«      |z
  }t        j                  | d	k7  d
d«      j                  | j                  «      }t        j                  ||z   d
z
  «      j                  | j                  «      }t        j                  d|j                  |j                  ¬«      }t        j                  ||kD  ||«      }|| z  | dz
  z   } ||«      }‰‰z  }|dkD  rBt        j                  |d¬«      j                  j                  |d«      j                  dd«      }	n-t        j                  |d|j                  |j                  ¬«      }	t        j
                  t        j                   ||«      d¬«      dz
  }
|
j#                  | j                  «      }
t        j                  |
|	k  dd«      }
|j                  t        j$                  «      |
j                  t        j$                  «      fS )a  Obtain the "fixed block" global id corresponding to each input token.

    This implementation is a simlified version of the original Flaxformr implementation adopted from:
    https://github.com/google/flaxformer/blob/main/flaxformer/architectures/longt5/long_attention.py.

    In our scenario, as we use this strategy only for a decoder, orphan tokens, i.e. those tokens which do not make for
    the whole fixed block, are assigned to the preceding block.

    Padding tokens from the original sequence are represented by -1.
    Nrj   Ú	block_idsr+   c                 óX  •— t        j                  ‰«      ‰z  ‰dz
  k(  }|j                  | j                  «      }t        j                  || dk\  «      }|j                  d«      j                  d«      j                  | j                  «      dz
  }t        j                  | |k  | |«      } | S )Nr#   r   r0   )
r:   rZ   rd   rF   re   r=   r\   Útyper.   Úwhere)rr   Ú
block_endsÚtrue_block_endsÚfull_blocksrp   Úseq_lens       €€rA   Úhandle_orphan_tokensz:_make_global_fixed_block_ids.<locals>.handle_orphan_tokens«   sš   ø€ Ü—l‘l 7Ó+Ð.?Ñ?ÐDUÐXYÑDYÑYˆ
Ø—]‘] 9×#3Ñ#3Ó4ˆ
Ü×+Ñ+¨J¸	ÀQ¹ÓGˆØ%×)Ñ)¨"Ó-×7Ñ7¸Ó;×@Ñ@ÀÇÁÓQÐTUÑUˆÜ—K‘K 	¨KÑ 7¸ÀKÓPˆ	ØÐrC   ©rF   r#   )Úaxisç        ç      ð?g     @�Àr0   rE   r   rO   )r7   r:   ÚTensorÚ	ones_likerF   Úcumsumru   rt   r.   ÚfloorÚtensorÚmaxÚvaluesÚrepeatÚ	transposer;   Úonesrd   Úint)rh   rp   Ú
batch_sizerz   Úfixed_block_maskÚmaskÚglobal_block_idsÚ_global_block_ids_lower_boundÚnum_globalsÚ_sequence_block_ids_maxÚglobal_segment_idsry   s    `         @rA   Ú_make_global_fixed_block_idsr’   œ   s  ù€ ð )×.Ñ.¨r°Ð2Ñ€J�ð¬¯©ð ¼¿¹ö ô —‘ ~¸n×>SÑ>SÔTÐWhÑhÐÜ—|‘|Ð$4¸1Ô=Ð@PÑPÐÜ�;‰;�~¨Ñ,¨c°7Ó;×@Ñ@À×AUÑAUÓV€DÜ—{‘{ 4Ð*:Ñ#:¸SÑ#@ÓA×FÑFÀ~×G[ÑG[Ó\ÐÜ$)§L¡L°Ð;K×;QÑ;QÐZj×ZqÑZqÔ$rÐ!Ü—{‘{ØÐ8Ñ8Ð:JÐLióÐð )¨>Ñ9¸nÈqÑ>PÑQÐá+Ð,<Ó=ÐØÐ.Ñ.€Kà�Q‚Ü"'§)¡)Ð,<À"Ô"E×"LÑ"L×"SÑ"SÐT_ÐabÓ"c×"mÑ"mÐnoÐqrÓ"sÑä"'§+¡+Ø˜Ð!1×!7Ñ!7Ð@P×@WÑ@Wô#
Ðô Ÿ™¤e§j¡j°¸[Ó&IÈrÔRÐUVÑVÐØ+×.Ñ.¨~×/DÑ/DÓEÐÜŸ™Ð%7Ð;RÑ%RÐTUÐWXÓYÐØ× Ñ ¤§¡Ó+Ð-?×-DÑ-DÄUÇYÁYÓ-OÐOÐOrC   c                 óÎ   — t        | |«      \  }}|j                  d   }t        j                  ||j                  ¬«      }||d   z
  }|j                  t        j                  «      S )zBCreate the relative position tensor for local -> global attention.r0   r{   ©.N)r’   r7   r:   rZ   rF   rt   Úint64)rh   rp   rr   r‘   Úglobal_seq_lenÚglobal_positionsÚside_relative_positions          rA   Ú _make_side_relative_position_idsr™   Í   sa   € ä$@ÀÐQbÓ$cÑ!€IÐ!Ø'×-Ñ-¨bÑ1€NÜ—|‘| N¸9×;KÑ;KÔLÐØ-°	¸)Ñ0DÑDÐØ!×&Ñ&¤u§{¡{Ó3Ð3rC   Úhidden_statesrr   r–   c                 óx  — |j                  |dk\  t        j                  ||j                  |j                  ¬«      «      }t
        j                  j                  |j                  t        j                  «      |dz   «      dd…dd…dd…f   }t        j                  d| |j                  | j                  «      «      S )zFCompute individual block aggregates by summing over individual blocks.r   rE   r#   Nr0   z...nd,...ng->...gd)ru   r:   rƒ   r.   rF   r   r>   Úone_hotrt   r•   Úeinsum)rš   rr   r–   Úone_hot_block_idss       rA   Ú_create_global_aggregatesrŸ   Ö   s›   € ð
 —‘Ø�Q‰œŸ™ ^¸9¿?¹?ÐS\×ScÑScÔdó€Iô Ÿ™×-Ñ-¨i¯n©n¼U¿[¹[Ó.IÈ>Ð\]ÑK]Ó^Ò_`ÒbcÐehÐfhÐehÐ_hÑiÐÜ�<‰<Ð,¨mÐ=N×=SÑ=SÐTa×TgÑTgÓ=hÓiÐirC   c                   ó&   ‡ — e Zd Zdˆ fd„	Zd„ Zˆ xZS )ÚLongT5LayerNormc                 óŠ   •— t         ‰| �  «        t        j                  t	        j
                  |«      «      | _        || _        y)zg
        Construct a layernorm module in the LongT5 style. No bias and no subtraction of mean.
        N)ÚsuperÚ__init__r   Ú	Parameterr:   rˆ   ÚweightÚvariance_epsilon)ÚselfÚhidden_sizeÚepsÚ	__class__s      €rA   r¤   zLongT5LayerNorm.__init__ä   s1   ø€ ô 	‰ÑÔÜ—l‘l¤5§:¡:¨kÓ#:Ó;ˆŒØ #ˆÕrC   c                 óž  — |j                  t        j                  «      j                  d«      j	                  dd¬«      }|t        j
                  || j                  z   «      z  }| j                  j                  t        j                  t        j                  fv r%|j                  | j                  j                  «      }| j                  |z  S )Nrj   r0   T)Úkeepdim)rd   r:   Úfloat32ÚpowÚmeanÚrsqrtr§   r¦   r.   Úfloat16Úbfloat16)r¨   rš   Úvariances      rA   ÚforwardzLongT5LayerNorm.forwardì   s›   € ð !×#Ñ#¤E§M¡MÓ2×6Ñ6°qÓ9×>Ñ>¸rÈ4Ð>ÓPˆØ%¬¯©°H¸t×?TÑ?TÑ4TÓ(UÑUˆð �;‰;×Ñ¤§¡´·±Ð ?Ñ?Ø)×,Ñ,¨T¯[©[×->Ñ->Ó?ˆMà�{‰{˜]Ñ*Ð*rC   )g�íµ ÷Æ°>)Ú__name__Ú
__module__Ú__qualname__r¤   rµ   Ú__classcell__©r«   s   @rA   r¡   r¡   ã   s   ø„ õ$ö+rC   r¡   )ÚFusedRMSNormzSDiscovered apex.normalization.FusedRMSNorm - will use it instead of LongT5LayerNormzFdiscovered apex but it failed to load, falling back to LongT5LayerNormc                   ó*   ‡ — e Zd Zdefˆ fd„Zd„ Zˆ xZS )ÚLongT5DenseActDenseÚconfigc                 ó^  •— t         ‰| �  «        t        j                  |j                  |j
                  d¬«      | _        t        j                  |j
                  |j                  d¬«      | _        t        j                  |j                  «      | _
        t        |j                     | _        y ©NF©Úbias)r£   r¤   r   ÚLinearÚd_modelÚd_ffÚwiÚwoÚDropoutÚdropout_rateÚdropoutr   Údense_act_fnÚact©r¨   r¾   r«   s     €rA   r¤   zLongT5DenseActDense.__init__  sn   ø€ Ü‰ÑÔÜ—)‘)˜FŸN™N¨F¯K©K¸eÔDˆŒÜ—)‘)˜FŸK™K¨¯©¸eÔDˆŒÜ—z‘z &×"5Ñ"5Ó6ˆŒÜ˜&×-Ñ-Ñ.ˆ�rC   c                 ó  — | j                  |«      }| j                  |«      }| j                  |«      }t        | j                  j
                  t        j                  «      r�|j                  | j                  j
                  j                  k7  r`| j                  j
                  j                  t        j                  k7  r/|j                  | j                  j
                  j                  «      }| j	                  |«      }|S ©N)rÆ   rÌ   rÊ   Ú
isinstancerÇ   r¦   r:   r   r.   Úint8rd   )r¨   rš   s     rA   rµ   zLongT5DenseActDense.forward  sª   € ØŸ™ Ó.ˆØŸ™ Ó/ˆØŸ™ ]Ó3ˆä�t—w‘w—~‘~¤u§|¡|Ô4Ø×#Ñ# t§w¡w§~¡~×';Ñ';Ò;Ø—‘—‘×$Ñ$¬¯
©
Ò2à)×,Ñ,¨T¯W©W¯^©^×-AÑ-AÓBˆMØŸ™ Ó.ˆØÐrC   ©r¶   r·   r¸   r$   r¤   rµ   r¹   rº   s   @rA   r½   r½     s   ø„ ð/˜|õ /örC   r½   c                   ó*   ‡ — e Zd Zdefˆ fd„Zd„ Zˆ xZS )ÚLongT5DenseGatedActDenser¾   c                 óÀ  •— t         ‰| �  «        t        j                  |j                  |j
                  d¬«      | _        t        j                  |j                  |j
                  d¬«      | _        t        j                  |j
                  |j                  d¬«      | _        t        j                  |j                  «      | _        t        |j                     | _        y rÀ   )r£   r¤   r   rÃ   rÄ   rÅ   Úwi_0Úwi_1rÇ   rÈ   rÉ   rÊ   r   rË   rÌ   rÍ   s     €rA   r¤   z!LongT5DenseGatedActDense.__init__$  sŠ   ø€ Ü‰ÑÔÜ—I‘I˜fŸn™n¨f¯k©kÀÔFˆŒ	Ü—I‘I˜fŸn™n¨f¯k©kÀÔFˆŒ	Ü—)‘)˜FŸK™K¨¯©¸eÔDˆŒÜ—z‘z &×"5Ñ"5Ó6ˆŒÜ˜&×-Ñ-Ñ.ˆ�rC   c                 ó¶   — | j                  | j                  |«      «      }| j                  |«      }||z  }| j                  |«      }| j	                  |«      }|S rÏ   )rÌ   rÖ   r×   rÊ   rÇ   )r¨   rš   Úhidden_geluÚhidden_linears       rA   rµ   z LongT5DenseGatedActDense.forward,  sS   € Ø—h‘h˜tŸy™y¨Ó7Ó8ˆØŸ	™	 -Ó0ˆØ# mÑ3ˆØŸ™ ]Ó3ˆØŸ™ Ó.ˆØÐrC   rÒ   rº   s   @rA   rÔ   rÔ   #  s   ø„ ð/˜|õ /örC   rÔ   c                   ó*   ‡ — e Zd Zdefˆ fd„Zd„ Zˆ xZS )ÚLongT5LayerFFr¾   c                 ó  •— t         ‰| �  «        |j                  rt        |«      | _        nt        |«      | _        t        |j                  |j                  ¬«      | _	        t        j                  |j                  «      | _        y )N©rª   )r£   r¤   Úis_gated_actrÔ   ÚDenseReluDenser½   r¡   rÄ   Úlayer_norm_epsilonÚ
layer_normr   rÈ   rÉ   rÊ   rÍ   s     €rA   r¤   zLongT5LayerFF.__init__7  s_   ø€ Ü‰ÑÔØ×ÒÜ":¸6Ó"BˆDÕä"5°fÓ"=ˆDÔä)¨&¯.©.¸f×>WÑ>WÔXˆŒÜ—z‘z &×"5Ñ"5Ó6ˆ�rC   c                 ór   — | j                  |«      }| j                  |«      }|| j                  |«      z   }|S rÏ   )râ   rà   rÊ   )r¨   rš   Úforwarded_statess      rA   rµ   zLongT5LayerFF.forwardA  s=   € ØŸ?™?¨=Ó9ÐØ×.Ñ.Ð/?Ó@ÐØ%¨¯©Ð5EÓ(FÑFˆØÐrC   rÒ   rº   s   @rA   rÜ   rÜ   6  s   ø„ ð7˜|õ 7örC   rÜ   c                   ón   ‡ — e Zd Z	 	 ddedee   fˆ fd„Zd„ Zed	d„«       Z	d
d„Z
	 	 	 	 	 	 	 	 	 dd„Zˆ xZS )ÚLongT5Attentionr¾   Ú	layer_idxc                 ó  •— t         ‰| �  «        |j                  | _        || _        |j                  | _        |j
                  | _        |j                  | _        |j                  | _        |j                  | _
        |j                  | _        | j                  | j                  z  | _        || _        |€9| j                  r-t        j!                  d| j"                  j$                  › d�«       t'        j(                  | j                  | j                  d¬«      | _        t'        j(                  | j                  | j                  d¬«      | _        t'        j(                  | j                  | j                  d¬«      | _        t'        j(                  | j                  | j                  d¬«      | _        | j                  r/t'        j2                  | j                  | j                  «      | _        t7        «       | _        d| _        y )NzInstantiating a decoder z³ without passing `layer_idx` is not recommended and will to errors during the forward call, if caching is used. Please make sure to provide a `layer_idx` when creating this class.FrÁ   )r£   r¤   Ú
is_decoderÚhas_relative_attention_biasÚrelative_attention_num_bucketsÚrelative_attention_max_distancerÄ   Úd_kvÚkey_value_proj_dimÚ	num_headsÚn_headsrÉ   rÊ   Ú	inner_dimrç   ÚloggerÚwarning_oncer«   r¶   r   rÃ   ÚqÚkÚvÚoÚ	EmbeddingÚrelative_attention_biasÚsetÚpruned_headsÚgradient_checkpointing©r¨   r¾   rê   rç   r«   s       €rA   r¤   zLongT5Attention.__init__J  ss  ø€ ô 	‰ÑÔØ ×+Ñ+ˆŒØ+FˆÔ(Ø.4×.SÑ.SˆÔ+Ø/5×/UÑ/UˆÔ,Ø—~‘~ˆŒØ"(§+¡+ˆÔØ×'Ñ'ˆŒØ×*Ñ*ˆŒØŸ™¨×(?Ñ(?Ñ?ˆŒØ"ˆŒØÐ §¢Ü×ÑØ*¨4¯>©>×+BÑ+BÐ*Cð D,ð ,ôô —‘˜4Ÿ<™<¨¯©¸eÔDˆŒÜ—‘˜4Ÿ<™<¨¯©¸eÔDˆŒÜ—‘˜4Ÿ<™<¨¯©¸eÔDˆŒÜ—‘˜4Ÿ>™>¨4¯<©<¸eÔDˆŒà×+Ò+Ü+-¯<©<¸×8[Ñ8[Ð]a×]iÑ]iÓ+jˆDÔ(Ü›EˆÔØ&+ˆÕ#rC   c                 ó  — t        |«      dk(  ry t        || j                  | j                  | j                  «      \  }}t        | j                  |«      | _        t        | j                  |«      | _        t        | j                  |«      | _        t        | j                  |d¬«      | _	        | j                  t        |«      z
  | _        | j                  | j                  z  | _
        | j                  j                  |«      | _        y ©Nr   r#   rO   ©Úlenr   rð   rî   rû   r   rô   rõ   rö   r÷   rñ   Úunion©r¨   ÚheadsÚindexs      rA   Úprune_headszLongT5Attention.prune_headsm  óÆ   € Üˆu‹:˜Š?ØÜ7Ø�4—<‘< ×!8Ñ!8¸$×:KÑ:Kó
‰ˆˆuô $ D§F¡F¨EÓ2ˆŒÜ# D§F¡F¨EÓ2ˆŒÜ# D§F¡F¨EÓ2ˆŒÜ# D§F¡F¨E°qÔ9ˆŒà—|‘|¤c¨%£jÑ0ˆŒØ×0Ñ0°4·<±<Ñ?ˆŒØ ×-Ñ-×3Ñ3°EÓ:ˆÕrC   c                 óT  — d}|rC|dz  }|| dkD  j                  t        j                  «      |z  z  }t        j                  | «      } n*t        j                  | t        j
                  | «      «       } |dz  }| |k  }|t        j                  | j                  «       |z  «      t        j                  ||z  «      z  ||z
  z  j                  t        j                  «      z   }t        j                  |t        j                  ||dz
  «      «      }|t        j                  || |«      z  }|S ©aÒ  
        Adapted from Mesh Tensorflow:
        https://github.com/tensorflow/mesh/blob/0cb87fe07da627bf0b7e60475d59f95ed6b5be3d/mesh_tensorflow/transformer/transformer_layers.py#L593

        Translate relative position to a bucket number for relative attention. The relative position is defined as
        memory_position - query_position, i.e. the distance in tokens from the attending position to the attended-to
        position. If bidirectional=False, then positive relative positions are invalid. We use smaller buckets for
        small absolute relative_position and larger buckets for larger absolute relative_positions. All relative
        positions >=max_distance map to the same bucket. All relative positions <=-max_distance map to the same bucket.
        This should allow for more graceful generalization to longer sequences than the model has been trained on

        Args:
            relative_position: an int32 Tensor
            bidirectional: a boolean - whether the attention is bidirectional
            num_buckets: an integer
            max_distance: an integer

        Returns:
            a Tensor with the same shape as relative_position, containing int32 values in the range [0, num_buckets)
        r   rj   r#   ©rd   r:   Úlongrc   ÚminÚ
zeros_likeÚlogÚfloatÚmathÚ	full_likeru   ©Úrelative_positionÚbidirectionalÚnum_bucketsÚmax_distanceÚrelative_bucketsÚ	max_exactÚis_smallÚrelative_position_if_larges           rA   Ú_relative_position_bucketz)LongT5Attention._relative_position_bucket}  s(  € ð, ÐÙØ˜AÑˆKØÐ!2°QÑ!6× :Ñ :¼5¿:¹:Ó FÈÑ TÑTÐÜ %§	¡	Ð*;Ó <Ñä!&§¡Ð+<¼e×>NÑ>NÐO`Ó>aÓ!bÐ bÐð   1Ñ$ˆ	Ø$ yÑ0ˆð &/Ü�I‰IÐ'×-Ñ-Ó/°)Ñ;Ó<Ü�h‰h�| iÑ/Ó0ñ1à˜YÑ&ñ(÷ ‰"ŒU�Z‰Z‹.ñ	&Ð"ô
 &+§Y¡YØ&¬¯©Ð8RÐT_ÐbcÑTcÓ(dó&
Ð"ð 	œEŸK™K¨Ð2CÐE_Ó`Ñ`ÐØÐrC   c                 ó  — |€ | j                   j                  j                  }|€.t        j                  |t        j
                  |¬«      dd…df   }n|dd…df   j                  |«      }t        j                  |t        j
                  |¬«      ddd…f   }||z
  }| j                  || j                   | j                  | j                  ¬«      }| j                  |«      }	|	j                  g d¢«      j                  d«      }	|	S )ú%Compute binned relative position biasNrE   ©r  r  r  ©rj   r   r#   r   )rù   r¦   rF   r:   rZ   r  rd   r  ré   rë   rì   Úpermuter\   )
r¨   Úquery_lengthÚ
key_lengthrF   Úcache_positionÚcontext_positionÚmemory_positionr  Úrelative_position_bucketr…   s
             rA   Úcompute_biaszLongT5Attention.compute_bias­  s÷   € àˆ>Ø×1Ñ1×8Ñ8×?Ñ?ˆFØÐ!Ü$Ÿ|™|¨LÄÇ
Á
ÐSYÔZÒ[\Ð^bÐ[bÑcÑà-ªa°¨gÑ6×9Ñ9¸&ÓAÐÜŸ,™, z¼¿¹ÈFÔSÐTXÒZ[ÐT[Ñ\ˆØ+Ð.>Ñ>ÐØ#'×#AÑ#AØØ#Ÿ™Ð.Ø×;Ñ;Ø×=Ñ=ð	 $Bó $
Ð ð ×-Ñ-Ð.FÓGˆØ—‘¢	Ó*×4Ñ4°QÓ7ˆØˆrC   c                 ó  — |j                   dd \  }}|du}| j                  |«      }|j                  |d| j                  | j                  «      j                  dd«      }|�@|j                  j                  | j                  «      }|r|j                  }n|j                  }|r|n|}|r7|�5r3j                  | j                     }|j                  | j                     }nØ| j                  |«      }| j                  |«      }|j                  |d| j                  | j                  «      j                  dd«      }|j                  |d| j                  | j                  «      j                  dd«      }|�D|s|
nd}
j                  ||| j                  d|
i«      \  }}|rd|j                  | j                  <   t!        j"                  ||j                  dd«      «      }|€×|j                   d   }|�|n|
d   dz   }| j$                  sZt!        j&                  d| j                  ||f|j(                  |j*                  ¬	«      }| j,                  rE| j.                  r9d|_        n1| j3                  |||j(                  |
¬
«      }|dd…dd…| d…dd…f   }|�#|dd…dd…dd…d|j                   d   …f   }||z   }| j4                  rRt!        j6                  |j                   d   «      }d|t9        | j4                  «      <   |dd…|j;                  «       f   }n|}||z  }t<        j>                  jA                  |jC                  «       d¬«      jE                  |«      }t<        j>                  jG                  || jF                  | j.                  ¬«      }|�||z  }t!        j"                  ||«      }|j                  dd«      jI                  «       }|j                  |d| jJ                  «      }| jM                  |«      }|||f}|	r||fz   }|S )z€
        Self-attention (if key_value_states is None) or attention over source sentence (provided by key_value_states).
        Nrj   r0   r#   r#  Tr
   rl   ©rF   r.   )rF   r#  r   rO   ©ÚpÚtraining)'r7   rô   Úviewrð   rî   r‡   Ú
is_updatedÚgetrç   Úcross_attention_cacheÚself_attention_cacheÚ	key_cacheÚvalue_cacherõ   rö   Úupdater:   Úmatmulrê   r;   rF   r.   rü   r,  Úrequires_gradr'  rû   rˆ   r9   Úboolr   r>   Úsoftmaxr  Útype_asrÊ   Ú
contiguousrñ   r÷   )r¨   rš   rŒ   Úkey_value_statesÚposition_biasÚpast_key_valueÚlayer_head_maskr!  Ú	use_cacheÚoutput_attentionsr#  rŠ   Ú
seq_lengthÚis_cross_attentionÚquery_statesr.  Úcurr_past_key_valueÚcurrent_statesÚ
key_statesÚvalue_statesÚscoresr"  Úreal_seq_lengthÚcausal_maskÚposition_bias_maskedÚattn_weightsÚattn_outputÚoutputss                               rA   rµ   zLongT5Attention.forwardÁ  sã  € ð$ "/×!4Ñ!4°R°aÐ!8Ñˆ
�Jð .°TÐ9Ðà—v‘v˜mÓ,ˆØ#×(Ñ(¨°R¸¿¹Àt×G^ÑG^Ó_×iÑiÐjkÐmnÓoˆàÐ%Ø'×2Ñ2×6Ñ6°t·~±~ÓFˆJÙ!à&4×&JÑ&JÑ#à&4×&IÑ&IÐ#á-?Ñ)À]ˆÙ .Ð"<Áà,×6Ñ6°t·~±~ÑFˆJØ.×:Ñ:¸4¿>¹>ÑJ‰LàŸ™ Ó/ˆJØŸ6™6 .Ó1ˆLØ#Ÿ™¨°R¸¿¹Àt×G^ÑG^Ó_×iÑiÐjkÐmnÓoˆJØ'×,Ñ,¨Z¸¸T¿\¹\È4×KbÑKbÓc×mÑmÐnoÐqrÓsˆLàÐ)á7I¡Èt�Ø+>×+EÑ+EØ ¨d¯n©nÐ?OÐQ_Ð>`ó,Ñ(�
˜Lñ &Ø@D�N×-Ñ-¨d¯n©nÑ=ô —‘˜l¨J×,@Ñ,@ÀÀAÓ,FÓGˆàÐ Ø#×)Ñ)¨"Ñ-ˆJà.:Ð.F™lÈNÐ[]ÑL^ÐabÑLbˆOØ×3Ò3Ü %§¡Ø˜Ÿ™ j°*Ð=ÀfÇmÁmÐ[a×[gÑ[gô!�ð ×.Ò.°4·=²=Ø26�MÕ/à $× 1Ñ 1Ø# Z¸¿¹ÐVdð !2ó !�ð !.ªa²°Z°K±LÂ!Ð.CÑ D�àÐØ"¢1¢aªÐ,B¨j×.>Ñ.>¸rÑ.BÐ,BÐ#BÑC�Ø -°Ñ ;�à×ÒÜ—:‘:˜m×1Ñ1°!Ñ4Ó5ˆDØ,-ˆD”�d×'Ñ'Ó(Ñ)Ø#0²°D·I±I³K°Ñ#@Ñ à#0Ð àÐ&Ñ&ˆô —}‘}×,Ñ,¨V¯\©\«^ÀÐ,ÓD×LÑLÈVÓTˆÜ—}‘}×,Ñ,¨\¸T¿\¹\ÐTX×TaÑTaÐ,Óbˆð Ð&Ø'¨/Ñ9ˆLä—l‘l <°Ó>ˆà!×+Ñ+¨A¨qÓ1×<Ñ<Ó>ˆØ!×&Ñ& z°2°t·~±~ÓFˆØ—f‘f˜[Ó)ˆà °Ð>ˆáØ  Ñ/ˆGØˆrC   ©FN©Té    é€   )NN)	NNNNNNFFN)r¶   r·   r¸   r$   r   r‰   r¤   r  Ústaticmethodr  r'  rµ   r¹   rº   s   @rA   ræ   ræ   I  si   ø„ ð %*Ø#'ñ	!,àð!,ð ˜C‘=õ	!,òF;ð  ò- ó ð- ó^ð. ØØØØØØØØ÷irC   ræ   c                   ób   ‡ — e Zd Zddededdfˆ fd„Zd„ Zedd„«       Zde	fd	„Z
	 	 	 	 dd
„Zˆ xZS )ÚLongT5LocalAttentionr¾   rê   r+   Nc                 óÎ  •— t         ‰| �  «        |j                  | _        || _        |j                  | _        |j
                  | _        |j                  | _        |j                  | _        |j                  | _
        |j                  | _        | j                  dz   | _        |j                  | _        | j                  | j                  z  | _        t!        j"                  | j                  | j                  d¬«      | _        t!        j"                  | j                  | j                  d¬«      | _        t!        j"                  | j                  | j                  d¬«      | _        t!        j"                  | j                  | j                  d¬«      | _        | j                  r/t!        j,                  | j                  | j                  «      | _        t1        «       | _        d| _        y )Nr#   FrÁ   )r£   r¤   ré   rê   rë   rì   rÄ   rí   rî   rï   rð   Úlocal_radiusr(   rÉ   rÊ   rñ   r   rÃ   rô   rõ   rö   r÷   rø   rù   rú   rû   rü   ©r¨   r¾   rê   r«   s      €rA   r¤   zLongT5LocalAttention.__init__.  sS  ø€ Ü‰ÑÔØ ×+Ñ+ˆŒØ+FˆÔ(Ø.4×.SÑ.SˆÔ+Ø/5×/UÑ/UˆÔ,Ø—~‘~ˆŒØ"(§+¡+ˆÔØ×'Ñ'ˆŒØ"×/Ñ/ˆÔØ×*Ñ*¨QÑ.ˆŒØ×*Ñ*ˆŒØŸ™¨×(?Ñ(?Ñ?ˆŒô —‘˜4Ÿ<™<¨¯©¸eÔDˆŒÜ—‘˜4Ÿ<™<¨¯©¸eÔDˆŒÜ—‘˜4Ÿ<™<¨¯©¸eÔDˆŒÜ—‘˜4Ÿ>™>¨4¯<©<¸eÔDˆŒà×+Ò+Ü+-¯<©<¸×8[Ñ8[Ð]a×]iÑ]iÓ+jˆDÔ(Ü›EˆÔØ&+ˆÕ#rC   c                 ó  — t        |«      dk(  ry t        || j                  | j                  | j                  «      \  }}t        | j                  |«      | _        t        | j                  |«      | _        t        | j                  |«      | _        t        | j                  |d¬«      | _	        | j                  t        |«      z
  | _        | j                  | j                  z  | _
        | j                  j                  |«      | _        y rÿ   r   r  s      rA   r  z LongT5LocalAttention.prune_headsH  r  rC   c                 óT  — d}|rC|dz  }|| dkD  j                  t        j                  «      |z  z  }t        j                  | «      } n*t        j                  | t        j
                  | «      «       } |dz  }| |k  }|t        j                  | j                  «       |z  «      t        j                  ||z  «      z  ||z
  z  j                  t        j                  «      z   }t        j                  |t        j                  ||dz
  «      «      }|t        j                  || |«      z  }|S r	  r
  r  s           rA   r  z.LongT5LocalAttention._relative_position_bucketX  ó(  € ð. ÐÙØ˜AÑˆKØÐ!2°QÑ!6× :Ñ :¼5¿:¹:Ó FÈÑ TÑTÐÜ %§	¡	Ð*;Ó <Ñä!&§¡Ð+<¼e×>NÑ>NÐO`Ó>aÓ!bÐ bÐð   1Ñ$ˆ	Ø$ yÑ0ˆð &/Ü�I‰IÐ'×-Ñ-Ó/°)Ñ;Ó<Ü�h‰h�| iÑ/Ó0ñ1à˜YÑ&ñ(÷ ‰"ŒU�Z‰Z‹.ñ	&Ð"ô
 &+§Y¡YØ&¬¯©Ð8RÐT_ÐbcÑTcÓ(dó&
Ð"ð 	œEŸK™K¨Ð2CÐE_Ó`Ñ`ÐØÐrC   Úblock_lengthc                 ó  — | j                   j                  j                  j                  dk7  r | j                   j                  j                  nd}t	        j
                  d|z  t        j                  |¬«      }|||  }|ddd…f   |dd…df   z
  }| j                  || j                   | j                  | j                  ¬«      }| j                  |«      }|j                  g d¢«      j                  d«      j                  d«      }|S ©r  ÚmetaNr
   rE   r  r  r   ©rù   r¦   rF   rt   r:   rZ   r  r  ré   rë   rì   r   r\   ©r¨   r\  Útarget_devicer%  r$  r  r&  r…   s           rA   r'  z!LongT5LocalAttention.compute_bias‰  ó  € ð ×+Ñ+×2Ñ2×9Ñ9×>Ñ>À&ÒHð ×(Ñ(×/Ñ/×6Ò6àð 	ô
  Ÿ,™, q¨<Ñ'7¼u¿z¹zÐR_Ô`ˆØ*¨<¸¸ÐFÐð ,¨D²!¨GÑ4Ð7GÊÈ4ÈÑ7PÑPÐØ#'×#AÑ#AØØ#Ÿ™Ð.Ø×;Ñ;Ø×=Ñ=ð	 $Bó $
Ð ð ×-Ñ-Ð.FÓGˆà—‘¢	Ó*×4Ñ4°QÓ7×AÑAÀ!ÓDˆØˆrC   c                 ó6  ‡ ‡— |j                   d d \  Š}ˆˆ fd„}ˆˆ fd„} |‰ j                  |«      «      }	 |‰ j                  |«      «      }
 |‰ j                  |«      «      }t	        |	‰ j
                  d¬«      }	t	        |
‰ j
                  d¬«      }
t	        |‰ j
                  d¬«      }t        |
dd¬«      }
t        |dd¬«      }t        j                  d|	|
«      }|€Ê‰ j                  srt        j                  dd‰ j                  ‰ j
                  d‰ j
                  z  f|j                  |j                  ¬	«      }‰ j                  r/‰ j                  r#d
|_        n‰ j#                  ‰ j
                  «      }|�/t        j$                  |dkD  dd«      }||j'                  dd«      z   }||z  }t(        j*                  j-                  |j/                  «       d¬«      j1                  |«      }t(        j*                  j3                  |‰ j2                  ‰ j                  ¬«      }|�||z  }|j5                  |j                  «      } |t        j                  d||«      «      }|d d …d |…d d …f   }‰ j7                  |«      }d }|f|fz   |fz   }|r||fz   }|S )Nrj   c                 óT   •— | j                  ‰d‰j                  ‰j                  «      S ©Ú
projectionr0   ©r-  rð   rî   ©ÚstatesrŠ   r¨   s    €€rA   r7   z+LongT5LocalAttention.forward.<locals>.shape«  ó"   ø€ à—;‘;˜z¨2¨t¯|©|¸T×=TÑ=TÓUÐUrC   c                 óZ   •— | j                  «       j                  ‰d‰j                  «      S ©rH   r0   ©r:  r-  rñ   ri  s    €€rA   Úunshapez-LongT5LocalAttention.forward.<locals>.unshape¯  ó%   ø€ à×$Ñ$Ó&×+Ñ+¨J¸¸D¿N¹NÓKÐKrC   r#   rO   rk   ú...qhd,...khd->...hqkr
   r)  Tr   r}   ç    _ Âr0   r*  ú...hqk,...khd->...qhd)r7   rô   rõ   rö   rK   r(   rX   r:   r�   rê   r;   rð   rF   r.   rü   r,  r6  r'  ru   r‡   r   r>   r8  r  r9  rÊ   rt   r÷   )r¨   rš   rŒ   r<  r>  r@  rA  r7   ro  rC  rF  rG  rH  rL  rM  Úpresent_key_value_staterN  rŠ   s   `                @rA   rµ   zLongT5LocalAttention.forward¡  sp  ù€ ð "/×!4Ñ!4°R°aÐ!8Ñˆ
�Jõ	Võ	Lñ
 ˜TŸV™V MÓ2Ó3ˆÙ˜4Ÿ6™6 -Ó0Ó1ˆ
Ù˜TŸV™V MÓ2Ó3ˆô *¨,¸¿¹ÈAÔNˆÜ'¨
°D·N±NÈÔJˆ
Ü)¨,¸¿¹ÈAÔNˆô +¨:ÀÐQRÔSˆ
Ü,¨\ÀQÐUVÔWˆô —‘Ø# \°:ó
ˆð Ð à×3Ò3Ü %§¡Ø˜˜4Ÿ<™<¨¯©¸¸T¿^¹^Ñ9KÐLÐU[×UbÑUbÐjp×jvÑjvô!�ð ×.Ò.°4·=²=Ø26�MÕ/à $× 1Ñ 1°$·.±.Ó A�àÐä—{‘{ 4¨!¡8¨S°%Ó8�à -°·±¸qÀ!Ó0DÑ D�à�-Ñˆä—}‘}×,Ñ,¨V¯\©\«^ÀÐ,ÓD×LÑLÈVÓTˆä—}‘}×,Ñ,¨\¸T¿\¹\ÐTX×TaÑTaÐ,Óbˆð Ð&Ø'¨/Ñ9ˆLØ#×(Ñ(¨×);Ñ);Ó<ˆÙœeŸl™lÐ+BÀLÐR^Ó_Ó`ˆØ!¢! [ j [²!Ð"3Ñ4ˆØ—f‘f˜[Ó)ˆà"&ÐØ�.Ð$;Ð#=Ñ=ÀÐ@PÑPˆáØ  Ñ/ˆGØˆrC   ©FrP  ©NNNF)r¶   r·   r¸   r$   r7  r¤   r  rS  r  r‰   r'  rµ   r¹   rº   s   @rA   rU  rU  -  sX   ø„ ñ,˜|ð ,È$ð ,Ð[_õ ,ò4;ð  ò- ó ð- ð^¨ó ð6 ØØØ÷IrC   rU  c                   ó²   ‡ — e Zd Zddededdfˆ fd„Zd„ Zedd„«       Zde	fd	„Z
d
ej                  dej                  dej                  fd„Z	 	 	 	 dd„Zˆ xZS )ÚLongT5TransientGlobalAttentionr¾   rê   r+   Nc                 ó¤  •— t         ‰| �  «        |j                  | _        || _        |j                  | _        |j
                  | _        |j                  | _        |j                  | _        |j                  | _
        |j                  | _        | j                  dz   | _        |j                  | _        |j                  | _        | j                  | j                  z  | _        t#        j$                  | j                  | j                   d¬«      | _        t#        j$                  | j                  | j                   d¬«      | _        t#        j$                  | j                  | j                   d¬«      | _        t#        j$                  | j                   | j                  d¬«      | _        | j                  r/t#        j.                  | j                  | j                  «      | _        t3        «       | _        | j                  r/t#        j.                  | j                  | j                  «      | _        t9        |j                  |j:                  ¬«      | _        y )Nr#   FrÁ   rÞ   )r£   r¤   ré   rê   rë   rì   rÄ   rí   rî   rï   rð   rW  r(   rp   rÉ   rÊ   rñ   r   rÃ   rô   rõ   rö   r÷   rø   rù   rú   rû   Úglobal_relative_attention_biasr¡   rá   Úglobal_input_layer_normrX  s      €rA   r¤   z'LongT5TransientGlobalAttention.__init__î  s�  ø€ Ü‰ÑÔØ ×+Ñ+ˆŒØ+FˆÔ(Ø.4×.SÑ.SˆÔ+Ø/5×/UÑ/UˆÔ,Ø—~‘~ˆŒØ"(§+¡+ˆÔØ×'Ñ'ˆŒØ"×/Ñ/ˆÔØ×*Ñ*¨QÑ.ˆŒØ!'×!9Ñ!9ˆÔØ×*Ñ*ˆŒØŸ™¨×(?Ñ(?Ñ?ˆŒô —‘˜4Ÿ<™<¨¯©¸eÔDˆŒÜ—‘˜4Ÿ<™<¨¯©¸eÔDˆŒÜ—‘˜4Ÿ<™<¨¯©¸eÔDˆŒÜ—‘˜4Ÿ>™>¨4¯<©<¸eÔDˆŒà×+Ò+Ü+-¯<©<¸×8[Ñ8[Ð]a×]iÑ]iÓ+jˆDÔ(Ü›EˆÔð ×+Ò+Ü24·,±,¸t×?bÑ?bÐdh×dpÑdpÓ2qˆDÔ/Ü'6°v·~±~È6×KdÑKdÔ'eˆÕ$rC   c                 ó  — t        |«      dk(  ry t        || j                  | j                  | j                  «      \  }}t        | j                  |«      | _        t        | j                  |«      | _        t        | j                  |«      | _        t        | j                  |d¬«      | _	        | j                  t        |«      z
  | _        | j                  | j                  z  | _
        | j                  j                  |«      | _        y rÿ   r   r  s      rA   r  z*LongT5TransientGlobalAttention.prune_heads  r  rC   c                 óT  — d}|rC|dz  }|| dkD  j                  t        j                  «      |z  z  }t        j                  | «      } n*t        j                  | t        j
                  | «      «       } |dz  }| |k  }|t        j                  | j                  «       |z  «      t        j                  ||z  «      z  ||z
  z  j                  t        j                  «      z   }t        j                  |t        j                  ||dz
  «      «      }|t        j                  || |«      z  }|S r	  r
  r  s           rA   r  z8LongT5TransientGlobalAttention._relative_position_bucket  r[  rC   r\  c                 ó  — | j                   j                  j                  j                  dk7  r | j                   j                  j                  nd}t	        j
                  d|z  t        j                  |¬«      }|||  }|ddd…f   |dd…df   z
  }| j                  || j                   | j                  | j                  ¬«      }| j                  |«      }|j                  g d¢«      j                  d«      j                  d«      }|S r^  r`  ra  s           rA   r'  z+LongT5TransientGlobalAttention.compute_biasN  rc  rC   rŒ   r‘   c                 óv  — t        j                  |d   |d d …d d d …f   «      d d …d df   }t        j                  |dkD  dd«      }t        || j                  «      }| j                  || j                   | j                  | j                  ¬«      }| j                  |«      }|j                  g d¢«      }||z   }|S )Nr”   .r   r}   rr  r  )r   r
   r#   rj   )r:   Úeqru   r™   rp   r  ré   rë   rì   rz  r   )r¨   rŒ   r‘   Úside_attention_maskÚattention_side_biasr˜   Úside_relative_position_bucketÚ	side_biass           rA   Úcompute_side_biasz0LongT5TransientGlobalAttention.compute_side_biasf  sË   € ä#Ÿh™h t¨I¡Ð8JÊ1ÈdÒTUÈ:Ñ8VÓWÒXYÐ[_ÐadÐXdÑeÐÜ#Ÿk™kÐ*=ÀÑ*AÀ3ÈÓNÐä!AÀ$È×H^ÑH^Ó!_ÐØ(,×(FÑ(FØ"Ø#Ÿ™Ð.Ø×;Ñ;Ø×=Ñ=ð	 )Gó )
Ð%ð ×7Ñ7Ð8UÓVˆ	ð ×%Ñ%¢lÓ3ˆ	à1°IÑ=ÐØ"Ð"rC   c                 óV	  ‡ ‡— |j                   d d \  Š}ˆˆ fd„}ˆˆ fd„}t        |�|n!t        j                  |j                   d d «      ‰ j                  «      \  }	}
|
j                   d   }t        ||	|«      }‰ j                  |«      } |‰ j                  |«      «      } |‰ j                  |«      «      } |‰ j                  |«      «      } |‰ j                  |«      «      } |‰ j                  |«      «      }t        |‰ j                  d¬«      }t        |‰ j                  d¬«      }t        |‰ j                  d¬«      }t        |dd¬«      }t        |dd¬«      }dg|j                  dz   z  }|j                   d   |d<   |j                  d«      j                  |«      }|j                  d«      j                  |«      }t        j                   ||gd¬«      }t        j                   ||gd¬«      }t        j"                  d||«      }|�<t%        |‰ j                  |j&                  «      }t        j(                  |d	kD  d
d«      }nd }|�€j‰ j*                  srt        j,                  dd‰ j.                  ‰ j                  d‰ j                  z  f|j&                  |j0                  ¬«      }‰ j2                  r/‰ j4                  r#d|_        n‰ j9                  ‰ j                  «      }|�||j;                  dd«      z   }|j=                  |j0                  «      }|€t        j                  ‰|«      }‰ j?                  ||
«      }t        |‰ j                  d¬«      j;                  dd«      }|j=                  |j0                  «      jA                  |j&                  «      }t        j                   ||gd¬«      }||z  }tB        jD                  jG                  |jI                  «       d¬«      jK                  |«      }tB        jD                  jM                  |‰ jL                  ‰ j4                  ¬«      }|�||z  }|j=                  |j0                  «      } |t        j"                  d||«      «      }|d d …d |…d d …f   }‰ jO                  |«      }d }|f|fz   |fz   }|r||fz   }|S )Nrj   c                 óT   •— | j                  ‰d‰j                  ‰j                  «      S rf  rh  ri  s    €€rA   r7   z5LongT5TransientGlobalAttention.forward.<locals>.shape…  rk  rC   c                 óZ   •— | j                  «       j                  ‰d‰j                  «      S rm  rn  ri  s    €€rA   ro  z7LongT5TransientGlobalAttention.forward.<locals>.unshape‰  rp  rC   r0   r#   rO   rk   rq  r   r}   rr  r
   r)  Trl   r*  rs  )(r7   r’   r:   rˆ   rp   rŸ   r{  rô   rõ   rö   rK   r(   rX   r<   r\   r†   rT   r�   ro   rF   ru   rê   r;   rð   r.   rü   r,  r6  r'  r‡   rt   r…  rd   r   r>   r8  r  r9  rÊ   r÷   )r¨   rš   rŒ   r<  r>  r@  rA  r7   ro  rr   r‘   Ú_global_seq_lenÚglobal_inputsrC  rF  rG  Úside_key_statesÚside_value_statesÚrepsrH  ra   Úside_position_biasrL  rM  rt  rN  rŠ   s   `                         @rA   rµ   z&LongT5TransientGlobalAttention.forward{  sV  ù€ ð "/×!4Ñ!4°R°aÐ!8Ñˆ
�Jõ	Võ	Lô )EØÐ$‰D¬%¯*©*°]×5HÑ5HÈÈ"Ð5MÓ*NØ×"Ñ"ó)
Ñ%ˆ	Ð%ð
 -×2Ñ2°2Ñ6ˆÜ1°-ÀÈOÓ\ˆØ×4Ñ4°]ÓCˆñ ˜TŸV™V MÓ2Ó3ˆÙ˜4Ÿ6™6 -Ó0Ó1ˆ
Ù˜TŸV™V MÓ2Ó3ˆá §¡ }Ó 5Ó6ˆÙ! $§&¡&¨Ó"7Ó8Ðô *¨,¸¿¹ÈAÔNˆÜ'¨
°D·N±NÈÔJˆ
Ü)¨,¸¿¹ÈAÔNˆô +¨:ÀÐQRÔSˆ
Ü,¨\ÀQÐUVÔWˆð ˆs�o×*Ñ*¨QÑ.Ñ/ˆØ×"Ñ" 1Ñ%ˆˆQ‰Ø)×3Ñ3°AÓ6×=Ñ=¸dÓCˆØ-×7Ñ7¸Ó:×AÑAÀ$ÓGÐô —Y‘Y 
¨OÐ<À!ÔDˆ
Ü—y‘y ,Ð0AÐ!BÈÔJˆô —‘Ð5°|ÀZÓPˆàÐä#<¸TÀ4Ç>Á>ÐS`×SgÑSgÓ#hÐ ä#(§;¡;Ð/CÀaÑ/GÈÈeÓ#TÑ à#'Ð àÑ à×3Ò3Ü %§¡Ø˜˜4Ÿ<™<¨¯©¸¸T¿^¹^Ñ9KÐLØ!Ÿ=™=Ø Ÿ,™,ô!�ð
 ×.Ò.°4·=²=Ø26�MÕ/à $× 1Ñ 1°$·.±.Ó A�à#Ð/à -Ð0D×0NÑ0NÈqÐRSÓ0TÑ T�Ø)×.Ñ.¨v¯|©|Ó<ˆMð ˆ|Ü—z‘z *¨jÓ9�à!%×!7Ñ!7¸Ð>PÓ!QÐä!3Ð4FÈÏÉÐ\^Ô!_×!iÑ!iÐjkÐmnÓ!oÐØ!3×!8Ñ!8¸¿¹Ó!F×!IÑ!IÈ&Ï-É-Ó!XÐä!ŸI™I }Ð6HÐ&IÈrÔRˆMà�-Ñˆä—}‘}×,Ñ,¨V¯\©\«^ÀÐ,ÓD×LÑLÈVÓTˆÜ—}‘}×,Ñ,¨\¸T¿\¹\ÐTX×TaÑTaÐ,Óbˆð Ð&Ø'¨/Ñ9ˆLØ#×(Ñ(¨×);Ñ);Ó<ˆÙœeŸl™lÐ+BÀLÐR^Ó_Ó`ˆØ!¢! [ j [²!Ð"3Ñ4ˆØ—f‘f˜[Ó)ˆà"&ÐØ�.Ð$;Ð#=Ñ=ÀÐ@PÑPˆáØ  Ñ/ˆGØˆrC   ru  rP  rv  )r¶   r·   r¸   r$   r7  r¤   r  rS  r  r‰   r'  r:   r   r…  rµ   r¹   rº   s   @rA   rx  rx  í  s…   ø„ ñf˜|ð fÈ$ð fÐ[_õ fò>;ð  ò- ó ð- ð^¨ó ð0# e§l¡lð #ÈÏÉð #ÐY^×YeÑYeó #ð0 ØØØ÷vrC   rx  c                   óB   ‡ — e Zd Zddee   fˆ fd„Z	 	 	 	 	 	 	 dd„Zˆ xZS )ÚLongT5LayerSelfAttentionrç   c                 óÜ   •— t         ‰| �  «        t        |||¬«      | _        t	        |j
                  |j                  ¬«      | _        t        j                  |j                  «      | _        y )N©rê   rç   rÞ   )r£   r¤   ræ   ÚSelfAttentionr¡   rÄ   rá   râ   r   rÈ   rÉ   rÊ   rý   s       €rA   r¤   z!LongT5LayerSelfAttention.__init__ö  sT   ø€ Ü‰ÑÔÜ,ØÐ0KÐW`ô
ˆÔô *¨&¯.©.¸f×>WÑ>WÔXˆŒÜ—z‘z &×"5Ñ"5Ó6ˆ�rC   c	           
      óš   — | j                  |«      }	| j                  |	|||||||¬«      }
|| j                  |
d   «      z   }|f|
dd  z   }|S )N)rŒ   r<  r>  r=  r?  r@  r#  r   r#   )râ   r“  rÊ   )r¨   rš   rh   r<  r>  r=  r?  r@  r#  Únormed_hidden_statesÚattention_outputrN  s               rA   rµ   z LongT5LayerSelfAttention.forwardþ  sv   € ð  $Ÿ™¨}Ó=ÐØ×-Ñ-Ø ØØ'Ø+Ø)ØØ/Ø)ð .ó 	
Ðð &¨¯©Ð5EÀaÑ5HÓ(IÑIˆØ Ð"Ð%5°a°bÐ%9Ñ9ˆØˆrC   rO  )NNNNFFN©r¶   r·   r¸   r   r‰   r¤   rµ   r¹   rº   s   @rA   r�  r�  õ  s0   ø„ ñ7ÈXÐVYÉ]õ 7ð ØØØØØØ÷rC   r�  c                   óF   ‡ — e Zd ZdZddee   fˆ fd„Z	 	 	 	 ddefd„Zˆ xZ	S )ÚLongT5LayerLocalSelfAttentionz$Local self attention used in encoderrç   c                 óÚ   •— t         ‰| �  «        t        ||¬«      | _        t	        |j
                  |j                  ¬«      | _        t        j                  |j                  «      | _        y ©N)rê   rÞ   )r£   r¤   rU  ÚLocalSelfAttentionr¡   rÄ   rá   râ   r   rÈ   rÉ   rÊ   rý   s       €rA   r¤   z&LongT5LayerLocalSelfAttention.__init__  sL   ø€ Ü‰ÑÔÜ"6°vÐ[vÔ"wˆÔÜ)¨&¯.©.¸f×>WÑ>WÔXˆŒÜ—z‘z &×"5Ñ"5Ó6ˆ�rC   Úkwargsc                 ó”   — | j                  |«      }| j                  |||||¬«      }|| j                  |d   «      z   }|f|dd  z   }	|	S ©N)rŒ   r<  r>  r@  r   r#   )râ   rœ  rÊ   ©
r¨   rš   rh   r<  r>  r@  r�  r•  r–  rN  s
             rA   rµ   z%LongT5LayerLocalSelfAttention.forward"  sm   € ð  $Ÿ™¨}Ó=ÐØ×2Ñ2Ø ØØ'Ø+Ø/ð 3ó 
Ðð &¨¯©Ð5EÀaÑ5HÓ(IÑIˆØ Ð"Ð%5°a°bÐ%9Ñ9ˆØˆrC   rO  rv  ©
r¶   r·   r¸   Ú__doc__r   r‰   r¤   r   rµ   r¹   rº   s   @rA   r™  r™    s4   ø„ Ù.ñ7ÈXÐVYÉ]õ 7ð ØØØñð ÷rC   r™  c                   óF   ‡ — e Zd ZdZddee   fˆ fd„Z	 	 	 	 ddefd„Zˆ xZ	S )Ú'LongT5LayerTransientGlobalSelfAttentionz/Transient-Global self attention used in encoderrç   c                 óÚ   •— t         ‰| �  «        t        ||¬«      | _        t	        |j
                  |j                  ¬«      | _        t        j                  |j                  «      | _        y r›  )r£   r¤   rx  ÚTransientGlobalSelfAttentionr¡   rÄ   rá   râ   r   rÈ   rÉ   rÊ   rý   s       €rA   r¤   z0LongT5LayerTransientGlobalSelfAttention.__init__;  sQ   ø€ Ü‰ÑÔÜ,JØÐ0Kô-
ˆÔ)ô *¨&¯.©.¸f×>WÑ>WÔXˆŒÜ—z‘z &×"5Ñ"5Ó6ˆ�rC   r�  c                 ó”   — | j                  |«      }| j                  |||||¬«      }|| j                  |d   «      z   }|f|dd  z   }	|	S rŸ  )râ   r¦  rÊ   r   s
             rA   rµ   z/LongT5LayerTransientGlobalSelfAttention.forwardC  sm   € ð  $Ÿ™¨}Ó=ÐØ×<Ñ<Ø ØØ'Ø+Ø/ð =ó 
Ðð &¨¯©Ð5EÀaÑ5HÓ(IÑIˆØ Ð"Ð%5°a°bÐ%9Ñ9ˆØˆrC   rO  rv  r¡  rº   s   @rA   r¤  r¤  8  s4   ø„ Ù9ñ7ÈXÐVYÉ]õ 7ð ØØØñð ÷rC   r¤  c                   óD   ‡ — e Zd Zddee   fˆ fd„Z	 	 	 	 	 	 	 	 dd„Zˆ xZS )ÚLongT5LayerCrossAttentionrç   c                 óÜ   •— t         ‰| �  «        t        |d|¬«      | _        t	        |j
                  |j                  ¬«      | _        t        j                  |j                  «      | _        y )NFr’  rÞ   )r£   r¤   ræ   ÚEncDecAttentionr¡   rÄ   rá   râ   r   rÈ   rÉ   rÊ   )r¨   r¾   rç   r«   s      €rA   r¤   z"LongT5LayerCrossAttention.__init__[  sO   ø€ Ü‰ÑÔÜ.¨vÐSXÐdmÔnˆÔÜ)¨&¯.©.¸f×>WÑ>WÔXˆŒÜ—z‘z &×"5Ñ"5Ó6ˆ�rC   c                 óž   — | j                  |«      }| j                  |||||||||	|
¬«
      }|| j                  |d   «      z   }|f|dd  z   }|S )N)	rŒ   r;  r<  r>  r=  r?  r!  r@  r#  r   r#   )râ   r«  rÊ   )r¨   rš   r;  rh   r<  r>  r=  r?  r!  r@  r#  r•  r–  Úlayer_outputrN  s                  rA   rµ   z!LongT5LayerCrossAttention.forwarda  s{   € ð  $Ÿ™¨}Ó=ÐØ×/Ñ/Ø ØØ-Ø'Ø+Ø)ØØ%Ø/Ø)ð 0ó 
Ðð % t§|¡|Ð4DÀQÑ4GÓ'HÑHˆØ�/Ð$4°Q°RÐ$8Ñ8ˆØˆrC   rÏ   )NNNNFNFNr—  rº   s   @rA   r©  r©  Z  s2   ø„ ñ7¨(°3©-õ 7ð ØØØØØØØ÷rC   r©  c                   óL   ‡ — e Zd Zddee   fˆ fd„Z	 	 	 	 	 	 	 	 	 	 	 	 dd„Zˆ xZS )ÚLongT5Blockrç   c                 ó  •— t         ‰| �  «        |j                  | _        |j                  rt        }nE|j                  dk(  rt
        }n/|j                  dk(  rt        }nt        d|j                  › d�«      ‚t        j                  «       | _
        | j                  j                   ||||¬«      «       | j                  r&| j                  j                  t        ||¬«      «       | j                  j                  t        |«      «       y )NÚlocalztransient-globalzjFor encoder attention mechanism, either `local` or `transient-global` attention type is expected, but got ú.r’  )rç   )r£   r¤   ré   r�  Úencoder_attention_typer™  r¤  Ú
ValueErrorr   Ú
ModuleListÚlayerrS   r©  rÜ   )r¨   r¾   rê   rç   Úattention_layerr«   s        €rA   r¤   zLongT5Block.__init__�  sÞ   ø€ Ü‰ÑÔØ ×+Ñ+ˆŒØ×ÒÜ6‰OØ×*Ñ*¨gÒ5Ü;‰OØ×*Ñ*Ð.@Ò@ÜE‰OäðØ!×8Ñ8Ð9¸ð<óð ô —]‘]“_ˆŒ
Ø�
‰
×ÑÙ˜FÐ@[ÐgpÔqô	
ð �?Š?Ø�J‰J×ÑÔ7¸È)ÔTÔUà�
‰
×Ñœ-¨Ó/Õ0rC   c                 ó^  —  | j                   d   |||||	|
||¬«      }|d d \  }}	|dd  }|j                  t        j                  k(  rht        j                  |«      j                  «       rEt        j                  |j                  «      j                  dz
  }t        j                  || |¬«      }| j                  xr |d u}|rº | j                   d   ||||||	|d   dz   |
||¬«
      }|d d \  }}	|j                  t        j                  k(  rht        j                  |«      j                  «       rEt        j                  |j                  «      j                  dz
  }t        j                  || |¬«      }||dd  z   } | j                   d   |«      }|j                  t        j                  k(  rht        j                  |«      j                  «       rEt        j                  |j                  «      j                  dz
  }t        j                  || |¬«      }|f}|
r||	fz   |z   }|S ||z   }|S )	Nr   )rh   r<  r>  r=  r?  r@  r#  rj   iè  )r  r„   r#   r0   )	r;  rh   r<  r>  r=  r!  r?  r@  r#  )
r¶  r.   r:   r²   ÚisinfÚanyÚfinfor„   Úclampré   )r¨   rš   rh   r<  Úencoder_hidden_statesÚencoder_attention_maskÚencoder_decoder_position_biasr>  Úcross_attn_layer_head_maskr=  r?  r@  Úreturn_dictr#  Úself_attention_outputsÚattention_outputsÚclamp_valueÚdo_cross_attentionÚcross_attention_outputsrN  s                       rA   rµ   zLongT5Block.forward˜  s.  € ð  "/ §¡¨A¡ØØ)Ø'Ø+Ø)ØØ/Ø)ô	"
Ðð )?¸rÀÐ(BÑ%ˆ�~Ø2°1°2Ð6Ðð ×Ñ¤%§-¡-Ò/´E·K±KÀÓ4N×4RÑ4RÔ4TÜŸ+™+ m×&9Ñ&9Ó:×>Ñ>ÀÑEˆKÜ!ŸK™K¨¸K¸<È[ÔYˆMà!Ÿ_™_ÒRÐ1FÈdÐ1RÐÙØ&3 d§j¡j°¡mØØ!6Ø5Ø;Ø :Ø-Ø+¨BÑ/°!Ñ3Ø#Ø"3Ø-ô'Ð#ð -DÀBÀQÐ,GÑ)ˆM˜>ð ×"Ñ"¤e§m¡mÒ3¼¿¹ÀMÓ8R×8VÑ8VÔ8XÜ#Ÿk™k¨-×*=Ñ*=Ó>×BÑBÀTÑI�Ü %§¡¨MÀ¸|ÐQ\Ô ]�ð !2Ð4KÈAÈBÐ4OÑ OÐð '˜Ÿ
™
 2™ }Ó5ˆð ×Ñ¤%§-¡-Ò/´E·K±KÀÓ4N×4RÑ4RÔ4TÜŸ+™+ m×&9Ñ&9Ó:×>Ñ>ÀÑEˆKÜ!ŸK™K¨¸K¸<È[ÔYˆMà Ð"ˆáØ Ð 1Ñ1Ð4EÑEˆGð ˆð Ð 1Ñ1ˆGàˆrC   rO  )NNNNNNNNFFTNr—  rº   s   @rA   r¯  r¯  €  s@   ø„ ñ1ÈXÐVYÉ]õ 1ð4 ØØ"Ø#Ø&*ØØ#'ØØØØØ÷IrC   r¯  c                   óF   — e Zd ZdZeZdZdZdgZdZ	dZ
ed„ «       Zd„ Zd„ Zy	)
ÚLongT5PreTrainedModelz†
    An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained
    models.
    ÚtransformerTr¯  Fc                 óv   — t        j                  t        «      }t        j                  t        «      }|||dœ}|S )N)Údecoder_input_idsÚ	input_idsÚdecoder_attention_mask)r:   rƒ   r   r   )r¨   rÌ  Ú
input_maskÚdummy_inputss       rA   rÏ  z"LongT5PreTrainedModel.dummy_inputsñ  s8   € ô —L‘L¤Ó.ˆ	Ü—\‘\¤*Ó-ˆ
à!*Ø"Ø&0ñ
ˆð
 ÐrC   c                 óN  — | j                   j                  }t        |t        «      r)|j                  j
                  j                  |dz  «       yt        |t        t        t        f«      r�|j                  j                  j
                  j                  d|dz  ¬«       t        |d«      rL| j                   j                  s5|j                  j                  j
                  j                  d|dz  ¬«       yyyt        |t        «      �rM|j                   j                  j
                  j                  d|| j                   j"                  dz  z  ¬«       t        |j                   d«      rD|j                   j$                  �.|j                   j$                  j
                  j'                  «        |j(                  j                  j
                  j                  d|| j                   j*                  dz  z  ¬«       t        |j(                  d«      rF|j(                  j$                  �/|j(                  j$                  j
                  j'                  «        yyyt        |t,        «      �rò|j.                  j                  j
                  j                  d|| j                   j"                  dz  z  ¬«       t        |j.                  d«      rD|j.                  j$                  �.|j.                  j$                  j
                  j'                  «        |j0                  j                  j
                  j                  d|| j                   j"                  dz  z  ¬«       t        |j0                  d«      rD|j0                  j$                  �.|j0                  j$                  j
                  j'                  «        |j(                  j                  j
                  j                  d|| j                   j*                  dz  z  ¬«       t        |j(                  d«      rF|j(                  j$                  �/|j(                  j$                  j
                  j'                  «        yyyt        |t2        t4        t6        f«      �r±| j                   j"                  }| j                   j8                  }| j                   j:                  }|j<                  j                  j
                  j                  d|||z  dz  z  ¬«       |j>                  j                  j
                  j                  d||dz  z  ¬«       |j@                  j                  j
                  j                  d||dz  z  ¬«       |jB                  j                  j
                  j                  d|||z  dz  z  ¬«       |jD                  r€|jF                  j                  j
                  j                  d||dz  z  ¬«       t        |t6        «      r8|jH                  j                  j
                  j                  d||dz  z  ¬«       yyyy)zInitialize the weightsr~   r}   )r°   ÚstdÚlm_headç      à¿rÂ   N)%r¾   Úinitializer_factorrÐ   r¡   r¦   ÚdataÚfill_ÚLongT5ModelÚLongT5ForConditionalGenerationÚLongT5EncoderModelÚsharedÚnormal_ÚhasattrÚtie_word_embeddingsrÒ  r½   rÆ   rÄ   rÂ   Úzero_rÇ   rÅ   rÔ   rÖ   r×   ræ   rU  rx  rí   rï   rô   rõ   rö   r÷   rê   rù   rz  )r¨   ÚmoduleÚfactorrÄ   rî   rð   s         rA   Ú_init_weightsz#LongT5PreTrainedModel._init_weightsý  s“  € à—‘×/Ñ/ˆÜ�fœoÔ.Ø�M‰M×Ñ×$Ñ$ V¨c¡\Õ2Ü˜¤Ô.LÔN`Ð aÔbð �M‰M× Ñ ×%Ñ%×-Ñ-°3¸FÀS¹LÐ-ÔIÜ�v˜yÔ)°$·+±+×2QÒ2QØ—‘×%Ñ%×*Ñ*×2Ñ2¸ÀÈ#ÁÐ2ÕNð 3RÐ)ä˜Ô 3Õ4ð �I‰I×Ñ×!Ñ!×)Ñ)¨s¸À4Ç;Á;×CVÑCVÐ[_ÑB_Ñ8`Ð)ÔaÜ�v—y‘y &Ô)¨f¯i©i¯n©nÐ.HØ—	‘	—‘×#Ñ#×)Ñ)Ô+Ø�I‰I×Ñ×!Ñ!×)Ñ)¨s¸À4Ç;Á;×CSÑCSÐX\ÑB\Ñ8]Ð)Ô^Ü�v—y‘y &Ô)¨f¯i©i¯n©nÐ.HØ—	‘	—‘×#Ñ#×)Ñ)Õ+ð /IÐ)ä˜Ô 8Õ9Ø�K‰K×Ñ×#Ñ#×+Ñ+°¸&ÀTÇ[Á[×EXÑEXÐ]aÑDaÑ:bÐ+ÔcÜ�v—{‘{ FÔ+°·±×0@Ñ0@Ð0LØ—‘× Ñ ×%Ñ%×+Ñ+Ô-Ø�K‰K×Ñ×#Ñ#×+Ñ+°¸&ÀTÇ[Á[×EXÑEXÐ]aÑDaÑ:bÐ+ÔcÜ�v—{‘{ FÔ+°·±×0@Ñ0@Ð0LØ—‘× Ñ ×%Ñ%×+Ñ+Ô-Ø�I‰I×Ñ×!Ñ!×)Ñ)¨s¸À4Ç;Á;×CSÑCSÐX\ÑB\Ñ8]Ð)Ô^Ü�v—y‘y &Ô)¨f¯i©i¯n©nÐ.HØ—	‘	—‘×#Ñ#×)Ñ)Õ+ð /IÐ)ä˜¤Ô2FÔHfÐ gÕhð —k‘k×)Ñ)ˆGØ!%§¡×!1Ñ!1ÐØ—k‘k×+Ñ+ˆGØ�H‰H�O‰O× Ñ ×(Ñ(¨c°vÀ'ÐL^ÑB^ÐcgÑAgÑ7hÐ(ÔiØ�H‰H�O‰O× Ñ ×(Ñ(¨c°vÀÈ$ÁÑ7OÐ(ÔPØ�H‰H�O‰O× Ñ ×(Ñ(¨c°vÀÈ$ÁÑ7OÐ(ÔPØ�H‰H�O‰O× Ñ ×(Ñ(¨c°vÀ'ÐL^ÑB^ÐcgÑAgÑ7hÐ(ÔiØ×1Ò1Ø×.Ñ.×5Ñ5×:Ñ:×BÑBÈÐQWÐ\cÐhlÑ[lÑQmÐBÔnÜ˜fÔ&DÔEØ×9Ñ9×@Ñ@×EÑE×MÑMØ  f°¸TÑ0AÑ&Bð Nõ ð Fð 2ð irC   c                 óÜ  — | j                   j                  }| j                   j                  }|€t        d«      ‚t	        |«      rGt        j                  |j                  d d dz   |«      }t        j                  ||dd d…f   gd¬«      }n>|j                  |j                  «      }|dd d…f   j                  «       |ddd …f<   ||d<   |€t        d«      ‚|j                  |d	k(  |«       |S )
Nz’self.model.config.decoder_start_token_id has to be defined. In LongT5 it is usually set to the pad_token_id. See LongT5 docs for more information.r0   )r#   .rO   r#   ).r   z1self.model.config.pad_token_id has to be defined.éœÿÿÿ)r¾   Údecoder_start_token_idÚpad_token_idr´  r   r:   Úfullr7   rT   Ú	new_zerosÚcloneÚmasked_fill_)r¨   rÌ  rä  rå  Úshifted_input_idss        rA   Ú_shift_rightz"LongT5PreTrainedModel._shift_right.  sÿ   € Ø!%§¡×!CÑ!CÐØ—{‘{×/Ñ/ˆà!Ð)Üð8óð ô ˜YÔ'ä %§
¡
¨9¯?©?¸3¸BÐ+?À$Ñ+FÐH^Ó _ÐÜ %§	¡	Ð+<¸iÈÈSÈbÈSÈÑ>QÐ*RÐXZÔ [Ñà )× 3Ñ 3°I·O±OÓ DÐØ)2°3¸¸¸°8Ñ)<×)BÑ)BÓ)DÐ˜c 1¡2˜gÑ&Ø(>Ð˜fÑ%àÐÜÐPÓQÐQà×&Ñ&Ð'8¸DÑ'@À,ÔOà Ð rC   N)r¶   r·   r¸   r¢  r$   Úconfig_classÚbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_supports_cache_classÚ_supports_static_cacheÚpropertyrÏ  rá  rë  r1   rC   rA   rÈ  rÈ  ä  sK   „ ñð
  €LØ%ÐØ&*Ð#Ø&˜ÐØ ÐØ"Ðàñó ðò.ób!rC   rÈ  c                   ó(  ‡ — e Zd Zdˆ fd„	Zd„ Zd„ Z	 	 	 	 	 	 	 	 	 	 	 	 	 dd„Z	 ddej                  dej                  dej                  de	d	e
f
d
„Zedej                  dededej                  dej                  dej                  defd„«       Zˆ xZS )ÚLongT5Stackc                 óš  •— t         ‰| �  |«       t        j                  |j                  |j
                  «      | _        |�|j                  | j                  _        |j                  | _        |j                  | _	        | j                  dz   | _
        t        j                  t        |j                  «      D �cg c]  }t        |t        |dk(  «      |¬«      ‘Œ c}«      | _        t#        |j
                  |j$                  ¬«      | _        t        j(                  |j*                  «      | _        d| _        | j1                  «        y c c}w )Nr#   r   r’  rÞ   F)r£   r¤   r   rø   Ú
vocab_sizerÄ   Úembed_tokensr¦   ré   rW  r(   rµ  rP   Ú
num_layersr¯  r7  Úblockr¡   rá   Úfinal_layer_normrÈ   rÉ   rÊ   rü   Ú	post_init)r¨   r¾   r÷  rV   r«   s       €rA   r¤   zLongT5Stack.__init__K  s  ø€ Ü‰Ñ˜Ô äŸL™L¨×):Ñ):¸F¿N¹NÓKˆÔØÐ#Ø'3×':Ñ':ˆD×ÑÔ$Ø ×+Ñ+ˆŒà"×/Ñ/ˆÔØ×*Ñ*¨QÑ.ˆŒä—]‘]ô ˜v×0Ñ0Ó1öàô ˜FÄÀQÈ!ÁVÃÐXYÖZòó
ˆŒ
ô !0°·±ÀF×D]ÑD]Ô ^ˆÔÜ—z‘z &×"5Ñ"5Ó6ˆŒà&+ˆÔ#ð 	�‰Õùòs   Â9!Ec                 ó   — | j                   S rÏ   ©r÷  ©r¨   s    rA   Úget_input_embeddingsz LongT5Stack.get_input_embeddingse  s   € Ø× Ñ Ð rC   c                 ó   — || _         y rÏ   rý  ©r¨   Únew_embeddingss     rA   Úset_input_embeddingsz LongT5Stack.set_input_embeddingsi  s
   € Ø*ˆÕrC   c                 ó
  — |	�|	n| j                   j                  }	|
�|
n| j                   j                  }
|�|n| j                   j                  }|�|n| j                   j                  }|�$|�"| j
                  rdnd}t        d|› d|› d�«      ‚|�&|j                  «       }|j                  d|d   «      }n8|�|j                  «       d d }n"| j
                  rdnd}t        d|› d|› d	�«      ‚| j                  r%| j                  r|	rt        j                  d
«       d}	|€$| j                  €J d«       ‚| j                  |«      }|\  }}d}d}| j
                  r—|	s|�“t        |t        «      r't        |t         «      sd}t!        |t#        «       «      }njt        |t         «      s-d}t        j                  d«       t!        j$                  |«      }n-|€+t!        t#        «       t#        «       «      }n| j
                  sd }|�|j'                  «       nd}|€%t)        j*                  |||z   |j,                  ¬«      }|€1t/        «       s'||z   }t)        j0                  |||j,                  ¬«      }| j
                  r$| j3                  ||||�|j4                  nd |
«      }n=| j                   j6                  dk(  r"t9        || j:                  |j,                  «      }n|}| j
                  rO|�M|j                  «       \  }}}||f}|€!t)        j0                  ||j,                  ¬«      }| j=                  |«      }nd }| j?                  || j                   j@                  «      }| j?                  || j                   j@                  «      }|rdnd }|
rdnd }|
r| j
                  rdnd }d }d } | jC                  |«      }!tE        | jF                  «      D ]Å  \  }"}#||"   }$||"   }%|r||!fz   }| j                  r5| j                  r)| jI                  |#jJ                  |!||||| |$|%d |	|
||«      }&n |#|!||||| |$|%||	|
||¬«      }&|	du r|&d d dz   |&dd  z   }&|&d d \  }!}'|&d   }| j
                  r|�	|&|
rdnd   } |
sŒ§||&d   fz   }| j
                  sŒ½||&d   fz   }ŒÇ | jM                  |!«      }!| jC                  |!«      }!|r||!fz   }|	r'nd }(|r|j4                  }(|r|jO                  «       }(|stQ        d„ |!|(|||fD «       «      S tS        |!|(|||¬«      S )NÚdecoder_Ú zYou cannot specify both zinput_ids and zinputs_embeds at the same timer0   zYou have to specify either zinput_ids or Úinputs_embedszZ`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...Fz<You have to initialize the model with valid token embeddingsTzìPassing a tuple of `past_key_values` is deprecated and will be removed in Transformers v4.48.0. You should pass an instance of `EncoderDecoderCache` instead, e.g. `past_key_values=EncoderDecoderCache.from_legacy_cache(past_key_values)`.r   r{   r±  r1   )rh   r<  r½  r¾  r¿  r>  rÀ  r=  r?  r@  rÁ  r#  r#   rÏ   rj   é   r
   é   c              3   ó$   K  — | ]  }|�|–— Œ
 y ­wrÏ   r1   )Ú.0rö   s     rA   ú	<genexpr>z&LongT5Stack.forward.<locals>.<genexpr>-  s   è ø€ ò 
àð �=ô ñ
ùs   ‚)Úlast_hidden_stateÚpast_key_valuesrš   Ú
attentionsÚcross_attentions)*r¾   r?  r@  Úoutput_hidden_statesÚuse_return_dictré   r´  Úsizer-  rü   r,  rò   ró   r÷  rÐ   r   r   r   Úfrom_legacy_cacheÚget_seq_lengthr:   rZ   rF   r    rˆ   Ú_update_causal_maskr1  r³  ro   r(   Úinvert_attention_maskÚget_head_maskrø  rÊ   Ú	enumeraterù  Ú_gradient_checkpointing_funcrµ   rú  Úto_legacy_cacherR   r   ))r¨   rÌ  rh   r½  r¾  r  Ú	head_maskÚcross_attn_head_maskr  r?  r@  r  rÁ  r#  Úerr_msg_prefixÚinput_shaperŠ   rA  Úreturn_legacy_cacheÚreturn_self_attention_cacheÚpast_key_values_lengthÚmask_seq_lengthrJ  Úencoder_batch_sizeÚencoder_sequence_lengthÚ_Úencoder_hidden_shapeÚencoder_extended_attention_maskÚall_hidden_statesÚall_attentionsÚall_cross_attentionsr<  r¿  rš   rV   Úlayer_moduler>  rÀ  Úlayer_outputsÚnext_decoder_cacheÚ
next_caches)                                            rA   rµ   zLongT5Stack.forwardl  sÌ  € ð  "+Ð!6‘I¸D¿K¹K×<QÑ<Qˆ	Ø1BÐ1NÑ-ÐTX×T_ÑT_×TqÑTqÐà$8Ð$DÑ È$Ï+É+×JjÑJjð 	ð &1Ð%<‘kÀ$Ç+Á+×B]ÑB]ˆàÐ  ]Ð%>Ø+/¯?ª?™ZÀˆNÜØ*¨>Ð*:¸.ÈÐHXÐXvÐwóð ð Ð"Ø#Ÿ.™.Ó*ˆKØ!Ÿ™ r¨;°r©?Ó;‰IØÐ&Ø'×,Ñ,Ó.¨s°Ð3‰Kà+/¯?ª?™ZÀˆNÜÐ:¸>Ð:JÈ-ÐXfÐWgÐgtÐuÓvÐvà×&Ò&¨4¯=ª=ÙÜ×#Ñ#Øpôð "�	àÐ Ø×$Ñ$Ð0ÐpÐ2pÓpÐ0Ø ×-Ñ-¨iÓ8ˆMà!,Ñˆ
�Jð $ÐØ&+Ð#Ø�?Š?¡	¨_Ð-HÜ˜/¬5Ô1¼*À_ÔViÔ:jØ.2Ð+Ü"5°oÄ|Ã~Ó"V‘Ü Ô1DÔEØ&*Ð#Ü×#Ñ#ð`ôô
 #6×"GÑ"GÈÓ"X‘Ø Ð(Ü"5´l³nÄlÃnÓ"U‘Ø—’ð #ˆOàETÐE` ×!?Ñ!?Ô!AÐfgÐØÐ!Ü"Ÿ\™\Ø&Ð(>ÀÑ(KÐTa×ThÑThôˆNð Ð!Ô*BÔ*Dà4°zÑAˆOÜ"ŸZ™Z¨
°OÈM×L`ÑL`ÔaˆNà�?Š?Ø×2Ñ2ØØØØ8GÐ8S�×4Ò4ÐY]Ø!ó‰Kð �[‰[×/Ñ/°7Ò:Ü3°NÀDÇNÁNÐTa×ThÑThÓi‰Kà(ˆKð �?Š?Ð4Ð@Ø=R×=WÑ=WÓ=YÑ:ÐÐ 7¸Ø$6Ð8OÐ#PÐ Ø%Ð-Ü).¯©Ð4HÐQ^×QeÑQeÔ)fÐ&Ø.2×.HÑ.HÐI_Ó.`Ñ+à.2Ð+ð ×&Ñ& y°$·+±+×2HÑ2HÓIˆ	Ø#×1Ñ1Ð2FÈÏÉ×H^ÑH^Ó_ÐÙ"6™B¸DÐÙ0™°dˆÙ&7¸D¿OºO™rÐRVÐØˆØ(,Ð%àŸ™ ]Ó3ˆä(¨¯©Ó4ò :	V‰OˆAˆ|Ø'¨™lˆOØ)=¸aÑ)@Ð&á#Ø$5¸Ð8HÑ$HÐ!à×*Ò*¨t¯}ª}Ø $× AÑ AØ ×(Ñ(Ø!ØØ!Ø)Ø3Ø1Ø#Ø.ØØØ%ØØ"ó!‘ñ" !-Ø!Ø#.Ø"/Ø*?Ø+JØ2OØ$3Ø/IØ#2Ø'Ø&7Ø +Ø#1ô!�ð$ ˜EÑ!Ø -¨b¨qÐ 1°GÑ ;¸mÈAÈBÐ>OÑ O�à0=¸b¸qÐ0AÑ-ˆMÐ-ð
 *¨!Ñ,ˆMØ�ŠÐ#8Ð#DØ0=ÑCT¹aÐZ[Ñ0\Ð-â Ø!/°=ÀÑ3CÐ2EÑ!E�Ø—?“?Ø+?À=ÐQRÑCSÐBUÑ+UÑ(ðu:	Vðx ×-Ñ-¨mÓ<ˆØŸ™ ]Ó3ˆñ  Ø 1°]Ð4DÑ DÐá+4Ñ'¸$ˆ
Ù&Ø(×=Ñ=ˆJÙØ(×8Ñ8Ó:ˆJáÜñ 
ð "ØØ%Ø"Ø(ðô
ó 
ð 
ô 9Ø+Ø&Ø+Ø%Ø1ô
ð 	
rC   rh   Úinput_tensorr#  r  r@  c           
      óÆ  — | j                   j                  dk(  r|�|dk(  j                  «       r|S y | j                   j                  dk(  r7t        |t        j
                  «      rt        |«      }t        |t        «      r|S |�|j                  «       nd}t        |t        «      }| j                   j                  dk(  r(|s&|s$t        j                  |||| j                  ¬«      ry |j                  |j                  }	}|j                  d   }
|r|j!                  «       }n1t        |t        j
                  «      r|j                  d   n||
z   dz   }| j#                  ||
|||	||j                  d   ¬	«      }| j                   j                  dk(  rQ|�O|j                  j$                  d
v r7|s5t	        j&                  |«      j(                  }t        j*                  ||«      }|S )NÚflash_attention_2r}   Úflex_attentionr   Úsdpa)r  r"  Úis_trainingr#   r0   )Úsequence_lengthÚtarget_lengthr.   rF   r#  rŠ   )ÚcudaÚxpu)r¾   Ú_attn_implementationrº  rÐ   r:   r   r&   r%   r  r   r   Ú_ignore_causal_mask_sdpar,  r.   rF   r7   Úget_max_cache_shapeÚ5_prepare_4d_causal_attention_mask_with_cache_positionrt   r»  r  Ú_unmask_unattended)r¨   rh   r0  r#  r  r@  Úpast_seen_tokensÚusing_static_cacher.   rF   r6  r7  rJ  Ú	min_dtypes                 rA   r  zLongT5Stack._update_causal_maskA  sÖ  € ð �;‰;×+Ñ+Ð/BÒBØÐ)¨~ÀÑ/D×.IÑ.IÔ.KØ%Ð%ØØ�;‰;×+Ñ+Ð/?Ò?Ü˜.¬%¯,©,Ô7Ü!<¸^Ó!L�Ü˜.¬)Ô4Ø%Ð%ð
 @OÐ?Z˜?×9Ñ9Ô;Ð`aÐÜ'¨¼ÓEÐð �;‰;×+Ñ+¨vÒ5Ñ>PÑYjÜ%×>Ñ>ØØ*Ø'7Ø ŸM™Mõ	ð à$×*Ñ*¨L×,?Ñ,?ˆvˆØ&×,Ñ,¨QÑ/ˆÙØ+×?Ñ?ÓA‰Mô ˜n¬e¯l©lÔ;ð ×$Ñ$ RÒ(à%¨Ñ7¸!Ñ;ð ð ×PÑPØØ+Ø'ØØØ)Ø#×)Ñ)¨!Ñ,ð Qó 
ˆð �K‰K×,Ñ,°Ò6ØÐ*Ø×%Ñ%×*Ñ*¨oÑ=Ù%ô
 Ÿ™ EÓ*×.Ñ.ˆIÜ0×CÑCÀKÐQZÓ[ˆKàÐrC   r6  r7  r.   rF   rŠ   c                 ó˜  — | �| j                  «       dk(  r| }|S t        j                  |«      j                  }	t        j                  ||f|	||¬«      }|dk7  rt        j
                  |d¬«      }|t        j                  ||¬«      |j                  dd«      kD  z  }|dddd…dd…f   j                  |ddd«      }| �Œ|j                  «       }| j                  d   }
|dd…dd…dd…d|
…f   | dd…dddd…f   j                  |j                  «      z   }|dk(  }|dd…dd…dd…d|
…f   j                  ||	«      |dd…dd…dd…d|
…f<   |S )	a°  
        Creates a causal 4D mask of shape `(batch_size, 1, query_length, key_value_length)` from a 2D mask of shape
        `(batch_size, key_value_length)`, or if the input `attention_mask` is already 4D, do nothing.

        Args:
            attention_mask (`torch.Tensor`):
                A 2D attention mask of shape `(batch_size, key_value_length)` or a 4D attention mask of shape
                `(batch_size, 1, query_length, key_value_length)`.
            sequence_length (`int`):
                The sequence length being processed.
            target_length (`int`):
                The target length: when generating with static cache, the mask should be as long as the static cache,
                to account for the 0 padding, the part of the cache that is not filled yet.
            dtype (`torch.dtype`):
                The dtype to use for the 4D attention mask.
            device (`torch.device`):
                The device to place the 4D attention mask on.
            cache_position (`torch.Tensor`):
                Indices depicting the position of the input sequence tokens in the sequence.
            batch_size (`torch.Tensor`):
                Batch size.
        Nr  )Ú
fill_valuer.   rF   r#   )Údiagonalr{   r0   r   )r)   r:   r»  r  ræ  ÚtriurZ   rH   Úexpandrè  r7   rd   rF   Úmasked_fill)rh   r6  r7  r.   rF   r#  rŠ   r�  rJ  rA  Úmask_lengthÚpadding_masks               rA   r=  zALongT5Stack._prepare_4d_causal_attention_mask_with_cache_position‡  sy  € ðD Ð%¨.×*<Ñ*<Ó*>À!Ò*Cà(ˆKð* Ðô' Ÿ™ EÓ*×.Ñ.ˆIÜŸ*™*Ø  -Ð0¸YÈeÐ\bôˆKð  !Ò#Ü#Ÿj™j¨¸qÔA�Øœ5Ÿ<™<¨¸fÔEÈ×H^ÑH^Ð_aÐcdÓHeÑeÑeˆKØ% d¨D²!²QÐ&6Ñ7×>Ñ>¸zÈ1ÈbÐRTÓUˆKØÐ)Ø)×/Ñ/Ó1�Ø,×2Ñ2°2Ñ6�Ø*ª1ªa²°L°[°LÐ+@ÑAÀNÒSTÐVZÐ\`ÒbcÐScÑDd×DgÑDgØ×&Ñ&óEñ  �ð  ,¨qÑ0�Ø5@ÂÂAÂqÈ,È;È,ÐAVÑ5W×5cÑ5cØ  )ó6�šAšq¢! \ k \Ð1Ñ2ð ÐrC   rÏ   )NNNNNNNNNNNNNru  )r¶   r·   r¸   r¤   rÿ  r  rµ   r:   r   r   r7  r  rS  r‰   r.   rF   r=  r¹   rº   s   @rA   rô  rô  J  sõ   ø„ õò4!ò+ð
 ØØ"Ø#ØØØ!ØØØØ!ØØóR
ðv #(ñDàŸ™ðDð —l‘lðDð Ÿ™ð	Dð
 ðDð  óDðL ð7ØŸ™ð7àð7ð ð7ð �{‰{ð	7ð
 —‘ð7ð Ÿ™ð7ð ò7ó ô7rC   rô  aQ  

    The LongT5 model was proposed in [LongT5: Efficient Text-To-Text Transformer for Long
    Sequences](https://arxiv.org/abs/2112.07916) by Mandy Guo, Joshua Ainslie, David Uthus, Santiago Ontanon, Jianmo
    Ni, Yun-Hsuan Sung and Yinfei Yang. It's an encoder-decoder transformer pre-trained in a text-to-text denoising
    generative setting. LongT5 model is an extension of T5 model, and it enables using one of the two different
    efficient attention mechanisms - (1) Local attention, or (2) Transient-Global attention.

    This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
    library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
    etc.)

    This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
    Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
    and behavior.

    Parameters:
        config ([`LongT5Config`]): Model configuration class with all the parameters of the model.
            Initializing with a config file does not load the weights associated with the model, only the
            configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
a©  
    Args:
        input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
            Indices of input sequence tokens in the vocabulary. LongT5 is a model with relative position embeddings so
            you should be able to pad the inputs on both the right and the left.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for detail.

            [What are input IDs?](../glossary#input-ids)

            To know more on how to prepare `input_ids` for pretraining take a look a [LONGT5
            Training](./longt5#training).
        attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:

            - 1 for tokens that are **not masked**,
            - 0 for tokens that are **masked**.

            [What are attention masks?](../glossary#attention-mask)
        decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
            Indices of decoder input sequence tokens in the vocabulary.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are decoder input IDs?](../glossary#decoder-input-ids)

            LONGT5 uses the `pad_token_id` as the starting token for `decoder_input_ids` generation. If
            `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see
            `past_key_values`).

            To know more on how to prepare `decoder_input_ids` for pretraining take a look at [LONGT5
            Training](./longt5#training).
        decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
            Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also
            be used by default.
        head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*):
            Mask to nullify selected heads of the self-attention modules in the encoder. Mask values selected in `[0,
            1]`:

            - 1 indicates the head is **not masked**,
            - 0 indicates the head is **masked**.

        decoder_head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*):
            Mask to nullify selected heads of the self-attention modules in the decoder. Mask values selected in `[0,
            1]`:

            - 1 indicates the head is **not masked**,
            - 0 indicates the head is **masked**.

        cross_attn_head_mask (`torch.Tensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*):
                Mask to nullify selected heads of the cross-attention modules in the decoder. Mask values selected in
                `[0, 1]`:

                - 1 indicates the head is **not masked**,
                - 0 indicates the head is **masked**.

        encoder_outputs (`tuple(tuple(torch.FloatTensor)`, *optional*):
            Tuple consists of (`last_hidden_state`, `optional`: *hidden_states*, `optional`: *attentions*)
            `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)` is a sequence of hidden states at
            the output of the last layer of the encoder. Used in the cross-attention of the decoder.
        past_key_values (`tuple(tuple(torch.FloatTensor))` of length `config.n_layers` with each tuple having 4 tensors of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`):
            Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding.

            If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that
            don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all
            `decoder_input_ids` of shape `(batch_size, sequence_length)`.
        inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
            Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
            is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
            model's internal embedding lookup matrix.
        decoder_inputs_embeds (`torch.FloatTensor` of shape `(batch_size, target_sequence_length, hidden_size)`, *optional*):
            Optionally, instead of passing `decoder_input_ids` you can choose to directly pass an embedded
            representation. If `past_key_values` is used, optionally only the last `decoder_inputs_embeds` have to be
            input (see `past_key_values`). This is useful if you want more control over how to convert
            `decoder_input_ids` indices into associated vectors than the model's internal embedding lookup matrix.

            If `decoder_input_ids` and `decoder_inputs_embeds` are both unset, `decoder_inputs_embeds` takes the value
            of `inputs_embeds`.

        use_cache (`bool`, *optional*):
            If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see
            `past_key_values`).

        output_attentions (`bool`, *optional*):
            Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
            tensors for more detail.
        output_hidden_states (`bool`, *optional*):
            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
            more detail.
        return_dict (`bool`, *optional*):
            Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
        cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*):
            Indices depicting the position of the input sequence tokens in the sequence. It is used to update the
            cache in the correct position and to infer the complete sequence length.
a¤  
    Args:
        input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
            Indices of input sequence tokens in the vocabulary. LongT5 is a model with relative position embeddings so
            you should be able to pad the inputs on both the right and the left.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for detail.

            To know more on how to prepare `input_ids` for pretraining take a look a [LONGT5
            Training](./longt5#training).
        attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:

            - 1 for tokens that are **not masked**,
            - 0 for tokens that are **masked**.

            [What are attention masks?](../glossary#attention-mask)
        head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*):
            Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`:

            - 1 indicates the head is **not masked**,
            - 0 indicates the head is **masked**.

        inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
            Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
            is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
            model's internal embedding lookup matrix.
        output_attentions (`bool`, *optional*):
            Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
            tensors for more detail.
        output_hidden_states (`bool`, *optional*):
            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
            more detail.
        return_dict (`bool`, *optional*):
            Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
a_  
The input argument `head_mask` was split into two arguments `head_mask` and `decoder_head_mask`. Currently,
`decoder_head_mask` is set to copy `head_mask`, but this feature is deprecated and will be removed in future versions.
If you do not want to use any `decoder_head_mask` now, please set `decoder_head_mask = torch.ones(num_layers,
num_heads)`.
z`The bare LONGT5 Model transformer outputting raw hidden-states without any specific head on top.c            '       ó€  ‡ — e Zd ZdgZddgZdefˆ fd„Zd„ Zd„ Zd„ Z	d	„ Z
d
„ Zd„ Z ee«       eee¬«      	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 ddeej(                     deej*                     deej(                     deej,                     deej*                     deej*                     deej.                     deeeej*                           deeeej*                           deej.                     deej.                     dee   dee   dee   dee   deej(                     deeej*                     ef   f"d„«       «       Zˆ xZS ) r×  úFdecoder.block.0.layer.1.EncDecAttention.relative_attention_bias.weightúencoder.embed_tokens.weightúdecoder.embed_tokens.weightr¾   c                 óÊ  •— t         ‰| �  |«       t        j                  |j                  |j
                  «      | _        t        j                  |«      }d|_	        d|_
        d|_        t        || j                  «      | _        t        j                  |«      }d|_	        d|_        |j                  |_        t        || j                  «      | _        | j#                  «        y )NFT)r£   r¤   r   rø   rö  rÄ   rÚ  ÚcopyÚdeepcopyré   r?  Úis_encoder_decoderrô  ÚencoderÚnum_decoder_layersrø  Údecoderrû  ©r¨   r¾   Úencoder_configÚdecoder_configr«   s       €rA   r¤   zLongT5Model.__init__t  s®   ø€ Ü‰Ñ˜Ô Ü—l‘l 6×#4Ñ#4°f·n±nÓEˆŒäŸ™ vÓ.ˆØ$)ˆÔ!Ø#(ˆÔ Ø,1ˆÔ)Ü" >°4·;±;Ó?ˆŒäŸ™ vÓ.ˆØ$(ˆÔ!Ø,1ˆÔ)Ø$*×$=Ñ$=ˆÔ!Ü" >°4·;±;Ó?ˆŒð 	�‰ÕrC   c                 ó   — | j                   S rÏ   ©rÚ  rþ  s    rA   rÿ  z LongT5Model.get_input_embeddings‡  ó   € Ø�{‰{ÐrC   c                 ó~   — || _         | j                  j                  |«       | j                  j                  |«       y rÏ   ©rÚ  rR  r  rT  r  s     rA   r  z LongT5Model.set_input_embeddingsŠ  ó-   € Ø$ˆŒØ�‰×)Ñ)¨.Ô9Ø�‰×)Ñ)¨.Õ9rC   c                 óò   — | j                   j                  ra| j                  | j                  j                  | j
                  «       | j                  | j                  j                  | j
                  «       y y rÏ   ©r¾   rÝ  Ú_tie_or_clone_weightsrR  r÷  rÚ  rT  rþ  s    rA   Ú_tie_weightszLongT5Model._tie_weights�  óP   € Ø�;‰;×*Ò*Ø×&Ñ& t§|¡|×'@Ñ'@À$Ç+Á+ÔNØ×&Ñ& t§|¡|×'@Ñ'@À$Ç+Á+ÕNð +rC   c                 ó   — | j                   S rÏ   ©rR  rþ  s    rA   Úget_encoderzLongT5Model.get_encoder”  ó   € Ø�|‰|ÐrC   c                 ó   — | j                   S rÏ   ©rT  rþ  s    rA   Úget_decoderzLongT5Model.get_decoder—  rf  rC   c                 ó˜   — |j                  «       D ]7  \  }}| j                  j                  |   j                  j	                  |«       Œ9 y©z�
        Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base
        class PreTrainedModel
        N©ÚitemsrR  r¶  Ú	attentionr  ©r¨   Úheads_to_pruner¶  r  s       rA   Ú_prune_headszLongT5Model._prune_headsš  óE   € ð
 +×0Ñ0Ó2ò 	C‰LˆE�5Ø�L‰L×Ñ˜uÑ%×/Ñ/×;Ñ;¸EÕBñ	CrC   ©Úoutput_typerì  rÌ  rh   rË  rÍ  r  Údecoder_head_maskr  Úencoder_outputsr  r  Údecoder_inputs_embedsr?  r@  r  rÁ  r#  r+   c                 óî  — |�|n| j                   j                  }|�|n| j                   j                  }|�O|€M| j                   j                  | j                   j                  k(  r t        j                  t        t        «       |}|€| j                  |||
||||¬«      }nI|rGt        |t        «      s7t        |d   t        |«      dkD  r|d   ndt        |«      dkD  r|d   nd¬«      }|d   }| j                  ||||	|||||||||¬«      }|s||z   S t        |j                  |j                   |j"                  |j$                  |j&                  |j                  |j"                  |j$                  ¬«      S )	a%  
        Returns:

        Example:

        ```python
        >>> from transformers import AutoTokenizer, LongT5Model

        >>> tokenizer = AutoTokenizer.from_pretrained("google/long-t5-local-base")
        >>> model = LongT5Model.from_pretrained("google/long-t5-local-base")

        >>> # Let's try a very long encoder input.
        >>> input_ids = tokenizer(
        ...     100 * "Studies have been shown that owning a dog is good for you", return_tensors="pt"
        ... ).input_ids  # Batch size 1

        >>> decoder_input_ids = tokenizer("Studies show that", return_tensors="pt").input_ids  # Batch size 1

        >>> # forward pass
        >>> outputs = model(input_ids=input_ids, decoder_input_ids=decoder_input_ids)
        >>> last_hidden_states = outputs.last_hidden_state
        ```N©rÌ  rh   r  r  r@  r  rÁ  r   r#   rj   ©r  rš   r  ©rÌ  rh   r  r  r½  r¾  r  r  r?  r@  r  rÁ  r#  )r  r  Údecoder_hidden_statesÚdecoder_attentionsr  Úencoder_last_hidden_stater½  Úencoder_attentions)r¾   r?  r  rø  rS  ÚwarningsÚwarnÚ#_LongT5Model__HEAD_MASK_WARNING_MSGÚFutureWarningrR  rÐ   r   r  rT  r   r  r  rš   r  r  )r¨   rÌ  rh   rË  rÍ  r  ru  r  rv  r  r  rw  r?  r@  r  rÁ  r#  rš   Údecoder_outputss                      rA   rµ   zLongT5Model.forward¢  sš  € ðV "+Ð!6‘I¸D¿K¹K×<QÑ<Qˆ	Ø%0Ð%<‘kÀ$Ç+Á+×B]ÑB]ˆð Ð Ð%6Ð%>Ø�{‰{×%Ñ%¨¯©×)GÑ)GÒGÜ—‘Ô5´}ÔEØ$-Ð!ð Ð"Ø"Ÿl™lØ#Ø-Ø+Ø#Ø"3Ø%9Ø'ð +ó ‰Oñ ¤¨O¼_Ô!MÜ-Ø"1°!Ñ"4Ü47¸Ó4HÈ1Ò4L˜o¨aÒ0ÐRVÜ14°_Ó1EÈÒ1I˜?¨1Ò-ÈtôˆOð (¨Ñ*ˆð Ÿ,™,Ø'Ø1Ø/Ø+Ø"/Ø#1Ø'Ø!5ØØ/Ø!5Ø#Ø)ð 'ó 
ˆñ  Ø" _Ñ4Ð4ä!Ø-×?Ñ?Ø+×;Ñ;Ø"1×"?Ñ"?Ø.×9Ñ9Ø,×=Ñ=Ø&5×&GÑ&GØ"1×"?Ñ"?Ø.×9Ñ9ô	
ð 		
rC   )NNNNNNNNNNNNNNNN)r¶   r·   r¸   Ú"_keys_to_ignore_on_load_unexpectedÚ_tied_weights_keysr$   r¤   rÿ  r  ra  re  ri  rq  r   ÚLONGT5_INPUTS_DOCSTRINGr"   r   Ú_CONFIG_FOR_DOCr   r:   Ú
LongTensorÚFloatTensorÚ
BoolTensorr   r   r7  r   rµ   r¹   rº   s   @rA   r×  r×  j  s  ø„ ð 	Rð*Ð&ð 8Ð9VÐWÐð˜|õ ò&ò:ò
Oò
òòCñ +Ð+BÓCÙÐ+=ÈOÔ\ð 15Ø6:Ø8<Ø=AØ15Ø9=Ø7;ØEIØEIØ04Ø8<Ø$(Ø,0Ø/3Ø&*Ø59ñ#c
à˜E×,Ñ,Ñ-ðc
ð ! ×!2Ñ!2Ñ3ðc
ð $ E×$4Ñ$4Ñ5ð	c
ð
 !)¨×)9Ñ)9Ñ :ðc
ð ˜E×-Ñ-Ñ.ðc
ð $ E×$5Ñ$5Ñ6ðc
ð ' u§|¡|Ñ4ðc
ð " %¨¨e×.?Ñ.?Ñ(@Ñ"AÑBðc
ð " %¨¨e×.?Ñ.?Ñ(@Ñ"AÑBðc
ð   §¡Ñ-ðc
ð  (¨¯©Ñ5ðc
ð ˜D‘>ðc
ð $ D™>ðc
ð ' t™nðc
ð  ˜d‘^ð!c
ð" ! ×!1Ñ!1Ñ2ð#c
ð$ 
ˆu�U×&Ñ&Ñ'Ð);Ð;Ñ	<ò%c
ó ]ó Dôc
rC   r×  z4LONGT5 Model with a `language modeling` head on top.c            )       óÌ  ‡ — e Zd ZdgZg d¢Zdefˆ fd„Zd„ Zd„ Zd„ Z	d„ Z
d	„ Zd
„ Zd„ Z ee«       eee¬«      	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d"deej*                     deej,                     deej*                     deej.                     deej,                     deej,                     deej0                     deeeej0                           deeeej0                           deej,                     deej,                     deej*                     dee   dee   dee   dee   deej*                     deeej,                     ef   f$d„«       «       Zdej0                  fd „Zd!„ Zˆ xZS )#rØ  rK  )rL  rM  zlm_head.weightr¾   c                 óN  •— t         ‰| �  |«       |j                  | _        t	        j
                  |j                  |j                  «      | _        t        j                  |«      }d|_
        d|_        d|_        t        || j                  «      | _        t        j                  |«      }d|_
        d|_        |j                  |_        t        || j                  «      | _        t	        j$                  |j                  |j                  d¬«      | _        | j)                  «        y )NFTrÁ   )r£   r¤   rÄ   Ú	model_dimr   rø   rö  rÚ  rO  rP  ré   r?  rQ  rô  rR  rS  rø  rT  rÃ   rÒ  rû  rU  s       €rA   r¤   z'LongT5ForConditionalGeneration.__init__  s×   ø€ Ü‰Ñ˜Ô ØŸ™ˆŒä—l‘l 6×#4Ñ#4°f·n±nÓEˆŒäŸ™ vÓ.ˆØ$)ˆÔ!Ø#(ˆÔ Ø,1ˆÔ)Ü" >°4·;±;Ó?ˆŒäŸ™ vÓ.ˆØ$(ˆÔ!Ø,1ˆÔ)Ø$*×$=Ñ$=ˆÔ!Ü" >°4·;±;Ó?ˆŒä—y‘y §¡°×1BÑ1BÈÔOˆŒð 	�‰ÕrC   c                 ó   — | j                   S rÏ   rY  rþ  s    rA   rÿ  z3LongT5ForConditionalGeneration.get_input_embeddings(  rZ  rC   c                 ó~   — || _         | j                  j                  |«       | j                  j                  |«       y rÏ   r\  r  s     rA   r  z3LongT5ForConditionalGeneration.set_input_embeddings+  r]  rC   c                 óò   — | j                   j                  ra| j                  | j                  j                  | j
                  «       | j                  | j                  j                  | j
                  «       y y rÏ   r_  rþ  s    rA   ra  z+LongT5ForConditionalGeneration._tie_weights0  rb  rC   c                 ó   — || _         y rÏ   ©rÒ  r  s     rA   Úset_output_embeddingsz4LongT5ForConditionalGeneration.set_output_embeddings5  s	   € Ø%ˆ�rC   c                 ó   — | j                   S rÏ   r“  rþ  s    rA   Úget_output_embeddingsz4LongT5ForConditionalGeneration.get_output_embeddings8  rf  rC   c                 ó   — | j                   S rÏ   rd  rþ  s    rA   re  z*LongT5ForConditionalGeneration.get_encoder;  rf  rC   c                 ó   — | j                   S rÏ   rh  rþ  s    rA   ri  z*LongT5ForConditionalGeneration.get_decoder>  rf  rC   rs  rÌ  rh   rË  rÍ  r  ru  r  rv  r  r  rw  Úlabelsr?  r@  r  rÁ  r#  r+   c                 ól  — |�|n| j                   j                  }|�|n| j                   j                  }|�O|€M| j                   j                  | j                   j                  k(  r t        j                  t        t        «       |}|€| j                  |||
||||¬«      }nI|rGt        |t        «      s7t        |d   t        |«      dkD  r|d   ndt        |«      dkD  r|d   nd¬«      }|d   }|�|€|€| j                  |«      }| j                  ||||	|||||||||¬«      }|d   }| j                   j                  r|| j                   dz  z  }| j#                  |«      }d}|�^t%        d	¬
«      }|j'                  |j(                  «      } ||j+                  d|j-                  d«      «      |j+                  d«      «      }|s|f|dd z   |z   }|�|f|z   S |S t/        |||j0                  |j2                  |j4                  |j6                  |j8                  |j2                  |j4                  ¬«	      S )a¨  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[-100, 0, ...,
            config.vocab_size - 1]`. All labels set to `-100` are ignored (masked), the loss is only computed for
            labels in `[0, ..., config.vocab_size]`

        Returns:

        Examples:

        ```python
        >>> from transformers import AutoTokenizer, LongT5ForConditionalGeneration

        >>> tokenizer = AutoTokenizer.from_pretrained("Stancld/longt5-tglobal-large-16384-pubmed-3k_steps")
        >>> model = LongT5ForConditionalGeneration.from_pretrained(
        ...     "Stancld/longt5-tglobal-large-16384-pubmed-3k_steps"
        ... )

        >>> # Let's try a very long input.
        >>> inputs = tokenizer(100 * "studies have shown that owning a dog is good for you ", return_tensors="pt")
        >>> input_ids = inputs.input_ids

        >>> outputs = model.generate(input_ids)
        >>> print(tokenizer.decode(outputs[0], skip_special_tokens=True))
        abstractthe aim of this article is to provide an overview of the literature on the role of dog
        ```Nry  r   r#   rj   rz  r{  rÓ  rã  )Úignore_indexr0   )	ÚlossÚlogitsr  r|  r}  r  r~  r½  r  )r¾   r?  r  rø  rS  r€  r�  Ú6_LongT5ForConditionalGeneration__HEAD_MASK_WARNING_MSGrƒ  rR  rÐ   r   r  rë  rT  rÝ  rŽ  rÒ  r	   rd   rF   r-  r  r   r  rš   r  r  r  )r¨   rÌ  rh   rË  rÍ  r  ru  r  rv  r  r  rw  r™  r?  r@  r  rÁ  r#  rš   r„  Úsequence_outputÚ	lm_logitsrœ  Úloss_fctÚoutputs                            rA   rµ   z&LongT5ForConditionalGeneration.forwardA  sl  € ð` "+Ð!6‘I¸D¿K¹K×<QÑ<Qˆ	Ø%0Ð%<‘kÀ$Ç+Á+×B]ÑB]ˆð Ð Ð%6Ð%>Ø�{‰{×%Ñ%¨¯©×)GÑ)GÒGÜ—‘Ô5´}ÔEØ$-Ð!ð Ð"à"Ÿl™lØ#Ø-Ø+Ø#Ø"3Ø%9Ø'ð +ó ‰Oñ ¤¨O¼_Ô!MÜ-Ø"1°!Ñ"4Ü47¸Ó4HÈ1Ò4L˜o¨aÒ0ÐRVÜ14°_Ó1EÈÒ1I˜?¨1Ò-ÈtôˆOð (¨Ñ*ˆàÐÐ"3Ð";Ð@UÐ@]à $× 1Ñ 1°&Ó 9Ðð Ÿ,™,Ø'Ø1Ø/Ø+Ø"/Ø#1Ø'Ø!5ØØ/Ø!5Ø#Ø)ð 'ó 
ˆð  *¨!Ñ,ˆà�;‰;×*Ò*ð .°·±ÀÑ1EÑFˆOà—L‘L Ó1ˆ	àˆØÐÜ'°TÔ:ˆHà—Y‘Y˜y×/Ñ/Ó0ˆFÙ˜IŸN™N¨2¨y¯~©~¸bÓ/AÓBÀFÇKÁKÐPRÃOÓTˆDñ Ø�\ O°A°BÐ$7Ñ7¸/ÑIˆFØ)-Ð)9�T�G˜fÑ$ÐE¸vÐEäØØØ+×;Ñ;Ø"1×"?Ñ"?Ø.×9Ñ9Ø,×=Ñ=Ø&5×&GÑ&GØ"1×"?Ñ"?Ø.×9Ñ9ô

ð 
	
rC   c                 ó$   — | j                  |«      S rÏ   )rë  )r¨   r™  s     rA   Ú%prepare_decoder_input_ids_from_labelszDLongT5ForConditionalGeneration.prepare_decoder_input_ids_from_labelsÅ  s   € Ø× Ñ  Ó(Ð(rC   c           	      ó:  — |€t         j                  d«       |S d}|D ]z  }d}|D ]1  }||j                  d|j                  |j                  «      «      fz   }Œ3 |d   j
                  |d   j
                  k(  sJ ‚t        |«      t        |«      k(  sJ ‚||fz   }Œ| |S )NzHYou might want to consider setting `use_cache=True` to speed up decodingr1   r   )rò   ÚwarningÚindex_selectrd   rF   r7   r  )r¨   r  Úbeam_idxÚreordered_decoder_pastÚlayer_past_statesÚreordered_layer_past_statesÚlayer_past_states          rA   Ú_reorder_cachez-LongT5ForConditionalGeneration._reorder_cacheÈ  sÖ   € ð Ð"Ü�N‰NÐeÔfØ"Ð"à!#ÐØ!0ò 	]Ðð +-Ð'Ø$5ò Ð à.IØ$×1Ñ1°!°X·[±[ÐAQ×AXÑAXÓ5YÓZðMñ /Ñ+ðð /¨qÑ1×7Ñ7Ð;LÈQÑ;O×;UÑ;UÒUÐUÐUÜÐ2Ó3´sÐ;LÓ7MÒMÐMÐMà%;Ð?ZÐ>\Ñ%\Ñ"ð	]ð &Ð%rC   )NNNNNNNNNNNNNNNNN) r¶   r·   r¸   r…  r†  r$   r¤   rÿ  r  ra  r”  r–  re  ri  r   r‡  r"   r   rˆ  r   r:   r‰  rŠ  r‹  r   r   r7  r   rµ   r¤  r­  r¹   rº   s   @rA   rØ  rØ  
  sH  ø„ ð 	Rð*Ð&ò jÐð˜|õ ò.ò:ò
Oò
&òòòñ +Ð+BÓCÙ¨?ÈÔYð 15Ø6:Ø8<Ø=AØ15Ø9=Ø7;Ø@DØ@DØ59Ø=AØ-1Ø$(Ø,0Ø/3Ø&*Ø59ñ%@
à˜E×,Ñ,Ñ-ð@
ð ! ×!2Ñ!2Ñ3ð@
ð $ E×$4Ñ$4Ñ5ð	@
ð
 !)¨×)9Ñ)9Ñ :ð@
ð ˜E×-Ñ-Ñ.ð@
ð $ E×$5Ñ$5Ñ6ð@
ð ' u§|¡|Ñ4ð@
ð " %¨¨e¯l©lÑ(;Ñ"<Ñ=ð@
ð " %¨¨e¯l©lÑ(;Ñ"<Ñ=ð@
ð   × 1Ñ 1Ñ2ð@
ð  (¨×(9Ñ(9Ñ:ð@
ð ˜×)Ñ)Ñ*ð@
ð ˜D‘>ð@
ð $ D™>ð@
ð  ' t™nð!@
ð" ˜d‘^ð#@
ð$ ! ×!1Ñ!1Ñ2ð%@
ð& 
ˆu�U×&Ñ&Ñ'¨Ð8Ñ	9ò'@
ó Zó Dð@
ðD)¸E¿L¹Ló )ö&rC   rØ  zjThe bare LONGT5 Model transformer outputting encoder's raw hidden-states without any specific head on top.c                   óT  ‡ — e Zd ZdgZdgZdefˆ fd„Zd„ Zd„ Zd„ Z	d„ Z
d	„ Z ee«       eee¬
«      	 	 	 	 	 	 	 ddeej&                     deej(                     deej(                     deej(                     dee   dee   dee   deeej(                     ef   fd„«       «       Zˆ xZS )rÙ  rL  rT  r¾   c                 ó  •— t         ‰| �  |«       t        j                  |j                  |j
                  «      | _        t        j                  |«      }d|_	        d|_
        t        || j                  «      | _        | j                  «        y )NF)r£   r¤   r   rø   rö  rÄ   rÚ  rO  rP  r?  rQ  rô  rR  rû  )r¨   r¾   rV  r«   s      €rA   r¤   zLongT5EncoderModel.__init__é  sh   ø€ Ü‰Ñ˜Ô Ü—l‘l 6×#4Ñ#4°f·n±nÓEˆŒäŸ™ vÓ.ˆØ#(ˆÔ Ø,1ˆÔ)Ü" >°4·;±;Ó?ˆŒð 	�‰ÕrC   c                 ó   — | j                   S rÏ   rY  rþ  s    rA   rÿ  z'LongT5EncoderModel.get_input_embeddingsõ  rZ  rC   c                 óH   — || _         | j                  j                  |«       y rÏ   )rÚ  rR  r  r  s     rA   r  z'LongT5EncoderModel.set_input_embeddingsø  s   € Ø$ˆŒØ�‰×)Ñ)¨.Õ9rC   c                 ó’   — | j                   j                  r1| j                  | j                  j                  | j
                  «       y y rÏ   )r¾   rÝ  r`  rR  r÷  rÚ  rþ  s    rA   ra  zLongT5EncoderModel._tie_weightsü  s2   € Ø�;‰;×*Ò*Ø×&Ñ& t§|¡|×'@Ñ'@À$Ç+Á+ÕNð +rC   c                 ó   — | j                   S rÏ   rd  rþ  s    rA   re  zLongT5EncoderModel.get_encoder 	  rf  rC   c                 ó˜   — |j                  «       D ]7  \  }}| j                  j                  |   j                  j	                  |«       Œ9 yrk  rl  ro  s       rA   rq  zLongT5EncoderModel._prune_heads	  rr  rC   rs  rÌ  rh   r  r  r@  r  rÁ  r+   c           	      ój   — |�|n| j                   j                  }| j                  |||||||¬«      }|S )a\  
        Returns:

        Example:

        ```python
        >>> from transformers import AutoTokenizer, LongT5ForConditionalGeneration

        >>> tokenizer = AutoTokenizer.from_pretrained("google/long-t5-local-base")
        >>> model = LongT5EncoderModel.from_pretrained("google/long-t5-local-base")
        >>> input_ids = tokenizer(
        ...     100 * "Studies have been shown that owning a dog is good for you ", return_tensors="pt"
        ... ).input_ids  # Batch size 1
        >>> outputs = model(input_ids=input_ids)
        >>> last_hidden_states = outputs.last_hidden_state
        ```ry  )r¾   r  rR  )	r¨   rÌ  rh   r  r  r@  r  rÁ  rv  s	            rA   rµ   zLongT5EncoderModel.forward	  sJ   € ð8 &1Ð%<‘kÀ$Ç+Á+×B]ÑB]ˆàŸ,™,ØØ)Ø'ØØ/Ø!5Ø#ð 'ó 
ˆð ÐrC   )NNNNNNN)r¶   r·   r¸   r†  r…  r$   r¤   rÿ  r  ra  re  rq  r   ÚLONGT5_ENCODER_INPUTS_DOCSTRINGr"   r   rˆ  r   r:   r‰  rŠ  r7  r   r   rµ   r¹   rº   s   @rA   rÙ  rÙ  á  s  ø„ ð
 8Ð8ÐØ*4¨Ð&ð
˜|õ 
òò:òOòòCñ +Ð+JÓKÙ¨?ÈÔYð 15Ø6:Ø15Ø59Ø,0Ø/3Ø&*ñ&à˜E×,Ñ,Ñ-ð&ð ! ×!2Ñ!2Ñ3ð&ð ˜E×-Ñ-Ñ.ð	&ð
   × 1Ñ 1Ñ2ð&ð $ D™>ð&ð ' t™nð&ð ˜d‘^ð&ð 
ˆu�U×&Ñ&Ñ'¨Ð8Ñ	9ò&ó Zó Lô&rC   rÙ  )rÙ  rØ  r×  rÈ  )r   )cr¢  rO  r  r€  Útypingr   r   r   r   r   r:   r   Útorch.nnr	   Úactivationsr   Úcache_utilsr   r   r   r   Ú
generationr   Úmodeling_attn_mask_utilsr   Úmodeling_outputsr   r   r   r   Úmodeling_utilsr   Úpytorch_utilsr   r   r   Úutilsr   r   r   r   r   r   r    r!   r"   Úconfiguration_longt5r$   Ú!torch.nn.attention.flex_attentionr%   Úintegrations.flex_attentionr&   Ú
get_loggerr¶   rò   rˆ  Ú_CHECKPOINT_FOR_DOCr   r‰   rB   rK   rX   r`   rg   rF   ro   r’   r™   rŸ   ÚModuler¡   Úapex.normalizationr»   ÚinfoÚImportErrorÚ	Exceptionr¦  rS   r½   rÔ   rÜ   ræ   rU  rx  r�  r™  r¤  r©  r¯  rÈ  rô  ÚLONGT5_START_DOCSTRINGr‡  r¶  Ú__HEAD_MASK_WARNING_MSGr×  rØ  rÙ  Ú__all__r1   rC   rA   ú<module>rÎ     s  ðñ ã Û Û ß 4Õ 4ã Ý Ý %å !ß PÓ PÝ )Ý >÷ó õ .ß gÑ g÷
÷ 
õ 
õ /ñ  Ô!Ý;åJð 
ˆ×	Ñ	˜HÓ	%€à €Ø1Ð ñ
˜Ÿ™ð °ð ¸3ð È3ð ÐW\×WcÑWcó ð #˜%Ÿ,™,ð #°3ð #¸Sð #ÀUÇ\Á\ó #ñ4˜UŸ\™\ð 4°cð 4Èð 4ÐY\ð 4Ðej×eqÑeqó 4ð2!°#ð !¸%¿,¹,ó !ðB°U·\±\ð BÈcð BÐV[×VbÑVbó Bð8¨e¯l©lð 8Àsð 8ÐTY×T`ÑT`ð 8Ðej×eqÑeqó 8ð .PØ—L‘Lð.PØ58ð.Pà
ˆ5�<‰<˜Ÿ™Ð%Ñ&ó.Pðb4°U·\±\ð 4ÐVYð 4Ð^c×^jÑ^jó 4ð	jØ—<‘<ð	jØ,1¯L©Lð	jØJMð	jà
‡\�\ó	jô+�b—i‘iô +ð2	Ý/à"€Oà
‡K�KÐeÔfð Ð × Ñ ˜OÔ ,ô˜"Ÿ)™)ô ô,˜rŸy™yô ô&�B—I‘Iô ô&a�b—i‘iô aôH}˜2Ÿ9™9ô }ô@D R§Y¡Yô DôP!˜rŸy™yô !ôH B§I¡Iô ô>¨b¯i©iô ôD# §	¡	ô #ôLa�"—)‘)ô aôHc!˜Oô c!ôLvÐ'ô vðrÐ ð,`Ð ðD$#Ð ðNÐ ñ ØfØóôY
Ð'ó Y
ó	ðY
ñx ÐPÐRhÓiôS&Ð%:¸Oó S&ó jðS&ñl ØpØóôNÐ.ó Nó	ðNòb k�øðiA ò 	âØò 	Ø
‡N�NÐ[Ô\Úð	ús   Ç7M2 Í2NÍ:NÎN