Ë
    S^(h� ã                   ó´  — d Z ddlZddlmZ ddlmZmZmZmZ ddl	Z
ddlZddlmZ ddlmZmZmZ ddlmZ dd	lmZmZmZmZmZmZ dd
lmZ ddlmZmZmZm Z m!Z!m"Z" ddl#m$Z$  e!jJ                  e&«      Z'dZ(dZ)dZ*d„ Z+ G d„ dejX                  «      Z- G d„ dejX                  «      Z.dej^                  de0de0dej^                  fd„Z1 G d„ dejX                  «      Z2 G d„ dejX                  «      Z3 G d„ d ejX                  «      Z4 G d!„ d"ejX                  «      Z5	 dNd#ej^                  d$e0d%e0d&e6d'e6dej^                  fd(„Z7 G d)„ d*ejX                  «      Z8 G d+„ d,ejX                  «      Z9 G d-„ d.e«      Z: G d/„ d0ejX                  «      Z;e G d1„ d2e«      «       Z<d3Z=d4Z> ed5e=«       G d6„ d7e:«      «       Z? ed8e=«       G d9„ d:e:«      «       Z@ ed;e=«        G d<„ d=e:«      ZA ed>e=«       G d?„ d@e:«      «       ZB edAe=«       G dB„ dCe:«      «       ZC edDe=«       G dE„ dFe:«      «       ZD edGe=«       G dH„ dIe:«      «       ZE edJe=«       G dK„ dLe:«      «       ZFg dM¢ZGy)Oz!PyTorch Funnel Transformer model.é    N)Ú	dataclass)ÚListÚOptionalÚTupleÚUnion)Únn)ÚBCEWithLogitsLossÚCrossEntropyLossÚMSELossé   )ÚACT2FN)ÚBaseModelOutputÚMaskedLMOutputÚMultipleChoiceModelOutputÚQuestionAnsweringModelOutputÚSequenceClassifierOutputÚTokenClassifierOutput)ÚPreTrainedModel)ÚModelOutputÚadd_code_sample_docstringsÚadd_start_docstringsÚ%add_start_docstrings_to_model_forwardÚloggingÚreplace_return_docstringsé   )ÚFunnelConfigr   zfunnel-transformer/smallg    €„.Ac                 ó|  — 	 ddl }ddl}ddl}t        j                  j                  |«      }t        j                  d|› �«       |j                  j                  |«      }g }g }	|D ]^  \  }
}t        j                  d|
› d|› �«       |j                  j                  ||
«      }|j                  |
«       |	j                  |«       Œ` ddd	d
dddddddddddœ}t        ||	«      D �]÷  \  }
}|
j                  d«      }
t!        d„ |
D «       «      r(t        j                  ddj#                  |
«      › �«       ŒR|
d   dk(  rŒ[| }d}|
dd D �]  }t%        |t&        «      s½|j)                  d|«      r«t+        |j-                  d|«      j/                  «       d   «      }||j0                  k  rQd}||j2                  |   k\  r*||j2                  |   z  }|dz  }||j2                  |   k\  rŒ*|j4                  |   |   }Œ²||j0                  z  }|j6                  |   }ŒÑ|dk(  rt%        |t8        «      r|j:                  } n%||v rt=        |||   «      }�Œ		 t=        ||«      }�Œ |r�Œ�tE        |jB                  «      tE        |jB                  «      k7  r|jG                  |jB                  «      }dk(  r |jH                  |«      }tK        jL                  |«      |_'        �Œú | S # t        $ r t        j                  d«       ‚ w xY w# t>        $ r. tA        ddj#                  |
«      › �|jB                  «       d}Y  ŒÙw xY w)z'Load tf checkpoints in a pytorch model.r   Nz™Loading a TensorFlow model in PyTorch, requires TensorFlow to be installed. Please see https://www.tensorflow.org/install/ for installation instructions.z&Converting TensorFlow checkpoint from zLoading TF weight z with shape Úk_headÚq_headÚv_headÚ	post_projÚlinear_1Úlinear_2Ú	attentionÚffnÚweightÚbiasÚword_embeddingsÚ
embeddings)ÚkÚqÚvÚoÚlayer_1Úlayer_2Úrel_attnÚffÚkernelÚgammaÚbetaÚlookup_tableÚword_embeddingÚinputú/c              3   ó$   K  — | ]  }|d v –— Œ
 y­w))Úadam_vÚadam_mÚAdamWeightDecayOptimizerÚAdamWeightDecayOptimizer_1Úglobal_stepN© )Ú.0Úns     úh/var/www/skyplay_api_hub/venv/lib/python3.12/site-packages/transformers/models/funnel/modeling_funnel.pyú	<genexpr>z,load_tf_weights_in_funnel.<locals>.<genexpr>f   s   è ø€ ò 
àð ÐnÔnñ
ùs   ‚z	Skipping Ú	generatorFr   z	layer_\d+zlayer_(\d+)ÚrTr2   )(ÚreÚnumpyÚ
tensorflowÚImportErrorÚloggerÚerrorÚosÚpathÚabspathÚinfoÚtrainÚlist_variablesÚload_variableÚappendÚzipÚsplitÚanyÚjoinÚ
isinstanceÚFunnelPositionwiseFFNÚ	fullmatchÚintÚsearchÚgroupsÚnum_hidden_layersÚblock_sizesÚblocksÚlayersÚFunnelRelMultiheadAttentionÚr_kernelÚgetattrÚAttributeErrorÚprintÚshapeÚlenÚreshapeÚ	transposeÚtorchÚ
from_numpyÚdata)ÚmodelÚconfigÚtf_checkpoint_pathrF   ÚnpÚtfÚtf_pathÚ	init_varsÚnamesÚarraysÚnamerg   ÚarrayÚ
_layer_mapÚpointerÚskippedÚm_nameÚlayer_indexÚ	block_idxs                      rB   Úload_tf_weights_in_funnelr   8   sI  € ð
ÛãÛô �g‰g�o‰oÐ0Ó1€GÜ
‡K�KÐ8¸¸	ÐBÔCà—‘×'Ñ'¨Ó0€IØ€EØ€FØ ò ‰ˆˆeÜ�‰Ð(¨¨¨l¸5¸'ÐBÔCØ—‘×&Ñ& w°Ó5ˆØ�‰�TÔØ�‰�eÕð	ð ØØØØØØØØØØØ Ø+Øñ€Jô" ˜5 &Ó)ó +3‰ˆˆeØ�z‰z˜#‹ˆô ñ 
àô
ô 
ô �K‰K˜) C§H¡H¨T£NÐ#3Ð4Ô5ØØ�‰7�kÒ!ØØˆØˆØ˜1˜2�hó 	ˆFÜ˜gÔ'<Ô=À"Ç,Á,È|Ð]cÔBdÜ! "§)¡)¨N¸FÓ"C×"JÑ"JÓ"LÈQÑ"OÓP�Ø ×!9Ñ!9Ò9Ø !�IØ%¨×);Ñ);¸IÑ)FÒFØ# v×'9Ñ'9¸)Ñ'DÑD˜Ø! Q™˜	ð &¨×);Ñ);¸IÑ)FÓFð &Ÿn™n¨YÑ7¸ÑD‘Gà 6×#;Ñ#;Ñ;�KØ%Ÿn™n¨[Ñ9‘GØ˜3’¤:¨gÔ7RÔ#SØ!×*Ñ*�ÙØ˜:Ñ%Ü! '¨:°fÑ+=Ó>’ðÜ% g¨vÓ6’Gð'	ó0 Ü�7—=‘=Ó!¤S¨¯©Ó%5Ò5ØŸ™ g§m¡mÓ4�Ø˜Ò!Ø$˜Ÿ™ UÓ+�Ü ×+Ñ+¨EÓ2ˆGŽLðW+3ðZ €Løôa ò Ü�‰ðQô	
ð 	ðûôJ &ò Ü˜I c§h¡h¨t£nÐ%5Ð6¸¿¹ÔDØ"�GÚðús   ‚K! ÉLË! LÌ3L;Ì:L;c                   óˆ   ‡ — e Zd Zdeddfˆ fd„Z	 ddeej                     deej                     dej                  fd„Zˆ xZ	S )	ÚFunnelEmbeddingsro   ÚreturnNc                 ó@  •— t         ‰| �  «        t        j                  |j                  |j
                  |j                  ¬«      | _        t        j                  |j                  |j                  ¬«      | _        t        j                  |j                  «      | _        y )N)Úpadding_idx©Úeps)ÚsuperÚ__init__r   Ú	EmbeddingÚ
vocab_sizeÚhidden_sizeÚpad_token_idr(   Ú	LayerNormÚd_modelÚlayer_norm_epsÚ
layer_normÚDropoutÚhidden_dropoutÚdropout©Úselfro   Ú	__class__s     €rB   rˆ   zFunnelEmbeddings.__init__“   sh   ø€ Ü‰ÑÔÜ!Ÿ|™|¨F×,=Ñ,=¸v×?QÑ?QÐ_e×_rÑ_rÔsˆÔÜŸ,™, v§~¡~¸6×;PÑ;PÔQˆŒÜ—z‘z &×"7Ñ"7Ó8ˆ�ó    Ú	input_idsÚinputs_embedsc                 óp   — |€| j                  |«      }| j                  |«      }| j                  |«      }|S ©N)r(   r�   r“   )r•   r˜   r™   r)   s       rB   ÚforwardzFunnelEmbeddings.forward™   s<   € ð Ð Ø ×0Ñ0°Ó;ˆMØ—_‘_ ]Ó3ˆ
Ø—\‘\ *Ó-ˆ
ØÐr—   ©NN)
Ú__name__Ú
__module__Ú__qualname__r   rˆ   r   rk   ÚTensorrœ   Ú__classcell__©r–   s   @rB   r�   r�   ’   sS   ø„ ð9˜|ð 9°õ 9ð aeñØ! %§,¡,Ñ/ðØGOÐPU×P\ÑP\ÑG]ðà	�‰÷r—   r�   c                   óœ  ‡ — e Zd ZU dZdZeed<   deddfˆ fd„Z	 	 d de	j                  d	ee	j                     d
ee	j                     dee	j                     fd„Zd
e	j                  de	j                  fd„Zdede	j                  de	j                   deee	j                     eee	j                        f   fd„Zde	j                  defd„Zd!de	j                  dedede	j                  fd„Zdee	j                  ee	j                     ee	j                     f   deeee   ee   f   de	j                  fd„Z	 d"dee	j                  ee	j                     ee	j                     f   dedede	j                  fd„Zdee	j                     dee	j                  ee	j                     f   fd„Zdee	j                     dee	j                     fd„Zˆ xZS )#ÚFunnelAttentionStructurez>
    Contains helpers for `FunnelRelMultiheadAttention `.
    é   Úcls_token_type_idro   r‚   Nc                 óÎ   •— t         ‰| �  «        || _        t        j                  |j
                  «      | _        t        j                  |j
                  «      | _        d | _        y r›   )	r‡   rˆ   ro   r   r‘   r’   Úsin_dropoutÚcos_dropoutÚpooling_multr”   s     €rB   rˆ   z!FunnelAttentionStructure.__init__ª   sM   ø€ Ü‰ÑÔØˆŒÜŸ:™: f×&;Ñ&;Ó<ˆÔÜŸ:™: f×&;Ñ&;Ó<ˆÔð !ˆÕr—   r™   Úattention_maskÚtoken_type_idsc                 ób  — d| _         |j                  d«      x| _        }| j                  ||j                  |j
                  «      }|�| j                  |«      nd}| j                  j                  r7t        j                  j                  |j                  |dz
  |dz
  g«      d«      nd}||||fS )zCReturns the attention inputs associated to the inputs of the model.r   N)r   r   r   r   )r«   ÚsizeÚseq_lenÚget_position_embedsÚdtypeÚdeviceÚtoken_type_ids_to_matro   Úseparate_clsr   Ú
functionalÚpadÚnew_ones)r•   r™   r¬   r­   r°   Úposition_embedsÚtoken_type_matÚcls_masks           rB   Úinit_attention_inputsz.FunnelAttentionStructure.init_attention_inputs³   s³   € ð ˆÔØ!.×!3Ñ!3°AÓ!6Ð6ˆŒ�wØ×2Ñ2°7¸M×<OÑ<OÐQ^×QeÑQeÓfˆØGUÐGa˜×3Ñ3°NÔCÐgkˆð �{‰{×'Ò'ô �M‰M×Ñ˜m×4Ñ4°gÀ±kÀ7ÈQÁ;Ð5OÓPÐR^Ô_àð 	ð
   °ÀÐJÐJr—   c                 ó‚   — |dd…dd…df   |dd…df   k(  }|| j                   k(  }|dd…dd…df   |dd…df   z  }||z  S )z-Convert `token_type_ids` to `token_type_mat`.N)r§   )r•   r­   rº   Úcls_idsÚcls_mats        rB   r´   z.FunnelAttentionStructure.token_type_ids_to_matÇ   sY   € à'ªª1¨d¨
Ñ3°~ÂaÈÀgÑ7NÑNˆà  D×$:Ñ$:Ñ:ˆØš!šQ ˜*Ñ%¨²°4°Ñ(8Ñ8ˆØ˜Ñ'Ð'r—   r°   r²   r³   c                 óz  — | j                   j                  }| j                   j                  dk(  �rEt        j                  d|dt        j
                  |¬«      j                  |«      }t        j                  d|dz  dt        j
                  |¬«      j                  |«      }dd||dz  z  z  z  }|dd…df   |d   z  }t        j                  |«      }	| j                  |	«      }
t        j                  |«      }| j                  |«      }t        j                  |
|
gd	¬
«      }t        j                  ||	gd	¬
«      }t        j                  ||gd	¬
«      }t        j                  |	 |gd	¬
«      }||||fS t        j                  d|dz  dt        j
                  |¬«      j                  |«      }dd||dz  z  z  z  }t        j                  | dz  |dz  dt        j
                  |¬«      j                  |«      }|dz  }|dd…df   |d   z  }| j                  t        j                  |«      «      }	| j                  t        j                  |«      «      }t        j                  |	|gd	¬
«      }t        j                  d|t        j
                  |¬«      j                  |«      }|}g }t        d| j                   j                  «      D ]í  }|dk(  rd}ns| j                  ||«      }d|dz
  z  }| j                  |||d¬«      }|dd…df   |z   }|j!                  |j#                  d«      |«      }t        j$                  |d|«      }|}d|z  }| j                  ||«      }|dd…df   |z   }|j!                  |j#                  d«      |«      }t        j$                  |d|«      }|j'                  ||g«       Œï |S )a  
        Create and cache inputs related to relative position encoding. Those are very different depending on whether we
        are using the factorized or the relative shift attention:

        For the factorized attention, it returns the matrices (phi, pi, psi, omega) used in the paper, appendix A.2.2,
        final formula.

        For the relative shift attention, it returns all possible vectors R used in the paper, appendix A.2.1, final
        formula.

        Paper link: https://arxiv.org/abs/2006.03236
        Ú
factorizedr   ç      ð?©r²   r³   r¦   r   i'  Néÿÿÿÿ©Údim)Úshift)ro   rŽ   Úattention_typerk   ÚarangeÚint64ÚtoÚsinr©   Úcosrª   ÚcatÚrangeÚ
num_blocksÚstride_pool_posÚrelative_posÚexpandr¯   ÚgatherrS   )r•   r°   r²   r³   rŽ   Úpos_seqÚfreq_seqÚinv_freqÚsinusoidÚ	sin_embedÚsin_embed_dÚ	cos_embedÚcos_embed_dÚphiÚpsiÚpiÚomegaÚ
rel_pos_idÚzero_offsetÚ	pos_embedÚposÚ
pooled_posÚposition_embeds_listÚblock_indexÚposition_embeds_poolingÚstrideÚrel_posÚposition_embeds_no_poolings                               rB   r±   z,FunnelAttentionStructure.get_position_embedsÏ   s|  € ð —+‘+×%Ñ%ˆØ�;‰;×%Ñ%¨Ó5ô —l‘l 1 g¨s¼%¿+¹+ÈfÔU×XÑXÐY^Ó_ˆGÜ—|‘| A w°!¡|°SÄÇÁÐTZÔ[×^Ñ^Ð_dÓeˆHØ˜E h°'¸Q±,Ñ&?Ñ@ÑAˆHØšq $˜wÑ'¨(°4©.Ñ8ˆHÜŸ	™	 (Ó+ˆIØ×*Ñ*¨9Ó5ˆKÜŸ	™	 (Ó+ˆIØ×*Ñ*¨9Ó5ˆKä—)‘)˜[¨+Ð6¸BÔ?ˆCÜ—)‘)˜Y¨	Ð2¸Ô;ˆCÜ—‘˜K¨Ð5¸2Ô>ˆBÜ—I‘I 	˜z¨9Ð5¸2Ô>ˆEØ˜˜S %Ð(Ð(ô —|‘| A w°!¡|°SÄÇÁÐTZÔ[×^Ñ^Ð_dÓeˆHØ˜E h°'¸Q±,Ñ&?Ñ@ÑAˆHäŸ™ w h°¡l°G¸a±KÀÌEÏKÉKÐ`fÔg×jÑjÐkpÓqˆJØ! A™+ˆKØ!¢! T 'Ñ*¨X°d©^Ñ;ˆHØ×(Ñ(¬¯©°8Ó)<Ó=ˆIØ×(Ñ(¬¯©°8Ó)<Ó=ˆIÜŸ	™	 9¨iÐ"8¸bÔAˆIä—,‘,˜q '´·±ÀVÔL×OÑOÐPUÓVˆCØˆJØ#%Ð Ü$ Q¨¯©×(>Ñ(>Ó?ò c�ð  !Ò#Ø.2Ñ+à!%×!5Ñ!5°c¸;Ó!G�Jð  ;°¡?Ñ3�FØ"×/Ñ/°°V¸ZÈqÐ/ÓQ�GØ%¢a¨ gÑ.°Ñ<�GØ%Ÿn™n¨W¯\©\¸!«_¸gÓF�GÜ.3¯l©l¸9ÀaÈÓ.QÐ+ð !�Ø˜K™�Ø×+Ñ+¨C°Ó8�à!¢! T 'Ñ*¨[Ñ8�Ø!Ÿ.™.¨¯©°a«¸'ÓB�Ü-2¯\©\¸)ÀQÈÓ-PÐ*à$×+Ñ+Ð-GÐI`Ð,aÕbð9cð: (Ð'r—   Úpos_idrç   c                 óì   — | j                   j                  rW|j                  d|z   dz   g«      }| j                   j                  r|dd n|dd }t	        j
                  ||ddd…   gd«      S |ddd…   S )ze
        Pool `pos_id` while keeping the cls token separate (if `config.separate_cls=True`).
        r¦   r   rÄ   Nr   )ro   rµ   Ú
new_tensorÚtruncate_seqrk   rÎ   )r•   rì   rç   Úcls_posÚpooled_pos_ids        rB   rÑ   z(FunnelAttentionStructure.stride_pool_pos  s€   € ð �;‰;×#Ò#ð
 ×'Ñ'¨1¨k©>Ð):¸QÑ)>Ð(?Ó@ˆGØ,0¯K©K×,DÒ,D˜F 1 R™LÈ&ÐQRÐQSÈ*ˆMÜ—9‘9˜g }±S°q°SÑ'9Ð:¸AÓ>Ð>à™#˜A˜#‘;Ðr—   rä   ré   rÇ   c                 óÎ   — |€|}|d   |d   z
  }|t        |«      z  }|||z  z   }|d   |d   z
  }t        j                  ||dz
  | t        j                  |j                  ¬«      S )zV
        Build the relative positional vector between `pos` and `pooled_pos`.
        r   rÄ   r   rÃ   )rh   rk   rÉ   Úlongr³   )	r•   rä   ré   rå   rÇ   Ú	ref_pointÚ
num_removeÚmax_distÚmin_dists	            rB   rÒ   z%FunnelAttentionStructure.relative_pos.  sx   € ð ÐØˆJà˜q‘M C¨¡FÑ*ˆ	ØœS ›_Ñ,ˆ
Ø˜z¨FÑ2Ñ2ˆØ˜a‘= 3 r¡7Ñ*ˆä�|‰|˜H h°¡l°V°GÄ5Ç:Á:ÐVY×V`ÑV`ÔaÐar—   ÚtensorÚaxisc                 óH  ‡ ‡— |€yt        ‰t        t        f«      r‰D ]  }‰ j                  ||«      }Œ |S t        |t        t        f«      r t	        |«      ˆˆ fd„|D «       «      S ‰|j
                  z  Š‰ j                  j                  r#‰ j                  j                  rt        ddd«      nt        ddd«      }t        d«      g‰z  |gz   }‰ j                  j                  r9t        d«      g‰z  t        dd«      gz   }t        j                  ||   |g‰¬«      }||   S )zT
        Perform pooling by stride slicing the tensor along the given axis.
        Nc              3   óB   •K  — | ]  }‰j                  |‰«      –— Œ y ­wr›   )Ústride_pool)r@   Úxrù   r•   s     €€rB   rC   z7FunnelAttentionStructure.stride_pool.<locals>.<genexpr>O  s   øè ø€ ÒJ¸a × 0Ñ 0°°D× 9ÑJùs   ƒrÄ   r¦   r   )rù   )rX   ÚlistÚtuplerü   ÚtypeÚndimro   rµ   rï   Úslicerk   rÎ   )r•   rø   rù   ÚaxÚ
axis_sliceÚ	enc_sliceÚ	cls_slices   ` `    rB   rü   z$FunnelAttentionStructure.stride_pool<  s  ù€ ð ˆ>Øô �dœT¤5˜MÔ*Øò 6�Ø×)Ñ)¨&°"Ó5‘ð6àˆMô �fœu¤d˜mÔ,Ø”4˜“<ÔJÀ6ÔJÓJÐJð 	�—‘Ñˆð #'§+¡+×":Ò":¸t¿{¹{×?WÒ?WŒE�$˜˜AÔÔ]bÐcgÐimÐopÓ]qð 	ô ˜4“[�M DÑ(¨J¨<Ñ7ˆ	Ø�;‰;×#Ò#Ü˜t›˜¨Ñ,´°d¸A³Ð/?Ñ?ˆIÜ—Y‘Y  yÑ 1°6Ð:ÀÔFˆFØ�iÑ Ð r—   Úmodec                 óî  ‡ ‡‡‡— ‰€yt        ‰t        t        f«      r t        ‰«      ˆˆ ˆˆfd„‰D «       «      S ‰ j                  j
                  rE‰ j                  j                  r‰dd…dd…f   n‰}t        j                  ‰dd…dd…f   |gd¬«      Š‰j                  }|dk(  r‰dd…ddd…df   Šn|dk(  r‰dd…ddd…dd…f   Š‰dfŠ‰dk(  r$t        j                  j                  ‰‰‰d	¬
«      Šn_‰dk(  r$t        j                  j                  ‰‰‰d	¬
«      Šn6‰dk(  r&t        j                  j                  ‰ ‰‰d	¬
«       Šnt        d«      ‚|dk(  r‰dd…ddd…df   S |dk(  r	‰dd…df   S ‰S )z3Apply 1D pooling to a tensor of size [B x T (x H)].Nc              3   óF   •K  — | ]  }‰j                  ‰‰‰¬ «      –— Œ y­w))r  ré   N)Úpool_tensor)r@   rý   r  r•   ré   rø   s     €€€€rB   rC   z7FunnelAttentionStructure.pool_tensor.<locals>.<genexpr>f  s$   øè ø€ ÒcÐWX × 0Ñ 0°¸dÈ6Ð 0× RÑcùs   ƒ!rÄ   r   rÅ   r¦   r   ÚmeanT)ré   Ú	ceil_modeÚmaxÚminz0The supported modes are 'mean', 'max' and 'min'.r   )rX   rÿ   rþ   r   ro   rµ   rï   rk   rÎ   r  r   r¶   Ú
avg_pool2dÚ
max_pool2dÚNotImplementedError)r•   rø   r  ré   Úsuffixr  s   ````  rB   r
  z$FunnelAttentionStructure.pool_tensor]  s{  û€ ð ˆ>Øô �fœu¤d˜mÔ,Ø”4˜“<ÖcÐ\bÔcÓcÐcà�;‰;×#Ò#Ø'+§{¡{×'?Ò'?�VšA˜s ˜s˜F’^ÀVˆFÜ—Y‘Y ¢q¨"¨1¨" u¡¨vÐ6¸AÔ>ˆFà�{‰{ˆØ�1Š9ØšA˜t¢Q¨Ð,Ñ-‰FØ�QŠYØšA˜t¢Qª˜MÑ*ˆFà˜!�ˆà�6Š>Ü—]‘]×-Ñ-¨f°fÀVÐW[Ð-Ó\‰FØ�UŠ]Ü—]‘]×-Ñ-¨f°fÀVÐW[Ð-Ó\‰FØ�UŠ]Ü—m‘m×.Ñ.°¨w¸ÀvÐY]Ð.Ó^Ð^‰Fä%Ð&XÓYÐYà�1Š9Øš!˜Q¢ 1˜*Ñ%Ð%Ø�QŠYØš!˜Q˜$‘<ÐØˆr—   Úattention_inputsc                 ó”  — |\  }}}}| j                   j                  r€| j                   j                  dk(  r| j                  |dd d«      |dd z   }| j                  |d«      }| j                  |d«      }| j	                  || j                   j
                  ¬«      }n¢| xj                  dz  c_        | j                   j                  dk(  r| j                  |d«      }| j                  |ddg«      }| j                  |ddg«      }| j	                  |d¬«      }| j	                  || j                   j
                  ¬«      }||||f}||fS )zTPool `output` and the proper parts of `attention_inputs` before the attention layer.rÁ   Nr¦   r   r   ©r  r  )ro   Úpool_q_onlyrÈ   rü   r
  Úpooling_typer«   )r•   Úoutputr  r¹   rº   r¬   r»   s          rB   Úpre_attention_poolingz.FunnelAttentionStructure.pre_attention_poolingƒ  sM  € ð EUÑAˆ˜¨¸Ø�;‰;×"Ò"Ø�{‰{×)Ñ)¨\Ò9Ø"&×"2Ñ"2°?À2ÀAÐ3FÈÓ"JÈ_Ð]^Ð]_ÐM`Ñ"`�Ø!×-Ñ-¨n¸aÓ@ˆNØ×'Ñ'¨°!Ó4ˆHØ×%Ñ% f°4·;±;×3KÑ3KÐ%ÓL‰Fà×Ò Ñ"ÕØ�{‰{×)Ñ)¨\Ò9Ø"&×"2Ñ"2°?ÀAÓ"F�Ø!×-Ñ-¨n¸qÀ!¸fÓEˆNØ×'Ñ'¨°1°a°&Ó9ˆHØ!×-Ñ-¨nÀ5Ð-ÓIˆNØ×%Ñ% f°4·;±;×3KÑ3KÐ%ÓLˆFØ+¨^¸^ÈXÐVÐØÐ'Ð'Ð'r—   c                 óL  — |\  }}}}| j                   j                  r€| xj                  dz  c_        | j                   j                  dk(  r|dd | j	                  |dd d«      z   }| j	                  |d«      }| j	                  |d«      }| j                  |d¬«      }||||f}|S )zFPool the proper parts of `attention_inputs` after the attention layer.r¦   rÁ   Nr   r   r  r  )ro   r  r«   rÈ   rü   r
  )r•   r  r¹   rº   r¬   r»   s         rB   Úpost_attention_poolingz/FunnelAttentionStructure.post_attention_pooling™  s¶   € àDTÑAˆ˜¨¸Ø�;‰;×"Ò"Ø×Ò Ñ"ÕØ�{‰{×)Ñ)¨\Ò9Ø"1°"°1Ð"5¸×8HÑ8HÈÐYZÐY[ÐI\Ð^_Ó8`Ñ"`�Ø!×-Ñ-¨n¸aÓ@ˆNØ×'Ñ'¨°!Ó4ˆHØ!×-Ñ-¨nÀ5Ð-ÓIˆNØ+¨^¸^ÈXÐVÐØÐr—   r�   ©Nr   )r  r¦   )rž   rŸ   r    Ú__doc__r§   r[   Ú__annotations__r   rˆ   rk   r¡   r   r   r¼   r´   r²   r³   r   r   r±   rÑ   rÒ   rü   Ústrr
  r  r  r¢   r£   s   @rB   r¥   r¥   £   sC  ø… ñð Ð�sÓð!˜|ð !°õ !ð 26Ø15ñ	Kà—|‘|ðKð ! §¡Ñ.ðKð ! §¡Ñ.ð	Kð
 
ˆu�|‰|Ñ	óKð((°E·L±Lð (ÀUÇ\Á\ó (ðN(ØðN(Ø#(§;¡;ðN(Ø8=¿¹ðN(à	ˆu�U—\‘\Ñ" D¨¨e¯l©lÑ);Ñ$<Ð<Ñ	=óN(ð` e§l¡lð Àó ñb §¡ð b°cð bÐSVð bÐ_d×_kÑ_kó bð!à�e—l‘l E¨%¯,©,Ñ$7¸¸e¿l¹lÑ9KÐKÑLð!ð �C˜˜s™ T¨#¡YÐ.Ñ/ð!ð 
�‰ó	!ðD wxñ$Ø˜EŸL™L¨%°·±Ñ*=¸tÀEÇLÁLÑ?QÐQÑRð$ØZ]ð$Øpsð$à	�‰ó$ðL(Ø(-¨e¯l©lÑ(;ð(à	ˆu�|‰|˜U 5§<¡<Ñ0Ð0Ñ	1ó(ð, °u¸U¿\¹\Ñ7Jð  ÈuÐUZ×UaÑUaÑOb÷  r—   r¥   Úpositional_attnÚcontext_lenrÇ   r‚   c                 óÊ   — | j                   \  }}}}t        j                  | ||||g«      } | d d …d d …|d …d d …f   } t        j                  | |||||z
  g«      } | dd |…f   } | S )N.)rg   rk   ri   )r   r!  rÇ   Ú
batch_sizeÚn_headr°   Úmax_rel_lens          rB   Ú_relative_shift_gatherr&  §  s   € Ø/>×/DÑ/DÑ,€J�˜ ô —m‘m O°jÀ&È+ÐW^Ð5_Ó`€OØ%¢aª¨E©F²A oÑ6€OÜ—m‘m O°jÀ&È'ÐS^ÐafÑSfÐ5gÓh€OØ% c¨<¨K¨<Ð&7Ñ8€OØÐr—   c                   óÔ   ‡ — e Zd Zdededdfˆ fd„Zdd„Zdd„Z	 ddej                  d	ej                  d
ej                  de
ej                     dede
ej                  df   fd„Zˆ xZS )rb   ro   rç   r‚   Nc                 óJ  •— t         ‰| �  «        || _        || _        |j                  |j
                  |j                  }}}t        j                  |j                  «      | _	        t        j                  |j                  «      | _
        t        j                  |||z  d¬«      | _        t        j                  |||z  «      | _        t        j                  |||z  «      | _        t        j                  t!        j"                  ||g«      «      | _        t        j                  t!        j"                  ||g«      «      | _        t        j                  t!        j"                  |||g«      «      | _        t        j                  t!        j"                  ||g«      «      | _        t        j                  t!        j"                  d||g«      «      | _        t        j                  ||z  |«      | _        t        j0                  ||j2                  ¬«      | _        d|dz  z  | _        y )NF)r'   r¦   r…   rÂ   g      à?)r‡   rˆ   ro   rç   rŽ   r$  Úd_headr   r‘   r’   Úattention_dropoutÚLinearr   r   r    Ú	Parameterrk   ÚzerosÚr_w_biasÚr_r_biasrc   Úr_s_biasÚ	seg_embedr!   r�   r�   r�   Úscale)r•   ro   rç   rŽ   r$  r)  r–   s         €rB   rˆ   z$FunnelRelMultiheadAttention.__init__¸  s€  ø€ Ü‰ÑÔØˆŒØ&ˆÔØ"(§.¡.°&·-±-ÀÇÁ˜�ˆä Ÿj™j¨×)>Ñ)>Ó?ˆÔÜ!#§¡¨F×,DÑ,DÓ!EˆÔä—i‘i ¨°&©¸uÔEˆŒÜ—i‘i ¨°&©Ó9ˆŒÜ—i‘i ¨°&©Ó9ˆŒäŸ™¤U§[¡[°&¸&Ð1AÓ%BÓCˆŒÜŸ™¤U§[¡[°&¸&Ð1AÓ%BÓCˆŒÜŸ™¤U§[¡[°'¸6À6Ð1JÓ%KÓLˆŒÜŸ™¤U§[¡[°&¸&Ð1AÓ%BÓCˆŒÜŸ™¤e§k¡k°1°f¸fÐ2EÓ&FÓGˆŒäŸ™ 6¨F¡?°GÓ<ˆŒÜŸ,™, w°F×4IÑ4IÔJˆŒØ˜F C™KÑ(ˆ�
r—   c                 ó~  — | j                   j                  dk(  rŽ|\  }}}}| j                  | j                  z  }	| j                  }
t        j                  d||	z   |
«      }||dd…df   z  }||dd…df   z  }t        j                  d||«      t        j                  d||«      z   }nŽ|j                  d   |k7  rdnd}|| j                     |dz
     }| j                  | j                  z  }| j                  }
t        j                  d||
«      }t        j                  d||z   |«      }t        |||«      }|�||z  }|S )	z5Relative attention score for the positional encodingsrÁ   zbinh,dnh->bindNzbind,jd->bnijr   r¦   ztd,dnh->tnhzbinh,tnh->bnit)
ro   rÈ   r/  r2  rc   rk   Úeinsumrg   rç   r&  )r•   r¹   r   r!  r»   rÝ   rß   rÞ   rà   ÚuÚw_rÚq_r_attentionÚq_r_attention_1Úq_r_attention_2r   rÇ   rE   r,   Úr_heads                      rB   Úrelative_positional_attentionz9FunnelRelMultiheadAttention.relative_positional_attentionÏ  sK  € ð �;‰;×%Ñ%¨Ò5ð #2ÑˆC��S˜%à—‘ §
¡
Ñ*ˆAà—-‘-ˆCô "ŸL™LÐ)9¸6ÀA¹:ÀsÓKˆMØ+¨c²!°T°'©lÑ:ˆOØ+¨b²°D°©kÑ9ˆOô $Ÿl™l¨?¸OÈSÓQÔTY×T`ÑT`Ø °%óUñ ‰Oð  Ÿ™ a™¨KÒ7‘A¸QˆEð   × 0Ñ 0Ñ1°%¸!±)Ñ<ˆAà—‘ §
¡
Ñ*ˆAà—-‘-ˆCô —\‘\ -°°CÓ8ˆFä#Ÿl™lÐ+;¸VÀa¹ZÈÓPˆOä4°_ÀkÐSXÓYˆOàÐØ˜xÑ'ˆOØÐr—   c                 óÎ  — |€y|j                   \  }}}| j                  | j                  z  }t        j                  d||z   | j
                  «      }|dd…df   j                  ||j                   d   ||g«      }t        j                  |dd¬«      \  }	}
t        j                  ||
j                  |j                   «      |	j                  |j                   «      «      }|�||z  }|S )z/Relative attention score for the token_type_idsNr   zbind,snd->bnisr¦   r   rÄ   rÅ   )	rg   r0  r2  rk   r4  r1  rÓ   rU   Úwhere)r•   rº   r   r»   r#  r°   r!  r0  Útoken_type_biasÚdiff_token_typeÚsame_token_typeÚtoken_type_attns               rB   Úrelative_token_type_attentionz9FunnelRelMultiheadAttention.relative_token_type_attentionù  sã   € àÐ!ØØ+9×+?Ñ+?Ñ(ˆ
�G˜[ð —=‘= 4§:¡:Ñ-ˆô  Ÿ,™,Ð'7¸À(Ñ9JÈDÏNÉNÓ[ˆà'ª¨4¨Ñ0×7Ñ7¸ÀVÇ\Á\ÐRSÁ_ÐV]Ð_jÐ8kÓlˆä+0¯;©;°ÈÈrÔ+RÑ(ˆ˜äŸ+™+Ø˜O×2Ñ2°>×3GÑ3GÓHÈ/×J`ÑJ`Ðao×auÑauÓJvó
ˆð ÐØ˜xÑ'ˆOØÐr—   ÚqueryÚkeyÚvaluer  Úoutput_attentions.c                 ó  — |\  }}}}	|j                   \  }
}}|j                   d   }| j                  j                  | j                  j                  }}| j	                  |«      j                  |
|||«      }| j                  |«      j                  |
|||«      }| j                  |«      j                  |
|||«      }|| j                  z  }| j                  | j                  z  }t        j                  d||z   |«      }| j                  ||||	«      }| j                  |||	«      }||z   |z   }|j                  }|j                  «       }|�%|t         d|d d …d d f   j                  «       z
  z  z
  }t        j"                  |d|¬«      }| j%                  |«      }t        j                  d||«      }| j'                  |j)                  |
|||z  «      «      }| j+                  |«      }| j-                  ||z   «      }|r||fS |fS )Nr   zbind,bjnd->bnijrÄ   )rÆ   r²   zbnij,bjnd->bind)rg   ro   r$  r)  r   Úviewr   r    r2  r.  rk   r4  r;  rB  r²   ÚfloatÚINFÚsoftmaxr*  r!   ri   r’   r�   )r•   rC  rD  rE  r  rF  r¹   rº   r¬   r»   r#  r°   Ú_r!  r$  r)  r   r   r    r.  Úcontent_scorer   rA  Ú
attn_scorer²   Ú	attn_probÚattn_vecÚattn_outr  s                                rB   rœ   z#FunnelRelMultiheadAttention.forward  sþ  € ð EUÑAˆ˜¨¸à!&§¡Ñˆ
�G˜QØ—i‘i ‘lˆØŸ™×+Ñ+¨T¯[©[×-?Ñ-?�ˆð —‘˜UÓ#×(Ñ(¨°W¸fÀfÓMˆà—‘˜SÓ!×&Ñ& z°;ÀÈÓOˆØ—‘˜UÓ#×(Ñ(¨°[À&È&ÓQˆà˜$Ÿ*™*Ñ$ˆà—=‘= 4§:¡:Ñ-ˆäŸ™Ð%6¸ÀÑ8IÈ6ÓRˆØ×<Ñ<¸_ÈfÐVaÐckÓlˆØ×<Ñ<¸^ÈVÐU]Ó^ˆð # _Ñ4°ÑFˆ
ð × Ñ ˆØ×%Ñ%Ó'ˆ
àÐ%Ø#¤c¨Q°ÂÀ4ÈÀÑ1N×1TÑ1TÓ1VÑ-VÑ&WÑWˆJä—M‘M *°"¸EÔBˆ	Ø×*Ñ*¨9Ó5ˆ	ô —<‘<Ð 1°9¸fÓEˆð —>‘> (×"2Ñ"2°:¸wÈÐQWÉÓ"XÓYˆØ×&Ñ& xÓ0ˆà—‘ ¨Ñ!1Ó2ˆÙ&7�˜	Ð"ÐF¸f¸YÐFr—   r›   ©F)rž   rŸ   r    r   r[   rˆ   r;  rB  rk   r¡   r   Úboolrœ   r¢   r£   s   @rB   rb   rb   ·  s–   ø„ ð)˜|ð )¸#ð )À$õ )ó.(óTð< #(ñ3Gà�|‰|ð3Gð �\‰\ð3Gð �|‰|ð	3Gð
   §¡Ñ-ð3Gð  ð3Gð 
ˆu�|‰|˜SÐ Ñ	!÷3Gr—   rb   c                   ó`   ‡ — e Zd Zdeddfˆ fd„Zdej                  dej                  fd„Zˆ xZS )rY   ro   r‚   Nc                 óü  •— t         ‰| �  «        t        j                  |j                  |j
                  «      | _        t        |j                     | _	        t        j                  |j                  «      | _        t        j                  |j
                  |j                  «      | _        t        j                  |j                  «      | _        t        j                  |j                  |j                   «      | _        y r›   )r‡   rˆ   r   r+  rŽ   Úd_innerr"   r   Ú
hidden_actÚactivation_functionr‘   Úactivation_dropoutr#   r’   r“   r�   r�   r�   r”   s     €rB   rˆ   zFunnelPositionwiseFFN.__init__H  sž   ø€ Ü‰ÑÔÜŸ	™	 &§.¡.°&·.±.ÓAˆŒÜ#)¨&×*;Ñ*;Ñ#<ˆÔ Ü"$§*¡*¨V×-FÑ-FÓ"GˆÔÜŸ	™	 &§.¡.°&·.±.ÓAˆŒÜ—z‘z &×"7Ñ"7Ó8ˆŒÜŸ,™, v§~¡~°v×7LÑ7LÓMˆ�r—   Úhiddenc                 óÔ   — | j                  |«      }| j                  |«      }| j                  |«      }| j                  |«      }| j	                  |«      }| j                  ||z   «      S r›   )r"   rX  rY  r#   r“   r�   )r•   rZ  Úhs      rB   rœ   zFunnelPositionwiseFFN.forwardQ  s^   € Ø�M‰M˜&Ó!ˆØ×$Ñ$ QÓ'ˆØ×#Ñ# AÓ&ˆØ�M‰M˜!ÓˆØ�L‰L˜‹OˆØ�‰˜v¨™zÓ*Ð*r—   )	rž   rŸ   r    r   rˆ   rk   r¡   rœ   r¢   r£   s   @rB   rY   rY   G  s4   ø„ ðN˜|ð N°õ Nð+˜eŸl™lð +¨u¯|©|÷ +r—   rY   c                   óˆ   ‡ — e Zd Zdededdfˆ fd„Z	 ddej                  dej                  dej                  d	ede	f
d
„Z
ˆ xZS )ÚFunnelLayerro   rç   r‚   Nc                 ód   •— t         ‰| �  «        t        ||«      | _        t	        |«      | _        y r›   )r‡   rˆ   rb   r$   rY   r%   )r•   ro   rç   r–   s      €rB   rˆ   zFunnelLayer.__init__[  s(   ø€ Ü‰ÑÔÜ4°V¸[ÓIˆŒÜ(¨Ó0ˆ�r—   rC  rD  rE  rF  c                 ón   — | j                  |||||¬«      }| j                  |d   «      }|r||d   fS |fS )N©rF  r   r   )r$   r%   )r•   rC  rD  rE  r  rF  Úattnr  s           rB   rœ   zFunnelLayer.forward`  sH   € ð �~‰~˜e S¨%Ð1AÐUfˆ~ÓgˆØ—‘˜$˜q™'Ó"ˆÙ$5�˜˜Q™Ð ÐD¸F¸9ÐDr—   rR  )rž   rŸ   r    r   r[   rˆ   rk   r¡   rS  r   rœ   r¢   r£   s   @rB   r^  r^  Z  si   ø„ ð1˜|ð 1¸#ð 1À$õ 1ð #(ñ
Eà�|‰|ð
Eð �\‰\ð
Eð �|‰|ð	
Eð  ð
Eð 
÷
Er—   r^  c                   óª   ‡ — e Zd Zdeddfˆ fd„Z	 	 	 	 	 ddej                  deej                     deej                     ded	ed
ede	e
ef   fd„Zˆ xZS )ÚFunnelEncoderro   r‚   Nc                 óT  •— t         ‰| �  «        || _        t        |«      | _        t        j                  t        |j                  «      D ���cg c];  \  }}t        j                  t        |«      D �cg c]  }t        ||«      ‘Œ c}«      ‘Œ= c}}}«      | _        y c c}w c c}}}w r›   )r‡   rˆ   ro   r¥   Úattention_structurer   Ú
ModuleListÚ	enumerater_   rÏ   r^  r`   )r•   ro   rç   Ú
block_sizerL  r–   s        €rB   rˆ   zFunnelEncoder.__init__n  s†   ø€ Ü‰ÑÔØˆŒÜ#;¸FÓ#CˆÔ Ü—m‘mô 09¸×9KÑ9KÓ/L÷ð á+�K ô —‘ÌÈzÓIZÖ[ÀAœ{¨6°;Õ?Ò[Õ\ôó
ˆ�ùâ[ùôs   Á$B#Á3BÂ	B#ÂB#r™   r¬   r­   rF  Úoutput_hidden_statesÚreturn_dictc           
      ó  — |j                  |«      }| j                  j                  |||¬«      }|}|r|fnd }	|rdnd }
t        | j                  «      D �]  \  }}|j                  d«      | j                  j                  rdndkD  }|xr |dkD  }|r| j                  j                  ||«      \  }}t        |«      D ]¥  \  }}t        | j                  j                  |   «      D ]{  }|dk(  xr	 |dk(  xr |}|r}| j                  j                  r|n|x}}n|x}x}} ||||||¬«      }|d   }|r| j                  j                  |«      }|r|
|dd  z   }
|sŒv|	|fz   }	Œ} Œ§ �Œ |st        d„ ||	|
fD «       «      S t        ||	|
¬«      S )	N©r¬   r­   r?   r   r¦   r   ra  c              3   ó&   K  — | ]	  }|€Œ|–— Œ y ­wr›   r?   ©r@   r,   s     rB   rC   z(FunnelEncoder.forward.<locals>.<genexpr>¨  ó   è ø€ Òa˜qÐSTÑS`œÑaùó   ‚Š©Úlast_hidden_stateÚhidden_statesÚ
attentions)Útype_asrf  r¼   rh  r`   r¯   ro   rµ   r  rÏ   Úblock_repeatsr  r  rÿ   r   )r•   r™   r¬   r­   rF  rj  rk  r  rZ  Úall_hidden_statesÚall_attentionsrç   ÚblockÚpooling_flagÚpooled_hiddenr}   ÚlayerÚrepeat_indexÚ
do_poolingrC  rD  rE  Úlayer_outputs                          rB   rœ   zFunnelEncoder.forwardy  sà  € ð (×/Ñ/°Ó>ˆØ×3Ñ3×IÑIØØ)Ø)ð Jó 
Ðð
 ˆá0D˜]Ñ,È$ÐÙ0™°dˆä"+¨D¯K©KÓ"8ó 	JÑˆK˜Ø!Ÿ;™; q›>°$·+±+×2JÒ2J©QÐPQÑRˆLØ'Ò;¨K¸!©OˆLÙØ26×2JÑ2J×2`Ñ2`ØÐ,ó3Ñ/�Ð/ô '0°Ó&6ò JÑ"�˜UÜ$)¨$¯+©+×*CÑ*CÀKÑ*PÓ$Qò J�LØ".°!Ñ"3Ò!\¸+ÈÑ:JÒ!\ÐP\�JÙ!Ø -˜Ø04·±×0GÒ0G¡fÈ]ÐZ˜™eà.4Ð4˜Ð4  eÙ#(¨°°UÐ<LÐ`qÔ#r�LØ)¨!™_�FÙ!Ø+/×+CÑ+C×+ZÑ+ZÐ[kÓ+lÐ(á(Ø)7¸,ÀqÀrÐ:JÑ)J˜Ú+Ø,=ÀÀ	Ñ,IÑ)ñJòJð	Jñ2 ÜÑa VÐ->ÀÐ$OÔaÓaÐaÜ°ÐGXÐesÔtÐtr—   ©NNFFT©rž   rŸ   r    r   rˆ   rk   r¡   r   rS  r   r   r   rœ   r¢   r£   s   @rB   rd  rd  m  s˜   ø„ ð	
˜|ð 	
°õ 	
ð 26Ø15Ø"'Ø%*Ø ñ0uà—|‘|ð0uð ! §¡Ñ.ð0uð ! §¡Ñ.ð	0uð
  ð0uð #ð0uð ð0uð 
ˆu�oÐ%Ñ	&÷0ur—   rd  rý   ré   Ú
target_lenrµ   rï   c           	      ó6  — |dk(  r| S |r| dd…dd…f   }| dd…dd…f   } t        j                  | |d¬«      }|rT|r)t        j                  j	                  |ddd|dz
  ddf«      }|dd…d|dz
  …f   }t        j
                  |gd¬«      }|S |dd…d|…f   }|S )z{
    Upsample tensor `x` to match `target_len` by repeating the tokens `stride` time on the sequence length dimension.
    r   N)ÚrepeatsrÆ   r   rÅ   )rk   Úrepeat_interleaver   r¶   r·   rÎ   )rý   ré   rƒ  rµ   rï   Úclsr  s          rB   Úupsamplerˆ  ¬  sÁ   € ð �‚{ØˆÙØ’�2�A�2�‰hˆØŠa�‘ˆe‰HˆÜ×$Ñ$ Q°¸AÔ>€FÙÙÜ—]‘]×&Ñ& v°°1°a¸À!¹ÀQÈÐ/JÓKˆFØšÐ+˜Z¨!™^Ð+Ð+Ñ,ˆÜ—‘˜C ˜=¨aÔ0ˆð €Mð š˜;˜J˜;˜Ñ'ˆØ€Mr—   c                   óÂ   ‡ — e Zd Zdeddfˆ fd„Z	 	 	 	 	 ddej                  dej                  deej                     deej                     d	ed
edede	e
ef   fd„Zˆ xZS )ÚFunnelDecoderro   r‚   Nc           	      óä   •— t         ‰| �  «        || _        t        |«      | _        t        j                  t        |j                  «      D �cg c]  }t        |d«      ‘Œ c}«      | _
        y c c}w )Nr   )r‡   rˆ   ro   r¥   rf  r   rg  rÏ   Únum_decoder_layersr^  ra   )r•   ro   rL  r–   s      €rB   rˆ   zFunnelDecoder.__init__Ã  sR   ø€ Ü‰ÑÔØˆŒÜ#;¸FÓ#CˆÔ Ü—m‘mÄUÈ6×KdÑKdÓEeÖ$fÀ¤[°¸Õ%;Ò$fÓgˆ�ùÒ$fs   ÁA-Úfinal_hiddenÚfirst_block_hiddenr¬   r­   rF  rj  rk  c                 óè  — t        |dt        | j                  j                  «      dz
  z  |j                  d   | j                  j
                  | j                  j                  ¬«      }||z   }	|r|	fnd }
|rdnd }| j                  j                  |	||¬«      }| j                  D ]'  } ||	|	|	||¬«      }|d   }	|r||dd  z   }|sŒ"|
|	fz   }
Œ) |st        d„ |	|
|fD «       «      S t        |	|
|¬	«      S )
Nr¦   r   )ré   rƒ  rµ   rï   r?   rm  ra  r   c              3   ó&   K  — | ]	  }|€Œ|–— Œ y ­wr›   r?   ro  s     rB   rC   z(FunnelDecoder.forward.<locals>.<genexpr>ï  rp  rq  rr  )rˆ  rh   ro   r_   rg   rµ   rï   rf  r¼   ra   rÿ   r   )r•   r�  rŽ  r¬   r­   rF  rj  rk  Úupsampled_hiddenrZ  rx  ry  r  r}  r€  s                  rB   rœ   zFunnelDecoder.forwardÉ  s%  € ô $ØØœ˜TŸ[™[×4Ñ4Ó5¸Ñ9Ñ:Ø)×/Ñ/°Ñ2ØŸ™×1Ñ1ØŸ™×1Ñ1ô
Ðð "Ð$6Ñ6ˆÙ)=˜V™IÀ4ÐÙ0™°dˆà×3Ñ3×IÑIØØ)Ø)ð Jó 
Ðð —[‘[ò 	BˆEÙ  ¨°Ð9IÐ]nÔoˆLØ! !‘_ˆFá Ø!/°,¸q¸rÐ2BÑ!B�Ú#Ø$5¸¸	Ñ$AÑ!ð	Bñ ÜÑa VÐ->ÀÐ$OÔaÓaÐaÜ°ÐGXÐesÔtÐtr—   r�  r‚  r£   s   @rB   rŠ  rŠ  Â  sª   ø„ ðh˜|ð h°õ hð 26Ø15Ø"'Ø%*Ø ñ'uà—l‘lð'uð "ŸL™Lð'uð ! §¡Ñ.ð	'uð
 ! §¡Ñ.ð'uð  ð'uð #ð'uð ð'uð 
ˆu�oÐ%Ñ	&÷'ur—   rŠ  c                   ód   ‡ — e Zd ZdZdeddfˆ fd„Zdej                  dej                  fd„Zˆ xZ	S )ÚFunnelDiscriminatorPredictionszEPrediction module for the discriminator, made up of two dense layers.ro   r‚   Nc                 óØ   •— t         ‰| �  «        || _        t        j                  |j
                  |j
                  «      | _        t        j                  |j
                  d«      | _        y r  )r‡   rˆ   ro   r   r+  rŽ   ÚdenseÚdense_predictionr”   s     €rB   rˆ   z'FunnelDiscriminatorPredictions.__init__ö  sF   ø€ Ü‰ÑÔØˆŒÜ—Y‘Y˜vŸ~™~¨v¯~©~Ó>ˆŒ
Ü "§	¡	¨&¯.©.¸!Ó <ˆÕr—   Údiscriminator_hidden_statesc                 ó¬   — | j                  |«      }t        | j                  j                     |«      }| j	                  |«      j                  d«      }|S )NrÄ   )r•  r   ro   rW  r–  Úsqueeze)r•   r—  rt  Úlogitss       rB   rœ   z&FunnelDiscriminatorPredictions.forwardü  sJ   € ØŸ
™
Ð#>Ó?ˆÜ˜tŸ{™{×5Ñ5Ñ6°}ÓEˆØ×&Ñ& }Ó5×=Ñ=¸bÓAˆØˆr—   )
rž   rŸ   r    r  r   rˆ   rk   r¡   rœ   r¢   r£   s   @rB   r“  r“  ó  s4   ø„ ÙOð=˜|ð =°õ =ð°5·<±<ð ÀEÇLÁL÷ r—   r“  c                   ó"   — e Zd ZdZeZeZdZd„ Z	y)ÚFunnelPreTrainedModelz†
    An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained
    models.
    Úfunnelc                 ó  — |j                   j                  }|j                  d«      dk7  rÛt        |dd «      �•| j                  j
                  €>|j                  j                  \  }}t        j                  dt        ||z   «      z  «      }n| j                  j
                  }t        j                  j                  |j                  |¬«       t        |dd «      �+t        j                  j                  |j                  d«       y y |dk(  �r<t        j                  j!                  |j"                  | j                  j$                  ¬	«       t        j                  j!                  |j&                  | j                  j$                  ¬	«       t        j                  j!                  |j(                  | j                  j$                  ¬	«       t        j                  j!                  |j*                  | j                  j$                  ¬	«       t        j                  j!                  |j,                  | j                  j$                  ¬	«       y |d
k(  rÀ| j                  j
                  €dn| j                  j
                  }t        j                  j                  |j.                  j                  |¬«       |j.                  j0                  �F|j.                  j                  j2                  |j.                  j0                     j5                  «        y y y )Nr+  rÄ   r&   rÂ   )Ústdr'   g        rb   )Úbr�   )r–   rž   Úfindrd   ro   Úinitializer_stdr&   rg   rq   ÚsqrtrI  r   ÚinitÚnormal_Ú	constant_r'   Úuniform_r.  Úinitializer_ranger/  rc   r0  r1  r(   r„   rm   Úzero_)r•   ÚmoduleÚ	classnameÚfan_outÚfan_inrŸ  s         rB   Ú_init_weightsz#FunnelPreTrainedModel._init_weights  s  € Ø×$Ñ$×-Ñ-ˆ	Ø�>‰>˜(Ó# rÒ)Ü�v˜x¨Ó.Ð:Ø—;‘;×.Ñ.Ð6Ø&,§m¡m×&9Ñ&9‘O�G˜VÜŸ'™' #¬¨f°wÑ.>Ó(?Ñ"?Ó@‘CàŸ+™+×5Ñ5�CÜ—‘—‘ §¡°3�Ô7Ü�v˜v tÓ,Ð8Ü—‘×!Ñ! &§+¡+¨sÕ3ð 9àÐ7Ó7Ü�G‰G×Ñ˜VŸ_™_°·±×0MÑ0MÐÔNÜ�G‰G×Ñ˜VŸ_™_°·±×0MÑ0MÐÔNÜ�G‰G×Ñ˜VŸ_™_°·±×0MÑ0MÐÔNÜ�G‰G×Ñ˜VŸ_™_°·±×0MÑ0MÐÔNÜ�G‰G×Ñ˜V×-Ñ-°·±×1NÑ1NÐÕOØÐ,Ò,ØŸ™×4Ñ4Ð<‘#À$Ç+Á+×B]ÑB]ˆCÜ�G‰G�O‰O˜F×2Ñ2×9Ñ9¸sˆOÔCØ×%Ñ%×1Ñ1Ð=Ø×&Ñ&×-Ñ-×2Ñ2°6×3IÑ3I×3UÑ3UÑV×\Ñ\Õ^ð >ð -r—   N)
rž   rŸ   r    r  r   Úconfig_classr   Úload_tf_weightsÚbase_model_prefixr®  r?   r—   rB   rœ  rœ    s   „ ñð
  €LØ/€OØ Ðó_r—   rœ  c                   ód   ‡ — e Zd Zdededdfˆ fd„Zdej                  dej                  fd„Zˆ xZ	S )ÚFunnelClassificationHeadro   Ún_labelsr‚   Nc                 ó  •— t         ‰| �  «        t        j                  |j                  |j                  «      | _        t        j                  |j                  «      | _        t        j                  |j                  |«      | _	        y r›   )
r‡   rˆ   r   r+  rŽ   Úlinear_hiddenr‘   r’   r“   Ú
linear_out)r•   ro   r´  r–   s      €rB   rˆ   z!FunnelClassificationHead.__init__'  sU   ø€ Ü‰ÑÔÜŸY™Y v§~¡~°v·~±~ÓFˆÔÜ—z‘z &×"7Ñ"7Ó8ˆŒÜŸ)™) F§N¡N°HÓ=ˆ�r—   rZ  c                 ó’   — | j                  |«      }t        j                  |«      }| j                  |«      }| j	                  |«      S r›   )r¶  rk   Útanhr“   r·  )r•   rZ  s     rB   rœ   z FunnelClassificationHead.forward-  s=   € Ø×#Ñ# FÓ+ˆÜ—‘˜FÓ#ˆØ—‘˜fÓ%ˆØ�‰˜vÓ&Ð&r—   )
rž   rŸ   r    r   r[   rˆ   rk   r¡   rœ   r¢   r£   s   @rB   r³  r³  &  s8   ø„ ð>˜|ð >°sð >¸tõ >ð'˜eŸl™lð '¨u¯|©|÷ 'r—   r³  c                   ó¾   — e Zd ZU dZdZeej                     ed<   dZ	eej                     ed<   dZ
eeej                        ed<   dZeeej                        ed<   y)ÚFunnelForPreTrainingOutputaé  
    Output type of [`FunnelForPreTraining`].

    Args:
        loss (*optional*, returned when `labels` is provided, `torch.FloatTensor` of shape `(1,)`):
            Total loss of the ELECTRA-style objective.
        logits (`torch.FloatTensor` of shape `(batch_size, sequence_length)`):
            Prediction scores of the head (scores for each token before SoftMax).
        hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
            Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
            shape `(batch_size, sequence_length, hidden_size)`.

            Hidden-states of the model at the output of each layer plus the initial embedding outputs.
        attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
            Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
            sequence_length)`.

            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
            heads.
    NÚlossrš  rt  ru  )rž   rŸ   r    r  r¼  r   rk   ÚFloatTensorr  rš  rt  r   ru  r?   r—   rB   r»  r»  4  sg   … ñð* )-€Dˆ(�5×$Ñ$Ñ
%Ó,Ø*.€FˆH�U×&Ñ&Ñ'Ó.Ø8<€M�8˜E %×"3Ñ"3Ñ4Ñ5Ó<Ø59€J�˜˜u×0Ñ0Ñ1Ñ2Ô9r—   r»  a(  

    The Funnel Transformer model was proposed in [Funnel-Transformer: Filtering out Sequential Redundancy for Efficient
    Language Processing](https://arxiv.org/abs/2006.03236) by Zihang Dai, Guokun Lai, Yiming Yang, Quoc V. Le.

    This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
    library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
    etc.)

    This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
    Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
    and behavior.

    Parameters:
        config ([`FunnelConfig`]): Model configuration class with all the parameters of the model.
            Initializing with a config file does not load the weights associated with the model, only the
            configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
aÅ  
    Args:
        input_ids (`torch.LongTensor` of shape `({0})`):
            Indices of input sequence tokens in the vocabulary.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are input IDs?](../glossary#input-ids)
        attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*):
            Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:

            - 1 for tokens that are **not masked**,
            - 0 for tokens that are **masked**.

            [What are attention masks?](../glossary#attention-mask)
        token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*):
            Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
            1]`:

            - 0 corresponds to a *sentence A* token,
            - 1 corresponds to a *sentence B* token.

            [What are token type IDs?](../glossary#token-type-ids)
        inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*):
            Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
            is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
            model's internal embedding lookup matrix.
        output_attentions (`bool`, *optional*):
            Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
            tensors for more detail.
        output_hidden_states (`bool`, *optional*):
            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
            more detail.
        return_dict (`bool`, *optional*):
            Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
z¨
    The base Funnel Transformer Model transformer outputting raw hidden-states without upsampling head (also called
    decoder) or any task-specific head on top.
    c                   ó¸  ‡ — e Zd Zdeddfˆ fd„Zdej                  fd„Zdej                  ddfd„Z e	e
j                  d«      «       ed	ee¬
«      	 	 	 	 	 	 	 	 	 ddeej"                     deej"                     deej"                     deej"                     deej"                     deej"                     dee   dee   dee   deeef   fd„«       «       Zˆ xZS )ÚFunnelBaseModelro   r‚   Nc                 ó„   •— t         ‰| �  |«       t        |«      | _        t	        |«      | _        | j                  «        y r›   )r‡   rˆ   r�   r)   rd  ÚencoderÚ	post_initr”   s     €rB   rˆ   zFunnelBaseModel.__init__“  s4   ø€ Ü‰Ñ˜Ô ä*¨6Ó2ˆŒÜ$ VÓ,ˆŒð 	�‰Õr—   c                 ó.   — | j                   j                  S r›   ©r)   r(   ©r•   s    rB   Úget_input_embeddingsz$FunnelBaseModel.get_input_embeddingsœ  ó   € Ø�‰×.Ñ.Ð.r—   Únew_embeddingsc                 ó&   — || j                   _        y r›   rÄ  ©r•   rÈ  s     rB   Úset_input_embeddingsz$FunnelBaseModel.set_input_embeddingsŸ  ó   € Ø*8ˆ�‰Õ'r—   úbatch_size, sequence_lengthúfunnel-transformer/small-base©Ú
checkpointÚoutput_typer¯  r˜   r¬   r­   Úposition_idsÚ	head_maskr™   rF  rj  rk  c
                 óV  — |�|n| j                   j                  }|�|n| j                   j                  }|	�|	n| j                   j                  }	|�|�t	        d«      ‚|�#| j                  ||«       |j                  «       }
n!|�|j                  «       d d }
nt	        d«      ‚|�|j                  n|j                  }|€t        j                  |
|¬«      }|€&t        j                  |
t        j                  |¬«      }| j                  ||¬«      }| j                  ||||||	¬«      }|S )NúDYou cannot specify both input_ids and inputs_embeds at the same timerÄ   ú5You have to specify either input_ids or inputs_embeds©r³   rÃ   ©r™   ©r¬   r­   rF  rj  rk  )ro   rF  rj  Úuse_return_dictÚ
ValueErrorÚ%warn_if_padding_and_no_attention_maskr¯   r³   rk   Úonesr-  ró   r)   rÁ  )r•   r˜   r¬   r­   rÒ  rÓ  r™   rF  rj  rk  Úinput_shaper³   Úencoder_outputss                rB   rœ   zFunnelBaseModel.forward¢  s=  € ð$ 2CÐ1NÑ-ÐTX×T_ÑT_×TqÑTqÐà$8Ð$DÑ È$Ï+É+×JjÑJjð 	ð &1Ð%<‘kÀ$Ç+Á+×B]ÑB]ˆàÐ  ]Ð%>ÜÐcÓdÐdØÐ"Ø×6Ñ6°yÀ.ÔQØ#Ÿ.™.Ó*‰KØÐ&Ø'×,Ñ,Ó.¨s°Ð3‰KäÐTÓUÐUà%.Ð%:�×!Ò!À×@TÑ@TˆàÐ!Ü"ŸZ™Z¨¸FÔCˆNØÐ!Ü"Ÿ[™[¨¼E¿J¹JÈvÔVˆNð Ÿ™¨	À˜ÓOˆàŸ,™,ØØ)Ø)Ø/Ø!5Ø#ð 'ó 
ˆð Ðr—   ©	NNNNNNNNN)rž   rŸ   r    r   rˆ   r   r‰   rÆ  rË  r   ÚFUNNEL_INPUTS_DOCSTRINGÚformatr   r   Ú_CONFIG_FOR_DOCr   rk   r¡   rS  r   r   rœ   r¢   r£   s   @rB   r¿  r¿  ‹  sE  ø„ ð˜|ð °õ ð/ b§l¡ló /ð9°2·<±<ð 9ÀDó 9ñ +Ð+B×+IÑ+IÐJgÓ+hÓiÙØ2Ø#Ø$ôð -1Ø15Ø15Ø/3Ø,0Ø04Ø,0Ø/3Ø&*ñ/à˜EŸL™LÑ)ð/ð ! §¡Ñ.ð/ð ! §¡Ñ.ð	/ð
 ˜uŸ|™|Ñ,ð/ð ˜EŸL™LÑ)ð/ð   §¡Ñ-ð/ð $ D™>ð/ð ' t™nð/ð ˜d‘^ð/ð 
ˆu�oÐ%Ñ	&ò/óó jô/r—   r¿  zlThe bare Funnel Transformer Model transformer outputting raw hidden-states without any specific head on top.c                   óx  ‡ — e Zd Zdeddfˆ fd„Zdej                  fd„Zdej                  ddfd„Z e	e
j                  d«      «       eeee¬	«      	 	 	 	 	 	 	 dd
eej$                     deej$                     deej$                     deej$                     dee   dee   dee   deeef   fd„«       «       Zˆ xZS )ÚFunnelModelro   r‚   Nc                 ó²   •— t         ‰| �  |«       || _        t        |«      | _        t        |«      | _        t        |«      | _        | j                  «        y r›   )
r‡   rˆ   ro   r�   r)   rd  rÁ  rŠ  ÚdecoderrÂ  r”   s     €rB   rˆ   zFunnelModel.__init__ß  sG   ø€ Ü‰Ñ˜Ô ØˆŒÜ*¨6Ó2ˆŒÜ$ VÓ,ˆŒÜ$ VÓ,ˆŒð 	�‰Õr—   c                 ó.   — | j                   j                  S r›   rÄ  rÅ  s    rB   rÆ  z FunnelModel.get_input_embeddingsé  rÇ  r—   rÈ  c                 ó&   — || j                   _        y r›   rÄ  rÊ  s     rB   rË  z FunnelModel.set_input_embeddingsì  rÌ  r—   rÍ  rÏ  r˜   r¬   r­   r™   rF  rj  rk  c           	      óÊ  — |�|n| j                   j                  }|�|n| j                   j                  }|�|n| j                   j                  }|�|�t	        d«      ‚|�#| j                  ||«       |j                  «       }n!|�|j                  «       d d }nt	        d«      ‚|�|j                  n|j                  }	|€t        j                  ||	¬«      }|€&t        j                  |t        j                  |	¬«      }| j                  ||¬«      }| j                  ||||d|¬«      }
| j                  |
d	   |
d
   | j                   j                  d	      |||||¬«      }|s6d	}|d	   f}|r|d
z  }||
d
   ||   z   fz   }|r|d
z  }||
d   ||   z   fz   }|S t!        |d	   |r|
j"                  |j"                  z   nd |r|
j$                  |j$                  z   ¬«      S d ¬«      S )NrÕ  rÄ   rÖ  r×  rÃ   rØ  TrÙ  r   r   )r�  rŽ  r¬   r­   rF  rj  rk  r¦   rr  )ro   rF  rj  rÚ  rÛ  rÜ  r¯   r³   rk   rÝ  r-  ró   r)   rÁ  rç  r_   r   rt  ru  )r•   r˜   r¬   r­   r™   rF  rj  rk  rÞ  r³   rß  Údecoder_outputsÚidxÚoutputss                 rB   rœ   zFunnelModel.forwardï  sA  € ð  2CÐ1NÑ-ÐTX×T_ÑT_×TqÑTqÐà$8Ð$DÑ È$Ï+É+×JjÑJjð 	ð &1Ð%<‘kÀ$Ç+Á+×B]ÑB]ˆàÐ  ]Ð%>ÜÐcÓdÐdØÐ"Ø×6Ñ6°yÀ.ÔQØ#Ÿ.™.Ó*‰KØÐ&Ø'×,Ñ,Ó.¨s°Ð3‰KäÐTÓUÐUà%.Ð%:�×!Ò!À×@TÑ@TˆàÐ!Ü"ŸZ™Z¨¸FÔCˆNØÐ!Ü"Ÿ[™[¨¼E¿J¹JÈvÔVˆNð Ÿ™¨	À˜ÓOˆàŸ,™,ØØ)Ø)Ø/Ø!%Ø#ð 'ó 
ˆð Ÿ,™,Ø(¨Ñ+Ø.¨qÑ1°$·+±+×2IÑ2IÈ!Ñ2LÑMØ)Ø)Ø/Ø!5Ø#ð 'ó 
ˆñ ØˆCØ& qÑ)Ð+ˆGÙ#Ø�q‘�Ø! _°QÑ%7¸/È#Ñ:NÑ%NÐ$PÑP�Ù Ø�q‘�Ø! _°QÑ%7¸/È#Ñ:NÑ%NÐ$PÑP�ØˆNäØ-¨aÑ0á#ð +×8Ñ8¸?×;XÑ;XÒXàÙTe˜×2Ñ2°_×5OÑ5OÑOô
ð 	
ð
 lpô
ð 	
r—   )NNNNNNN)rž   rŸ   r    r   rˆ   r   r‰   rÆ  rË  r   rá  râ  r   Ú_CHECKPOINT_FOR_DOCr   rã  r   rk   r¡   rS  r   r   rœ   r¢   r£   s   @rB   rå  rå  Ú  s#  ø„ ð
˜|ð °õ ð/ b§l¡ló /ð9°2·<±<ð 9ÀDó 9ñ +Ð+B×+IÑ+IÐJgÓ+hÓiÙØ&Ø#Ø$ôð -1Ø15Ø15Ø04Ø,0Ø/3Ø&*ñH
à˜EŸL™LÑ)ðH
ð ! §¡Ñ.ðH
ð ! §¡Ñ.ð	H
ð
   §¡Ñ-ðH
ð $ D™>ðH
ð ' t™nðH
ð ˜d‘^ðH
ð 
ˆu�oÐ%Ñ	&òH
óó jôH
r—   rå  zŒ
    Funnel Transformer model with a binary classification head on top as used during pretraining for identifying
    generated tokens.
    c                   óR  ‡ — e Zd Zdeddfˆ fd„Z eej                  d«      «       ee	e
¬«      	 	 	 	 	 	 	 	 ddeej                     deej                     d	eej                     d
eej                     deej                     dee   dee   dee   deee	f   fd„«       «       Zˆ xZS )ÚFunnelForPreTrainingro   r‚   Nc                 ó„   •— t         ‰| �  |«       t        |«      | _        t	        |«      | _        | j                  «        y r›   )r‡   rˆ   rå  r�  r“  Údiscriminator_predictionsrÂ  r”   s     €rB   rˆ   zFunnelForPreTraining.__init__J  s3   ø€ Ü‰Ñ˜Ô ä! &Ó)ˆŒÜ)GÈÓ)OˆÔ&à�‰Õr—   rÍ  )rÑ  r¯  r˜   r¬   r­   r™   ÚlabelsrF  rj  rk  c	           	      ó`  — |�|n| j                   j                  }| j                  |||||||¬«      }	|	d   }
| j                  |
«      }d}|�«t	        j
                  «       }|�a|j                  d|
j                  d   «      dk(  }|j                  d|
j                  d   «      |   }||   } |||j                  «       «      }n4 ||j                  d|
j                  d   «      |j                  «       «      }|s|f|	dd z   }|�|f|z   S |S t        |||	j                  |	j                  ¬«      S )a4  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the ELECTRA-style loss. Input should be a sequence of tokens (see `input_ids`
            docstring) Indices should be in `[0, 1]`:

            - 0 indicates the token is an original token,
            - 1 indicates the token was replaced.

        Returns:

        Examples:

        ```python
        >>> from transformers import AutoTokenizer, FunnelForPreTraining
        >>> import torch

        >>> tokenizer = AutoTokenizer.from_pretrained("funnel-transformer/small")
        >>> model = FunnelForPreTraining.from_pretrained("funnel-transformer/small")

        >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="pt")
        >>> logits = model(**inputs).logits
        ```N©r¬   r­   r™   rF  rj  rk  r   rÄ   r   ©r¼  rš  rt  ru  )ro   rÚ  r�  rò  r   r	   rH  rg   rI  r»  rt  ru  )r•   r˜   r¬   r­   r™   ró  rF  rj  rk  r—  Údiscriminator_sequence_outputrš  r¼  Úloss_fctÚactive_lossÚactive_logitsÚactive_labelsr  s                     rB   rœ   zFunnelForPreTraining.forwardR  sj  € ðF &1Ð%<‘kÀ$Ç+Á+×B]ÑB]ˆà&*§k¡kØØ)Ø)Ø'Ø/Ø!5Ø#ð '2ó '
Ð#ð )DÀAÑ(FÐ%à×/Ñ/Ð0MÓNˆàˆØÐÜ×+Ñ+Ó-ˆHØÐ)Ø,×1Ñ1°"Ð6S×6YÑ6YÐZ[Ñ6\Ó]ÐabÑb�Ø &§¡¨BÐ0M×0SÑ0SÐTUÑ0VÓ WÐXcÑ d�Ø & {Ñ 3�Ù ¨}×/BÑ/BÓ/DÓE‘á §¡¨BÐ0M×0SÑ0SÐTUÑ0VÓ WÐY_×YeÑYeÓYgÓh�áØ�YÐ!<¸Q¸RÐ!@Ñ@ˆFØ)-Ð)9�T�G˜fÑ$ÐE¸vÐEä)ØØØ5×CÑCØ2×=Ñ=ô	
ð 	
r—   ©NNNNNNNN)rž   rŸ   r    r   rˆ   r   rá  râ  r   r»  rã  r   rk   r¡   rS  r   r   rœ   r¢   r£   s   @rB   rð  rð  I  s  ø„ ð˜|ð °õ ñ +Ð+B×+IÑ+IÐJgÓ+hÓiÙÐ+EÐTcÔdð -1Ø15Ø15Ø04Ø)-Ø,0Ø/3Ø&*ñD
à˜EŸL™LÑ)ðD
ð ! §¡Ñ.ðD
ð ! §¡Ñ.ð	D
ð
   §¡Ñ-ðD
ð ˜Ÿ™Ñ&ðD
ð $ D™>ðD
ð ' t™nðD
ð ˜d‘^ðD
ð 
ˆuÐ0Ð0Ñ	1òD
ó eó jôD
r—   rð  z@Funnel Transformer Model with a `language modeling` head on top.c                   ó   ‡ — e Zd ZdgZdeddfˆ fd„Zdej                  fd„Zdej                  ddfd„Z
 eej                  d	«      «       eeeed
¬«      	 	 	 	 	 	 	 	 ddeej(                     deej(                     deej(                     deej(                     deej(                     dee   dee   dee   deeef   fd„«       «       Zˆ xZS )ÚFunnelForMaskedLMzlm_head.weightro   r‚   Nc                 óÂ   •— t         ‰| �  |«       t        |«      | _        t	        j
                  |j                  |j                  «      | _        | j                  «        y r›   )
r‡   rˆ   rå  r�  r   r+  rŽ   rŠ   Úlm_headrÂ  r”   s     €rB   rˆ   zFunnelForMaskedLM.__init__Ÿ  sD   ø€ Ü‰Ñ˜Ô ä! &Ó)ˆŒÜ—y‘y §¡°×1BÑ1BÓCˆŒð 	�‰Õr—   c                 ó   — | j                   S r›   ©r   rÅ  s    rB   Úget_output_embeddingsz'FunnelForMaskedLM.get_output_embeddings¨  s   € Ø�|‰|Ðr—   rÈ  c                 ó   — || _         y r›   r  rÊ  s     rB   Úset_output_embeddingsz'FunnelForMaskedLM.set_output_embeddings«  s	   € Ø%ˆ�r—   rÍ  z<mask>)rÐ  rÑ  r¯  Úmaskr˜   r¬   r­   r™   ró  rF  rj  rk  c	           	      ó–  — |�|n| j                   j                  }| j                  |||||||¬«      }	|	d   }
| j                  |
«      }d}|�Ft	        «       } ||j                  d| j                   j                  «      |j                  d«      «      }|s|f|	dd z   }|�|f|z   S |S t        |||	j                  |	j                  ¬«      S )a¢  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
            config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
            loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
        Nrõ  r   rÄ   r   rö  )
ro   rÚ  r�  r   r
   rH  rŠ   r   rt  ru  )r•   r˜   r¬   r­   r™   ró  rF  rj  rk  rí  rs  Úprediction_logitsÚmasked_lm_lossrø  r  s                  rB   rœ   zFunnelForMaskedLM.forward®  sö   € ð0 &1Ð%<‘kÀ$Ç+Á+×B]ÑB]ˆà—+‘+ØØ)Ø)Ø'Ø/Ø!5Ø#ð ó 
ˆð $ A™JÐØ ŸL™LÐ):Ó;ÐàˆØÐÜ'Ó)ˆHÙ%Ð&7×&<Ñ&<¸RÀÇÁ×AWÑAWÓ&XÐZ`×ZeÑZeÐfhÓZiÓjˆNáØ'Ð)¨G°A°B¨KÑ7ˆFØ3AÐ3M�^Ð%¨Ñ.ÐYÐSYÐYäØØ$Ø!×/Ñ/Ø×)Ñ)ô	
ð 	
r—   rü  )rž   rŸ   r    Ú_tied_weights_keysr   rˆ   r   r+  r  r‰   r  r   rá  râ  r   rî  r   rã  r   rk   r¡   rS  r   r   rœ   r¢   r£   s   @rB   rþ  rþ  ›  s;  ø„ à*Ð+Ðð˜|ð °õ ð r§y¡yó ð&°B·L±Lð &ÀTó &ñ +Ð+B×+IÑ+IÐJgÓ+hÓiÙØ&Ø"Ø$Øô	ð -1Ø15Ø15Ø04Ø)-Ø,0Ø/3Ø&*ñ.
à˜EŸL™LÑ)ð.
ð ! §¡Ñ.ð.
ð ! §¡Ñ.ð	.
ð
   §¡Ñ-ð.
ð ˜Ÿ™Ñ&ð.
ð $ D™>ð.
ð ' t™nð.
ð ˜d‘^ð.
ð 
ˆu�nÐ$Ñ	%ò.
óó jô.
r—   rþ  zº
    Funnel Transformer Model with a sequence classification/regression head on top (two linear layer on top of the
    first timestep of the last hidden state) e.g. for GLUE tasks.
    c                   óT  ‡ — e Zd Zdeddfˆ fd„Z eej                  d«      «       ede	e
¬«      	 	 	 	 	 	 	 	 ddeej                     d	eej                     d
eej                     deej                     deej                     dee   dee   dee   deee	f   fd„«       «       Zˆ xZS )ÚFunnelForSequenceClassificationro   r‚   Nc                 óÊ   •— t         ‰| �  |«       |j                  | _        || _        t	        |«      | _        t        ||j                  «      | _        | j                  «        y r›   )	r‡   rˆ   Ú
num_labelsro   r¿  r�  r³  Ú
classifierrÂ  r”   s     €rB   rˆ   z(FunnelForSequenceClassification.__init__î  sN   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒØˆŒä% fÓ-ˆŒÜ2°6¸6×;LÑ;LÓMˆŒà�‰Õr—   rÍ  rÎ  rÏ  r˜   r¬   r­   r™   ró  rF  rj  rk  c	           	      ó,  — |�|n| j                   j                  }| j                  |||||||¬«      }	|	d   }
|
dd…df   }| j                  |«      }d}|��‡| j                   j                  €�| j
                  dk(  rd| j                   _        nl| j
                  dkD  rL|j                  t        j                  k(  s|j                  t        j                  k(  rd| j                   _        nd| j                   _        | j                   j                  dk(  rIt        «       }| j
                  dk(  r& ||j                  «       |j                  «       «      }nŒ |||«      }n‚| j                   j                  dk(  r=t        «       } ||j                  d| j
                  «      |j                  d«      «      }n,| j                   j                  dk(  rt        «       } |||«      }|s|f|	dd z   }|�|f|z   S |S t        |||	j                   |	j"                  ¬	«      S )
a�  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        Nrõ  r   r   Ú
regressionÚsingle_label_classificationÚmulti_label_classificationrÄ   rö  )ro   rÚ  r�  r  Úproblem_typer  r²   rk   ró   r[   r   r™  r
   rH  r	   r   rt  ru  )r•   r˜   r¬   r­   r™   ró  rF  rj  rk  rí  rs  Úpooled_outputrš  r¼  rø  r  s                   rB   rœ   z'FunnelForSequenceClassification.forwardø  sã  € ð. &1Ð%<‘kÀ$Ç+Á+×B]ÑB]ˆà—+‘+ØØ)Ø)Ø'Ø/Ø!5Ø#ð ó 
ˆð $ A™JÐØ)ª!¨Q¨$Ñ/ˆØ—‘ Ó/ˆàˆØÑØ�{‰{×'Ñ'Ð/Ø—?‘? aÒ'Ø/;�D—K‘KÕ,Ø—_‘_ qÒ(¨f¯l©l¼e¿j¹jÒ.HÈFÏLÉLÔ\a×\eÑ\eÒLeØ/L�D—K‘KÕ,à/K�D—K‘KÔ,à�{‰{×'Ñ'¨<Ò7Ü"›9�Ø—?‘? aÒ'Ù# F§N¡NÓ$4°f·n±nÓ6FÓG‘Dá# F¨FÓ3‘DØ—‘×)Ñ)Ð-JÒJÜ+Ó-�Ù §¡¨B°·±Ó @À&Ç+Á+ÈbÃ/ÓR‘Ø—‘×)Ñ)Ð-IÒIÜ,Ó.�Ù ¨Ó/�áØ�Y ¨¨ Ñ,ˆFØ)-Ð)9�T�G˜fÑ$ÐE¸vÐEä'ØØØ!×/Ñ/Ø×)Ñ)ô	
ð 	
r—   rü  )rž   rŸ   r    r   rˆ   r   rá  râ  r   r   rã  r   rk   r¡   rS  r   r   rœ   r¢   r£   s   @rB   r  r  æ  s  ø„ ð˜|ð °õ ñ +Ð+B×+IÑ+IÐJgÓ+hÓiÙØ2Ø,Ø$ôð -1Ø15Ø15Ø04Ø)-Ø,0Ø/3Ø&*ñA
à˜EŸL™LÑ)ðA
ð ! §¡Ñ.ðA
ð ! §¡Ñ.ð	A
ð
   §¡Ñ-ðA
ð ˜Ÿ™Ñ&ðA
ð $ D™>ðA
ð ' t™nðA
ð ˜d‘^ðA
ð 
ˆuÐ.Ð.Ñ	/òA
óó jôA
r—   r  zÐ
    Funnel Transformer Model with a multiple choice classification head on top (two linear layer on top of the first
    timestep of the last hidden state, and a softmax) e.g. for RocStories/SWAG tasks.
    c                   óT  ‡ — e Zd Zdeddfˆ fd„Z eej                  d«      «       ede	e
¬«      	 	 	 	 	 	 	 	 ddeej                     d	eej                     d
eej                     deej                     deej                     dee   dee   dee   deee	f   fd„«       «       Zˆ xZS )ÚFunnelForMultipleChoicero   r‚   Nc                 ó†   •— t         ‰| �  |«       t        |«      | _        t	        |d«      | _        | j                  «        y r  )r‡   rˆ   r¿  r�  r³  r  rÂ  r”   s     €rB   rˆ   z FunnelForMultipleChoice.__init__J  s4   ø€ Ü‰Ñ˜Ô ä% fÓ-ˆŒÜ2°6¸1Ó=ˆŒà�‰Õr—   z(batch_size, num_choices, sequence_lengthrÎ  rÏ  r˜   r¬   r­   r™   ró  rF  rj  rk  c	           	      óî  — |�|n| j                   j                  }|�|j                  d   n|j                  d   }	|�!|j                  d|j	                  d«      «      nd}|�!|j                  d|j	                  d«      «      nd}|�!|j                  d|j	                  d«      «      nd}|�1|j                  d|j	                  d«      |j	                  d«      «      nd}| j                  |||||||¬«      }
|
d   }|dd…df   }| j                  |«      }|j                  d|	«      }d}|�t        «       } |||«      }|s|f|
dd z   }|�|f|z   S |S t        |||
j                  |
j                  ¬«      S )aJ  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the multiple choice classification loss. Indices should be in `[0, ...,
            num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See
            `input_ids` above)
        Nr   rÄ   éþÿÿÿrõ  r   rö  )ro   rÚ  rg   rH  r¯   r�  r  r
   r   rt  ru  )r•   r˜   r¬   r­   r™   ró  rF  rj  rk  Únum_choicesrí  rs  r  rš  Úreshaped_logitsr¼  rø  r  s                     rB   rœ   zFunnelForMultipleChoice.forwardR  s³  € ð. &1Ð%<‘kÀ$Ç+Á+×B]ÑB]ˆØ,5Ð,A�i—o‘o aÒ(À}×GZÑGZÐ[\ÑG]ˆà>GÐ>S�I—N‘N 2 y§~¡~°bÓ'9Ô:ÐY]ˆ	ØM[ÐMg˜×,Ñ,¨R°×1DÑ1DÀRÓ1HÔIÐmqˆØM[ÐMg˜×,Ñ,¨R°×1DÑ1DÀRÓ1HÔIÐmqˆð Ð(ð ×Ñ˜r =×#5Ñ#5°bÓ#9¸=×;MÑ;MÈbÓ;QÔRàð 	ð —+‘+ØØ)Ø)Ø'Ø/Ø!5Ø#ð ó 
ˆð $ A™JÐØ)ª!¨Q¨$Ñ/ˆØ—‘ Ó/ˆØ Ÿ+™+ b¨+Ó6ˆàˆØÐÜ'Ó)ˆHÙ˜O¨VÓ4ˆDáØ%Ð'¨'°!°"¨+Ñ5ˆFØ)-Ð)9�T�G˜fÑ$ÐE¸vÐEä(ØØ"Ø!×/Ñ/Ø×)Ñ)ô	
ð 	
r—   rü  )rž   rŸ   r    r   rˆ   r   rá  râ  r   r   rã  r   rk   r¡   rS  r   r   rœ   r¢   r£   s   @rB   r  r  B  s	  ø„ ð˜|ð °õ ñ +Ð+B×+IÑ+IÐJtÓ+uÓvÙØ2Ø-Ø$ôð -1Ø15Ø15Ø04Ø)-Ø,0Ø/3Ø&*ñ:
à˜EŸL™LÑ)ð:
ð ! §¡Ñ.ð:
ð ! §¡Ñ.ð	:
ð
   §¡Ñ-ð:
ð ˜Ÿ™Ñ&ð:
ð $ D™>ð:
ð ' t™nð:
ð ˜d‘^ð:
ð 
ˆuÐ/Ð/Ñ	0ò:
óó wô:
r—   r  z±
    Funnel Transformer Model with a token classification head on top (a linear layer on top of the hidden-states
    output) e.g. for Named-Entity-Recognition (NER) tasks.
    c                   óT  ‡ — e Zd Zdeddfˆ fd„Z eej                  d«      «       ee	e
e¬«      	 	 	 	 	 	 	 	 ddeej                     deej                     d	eej                     d
eej                     deej                     dee   dee   dee   deee
f   fd„«       «       Zˆ xZS )ÚFunnelForTokenClassificationro   r‚   Nc                 ó,  •— t         ‰| �  |«       |j                  | _        t        |«      | _        t        j                  |j                  «      | _        t        j                  |j                  |j                  «      | _        | j                  «        y r›   )r‡   rˆ   r  rå  r�  r   r‘   r’   r“   r+  r‹   r  rÂ  r”   s     €rB   rˆ   z%FunnelForTokenClassification.__init__�  si   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒä! &Ó)ˆŒÜ—z‘z &×"7Ñ"7Ó8ˆŒÜŸ)™) F×$6Ñ$6¸×8IÑ8IÓJˆŒð 	�‰Õr—   rÍ  rÏ  r˜   r¬   r­   r™   ró  rF  rj  rk  c	           	      ó¤  — |�|n| j                   j                  }| j                  |||||||¬«      }	|	d   }
| j                  |
«      }
| j	                  |
«      }d}|�<t        «       } ||j                  d| j                  «      |j                  d«      «      }|s|f|	dd z   }|�|f|z   S |S t        |||	j                  |	j                  ¬«      S )zÛ
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`.
        Nrõ  r   rÄ   r   rö  )ro   rÚ  r�  r“   r  r
   rH  r  r   rt  ru  )r•   r˜   r¬   r­   r™   ró  rF  rj  rk  rí  rs  rš  r¼  rø  r  s                  rB   rœ   z$FunnelForTokenClassification.forward¨  sô   € ð* &1Ð%<‘kÀ$Ç+Á+×B]ÑB]ˆà—+‘+ØØ)Ø)Ø'Ø/Ø!5Ø#ð ó 
ˆð $ A™JÐØ ŸL™LÐ):Ó;ÐØ—‘Ð!2Ó3ˆàˆØÐÜ'Ó)ˆHÙ˜FŸK™K¨¨D¯O©OÓ<¸f¿k¹kÈ"»oÓNˆDáØ�Y ¨¨ Ñ,ˆFØ)-Ð)9�T�G˜fÑ$ÐE¸vÐEä$ØØØ!×/Ñ/Ø×)Ñ)ô	
ð 	
r—   rü  )rž   rŸ   r    r   rˆ   r   rá  râ  r   rî  r   rã  r   rk   r¡   rS  r   r   rœ   r¢   r£   s   @rB   r  r  •  s	  ø„ ð	˜|ð 	°õ 	ñ +Ð+B×+IÑ+IÐJgÓ+hÓiÙØ&Ø)Ø$ôð -1Ø15Ø15Ø04Ø)-Ø,0Ø/3Ø&*ñ-
à˜EŸL™LÑ)ð-
ð ! §¡Ñ.ð-
ð ! §¡Ñ.ð	-
ð
   §¡Ñ-ð-
ð ˜Ÿ™Ñ&ð-
ð $ D™>ð-
ð ' t™nð-
ð ˜d‘^ð-
ð 
ˆuÐ+Ð+Ñ	,ò-
óó jô-
r—   r  zê
    Funnel Transformer Model with a span classification head on top for extractive question-answering tasks like SQuAD
    (a linear layer on top of the hidden-states output to compute `span start logits` and `span end logits`).
    c                   ót  ‡ — e Zd Zdeddfˆ fd„Z eej                  d«      «       ee	e
e¬«      	 	 	 	 	 	 	 	 	 ddeej                     deej                     d	eej                     d
eej                     deej                     deej                     dee   dee   dee   deee
f   fd„«       «       Zˆ xZS )ÚFunnelForQuestionAnsweringro   r‚   Nc                 óä   •— t         ‰| �  |«       |j                  | _        t        |«      | _        t        j                  |j                  |j                  «      | _        | j                  «        y r›   )
r‡   rˆ   r  rå  r�  r   r+  r‹   Ú
qa_outputsrÂ  r”   s     €rB   rˆ   z#FunnelForQuestionAnswering.__init__æ  sS   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒä! &Ó)ˆŒÜŸ)™) F×$6Ñ$6¸×8IÑ8IÓJˆŒð 	�‰Õr—   rÍ  rÏ  r˜   r¬   r­   r™   Ústart_positionsÚend_positionsrF  rj  rk  c
           	      ó$  — |	�|	n| j                   j                  }	| j                  |||||||	¬«      }
|
d   }| j                  |«      }|j	                  dd¬«      \  }}|j                  d«      j                  «       }|j                  d«      j                  «       }d}|�·|�µt        |j                  «       «      dkD  r|j                  d«      }t        |j                  «       «      dkD  r|j                  d«      }|j                  d«      }|j                  d|«      }|j                  d|«      }t        |¬«      } |||«      } |||«      }||z   dz  }|	s||f|
dd z   }|�|f|z   S |S t        ||||
j                  |
j                  ¬	«      S )
a  
        start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for position (index) of the start of the labelled span for computing the token classification loss.
            Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
            are not taken into account for computing the loss.
        end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for position (index) of the end of the labelled span for computing the token classification loss.
            Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
            are not taken into account for computing the loss.
        Nrõ  r   r   rÄ   rÅ   )Úignore_indexr¦   )r¼  Ústart_logitsÚ
end_logitsrt  ru  )ro   rÚ  r�  r$  rU   r™  Ú
contiguousrh   r¯   ÚsquezeÚclampr
   r   rt  ru  )r•   r˜   r¬   r­   r™   r%  r&  rF  rj  rk  rí  rs  rš  r)  r*  Ú
total_lossÚignored_indexrø  Ú
start_lossÚend_lossr  s                        rB   rœ   z"FunnelForQuestionAnswering.forwardð  s¾  € ð8 &1Ð%<‘kÀ$Ç+Á+×B]ÑB]ˆà—+‘+ØØ)Ø)Ø'Ø/Ø!5Ø#ð ó 
ˆð $ A™JÐà—‘Ð!2Ó3ˆØ#)§<¡<°°r <Ó#:Ñ ˆ�jØ#×+Ñ+¨BÓ/×:Ñ:Ó<ˆØ×'Ñ'¨Ó+×6Ñ6Ó8ˆ
àˆ
ØÐ&¨=Ð+Dä�?×'Ñ'Ó)Ó*¨QÒ.Ø"1×"8Ñ"8¸Ó"<�Ü�=×%Ñ%Ó'Ó(¨1Ò,Ø -× 5Ñ 5°bÓ 9�à(×-Ñ-¨aÓ0ˆMØ-×3Ñ3°A°}ÓEˆOØ)×/Ñ/°°=ÓAˆMä'°]ÔCˆHÙ! ,°Ó@ˆJÙ 
¨MÓ:ˆHØ$ xÑ/°1Ñ4ˆJáØ" JÐ/°'¸!¸"°+Ñ=ˆFØ/9Ð/E�Z�M FÑ*ÐQÈ6ÐQä+ØØ%Ø!Ø!×/Ñ/Ø×)Ñ)ô
ð 	
r—   rà  )rž   rŸ   r    r   rˆ   r   rá  râ  r   rî  r   rã  r   rk   r¡   rS  r   r   rœ   r¢   r£   s   @rB   r"  r"  Þ  s+  ø„ ð˜|ð °õ ñ +Ð+B×+IÑ+IÐJgÓ+hÓiÙØ&Ø0Ø$ôð -1Ø15Ø15Ø04Ø26Ø04Ø,0Ø/3Ø&*ñD
à˜EŸL™LÑ)ðD
ð ! §¡Ñ.ðD
ð ! §¡Ñ.ð	D
ð
   §¡Ñ-ðD
ð " %§,¡,Ñ/ðD
ð   §¡Ñ-ðD
ð $ D™>ðD
ð ' t™nðD
ð ˜d‘^ðD
ð 
ˆuÐ2Ð2Ñ	3òD
óó jôD
r—   r"  )
r¿  rþ  r  rð  r"  r  r  rå  rœ  r   )TF)Hr  rL   Údataclassesr   Útypingr   r   r   r   rG   rq   rk   r   Útorch.nnr	   r
   r   Úactivationsr   Úmodeling_outputsr   r   r   r   r   r   Úmodeling_utilsr   Úutilsr   r   r   r   r   r   Úconfiguration_funnelr   Ú
get_loggerrž   rJ   rã  rî  rJ  r   ÚModuler�   r¥   r¡   r[   r&  rb   rY   r^  rd  rS  rˆ  rŠ  r“  rœ  r³  r»  ÚFUNNEL_START_DOCSTRINGrá  r¿  rå  rð  rþ  r  r  r  r"  Ú__all__r?   r—   rB   ú<module>r>     s'  ðñ (ã 	Ý !ß /Ó /ã Û Ý ß AÑ Aå !÷÷ õ .÷÷ õ /ð 
ˆ×	Ñ	˜HÓ	%€à €Ø0Ð ð 
€òWôt�r—y‘yô ô"A ˜rŸy™yô A ðH¨E¯L©Lð Àsð ÐSVð Ð[`×[gÑ[gó ô MG "§)¡)ô MGô`+˜BŸI™Iô +ô&E�"—)‘)ô Eô&<u�B—I‘Iô <uð@ diñØ‡|�|ðØ ðØ.1ðØAEðØ\`ðà
‡\�\óô,.u�B—I‘Iô .uôb R§Y¡Yô ô  _˜Oô  _ôF'˜rŸy™yô 'ð ô: ó :ó ð:ð8Ð ð&$Ð ñN ðð óôEÐ+ó EóðEñP ØrØóô_
Ð'ó _
ó	ð_
ñD ðð ôôO
Ð0ô O
ñd Ð\Ð^tÓuôG
Ð-ó G
ó vðG
ñT ðð óôR
Ð&;ó R
óðR
ñj ðð óôI
Ð3ó I
óðI
ñX ðð óô?
Ð#8ó ?
óð?
ñD ðð óôU
Ð!6ó U
óðU
òp�r—   