Ë
    T^(h³ ã                   óZ  — d dl Z d dlZd dlZd dlmZ d dlmZmZmZ d dl	Z	d dl	m
Z
 d dlmZmZmZ ddlmZ ddlmZmZmZmZmZmZmZmZ dd	lmZ dd
lmZmZ ddlm Z m!Z!m"Z"m#Z#m$Z$m%Z% ddl&m'Z'  e$jP                  e)«      Z*dZ+dZ,dZ-dZ.dZ/dZ0dZ1dZ2dZ3dZ4dZ5dZ6dZ7d„ Z8 G d„ de
jr                  «      Z:e
jv                  e:dœZ< G d„ d e
jr                  «      Z= G d!„ d"e
jr                  «      Z> G d#„ d$e
jr                  «      Z? G d%„ d&e
jr                  «      Z@ G d'„ d(e
jr                  «      ZA G d)„ d*e
jr                  «      ZB G d+„ d,e
jr                  «      ZC G d-„ d.e
jr                  «      ZD G d/„ d0e
jr                  «      ZE G d1„ d2e
jr                  «      ZF G d3„ d4e
jr                  «      ZG G d5„ d6e
jr                  «      ZH G d7„ d8e
jr                  «      ZI G d9„ d:e
jr                  «      ZJ G d;„ d<e
jr                  «      ZK G d=„ d>e
jr                  «      ZL G d?„ d@e
jr                  «      ZM G dA„ dBe
jr                  «      ZN G dC„ dDe«      ZOe G dE„ dFe «      «       ZPdGZQdHZR e"dIeQ«       G dJ„ dKeO«      «       ZS e"dLeQ«       G dM„ dNeO«      «       ZT e"dOeQ«       G dP„ dQeO«      «       ZU G dR„ dSe
jr                  «      ZV e"dTeQ«       G dU„ dVeO«      «       ZW e"dWeQ«       G dX„ dYeO«      «       ZX e"dZeQ«       G d[„ d\eO«      «       ZY e"d]eQ«       G d^„ d_eO«      «       ZZ e"d`eQ«       G da„ dbeO«      «       Z[g dc¢Z\y)dé    N)Ú	dataclass)ÚOptionalÚTupleÚUnion)Únn)ÚBCEWithLogitsLossÚCrossEntropyLossÚMSELossé   )ÚACT2FN)ÚBaseModelOutputÚBaseModelOutputWithPoolingÚMaskedLMOutputÚMultipleChoiceModelOutputÚNextSentencePredictorOutputÚQuestionAnsweringModelOutputÚSequenceClassifierOutputÚTokenClassifierOutput)ÚPreTrainedModel)Ú find_pruneable_heads_and_indicesÚprune_linear_layer)ÚModelOutputÚadd_code_sample_docstringsÚadd_start_docstringsÚ%add_start_docstrings_to_model_forwardÚloggingÚreplace_return_docstringsé   )ÚMobileBertConfigzgoogle/mobilebert-uncasedr   z mrm8488/mobilebert-finetuned-nerzK['I-ORG', 'I-ORG', 'O', 'O', 'O', 'O', 'O', 'I-LOC', 'O', 'I-LOC', 'I-LOC']g¸…ëQ¸ž?z#csarron/mobilebert-uncased-squad-v2z'a nice puppet'g×£p=
×@é   é   zlordtt13/emo-mobilebertz'others'z4.72c           	      óÔ  — 	 ddl }ddl}ddl}t        j                  j                  |«      }t        j                  d|› �«       |j                  j                  |«      }g }g }	|D ]^  \  }
}t        j                  d|
› d|› �«       |j                  j                  ||
«      }|j                  |
«       |	j                  |«       Œ` t        ||	«      D �]ú  \  }
}|
j                  dd«      }
|
j                  d	d
«      }
|
j                  dd«      }
|
j                  dd«      }
|
j!                  d«      }
t#        d„ |
D «       «      r(t        j                  ddj%                  |
«      › �«       Œš| }|
D ]À  }|j'                  d|«      r|j!                  d|«      }n|g}|d   dk(  s|d   dk(  rt)        |d«      }nW|d   dk(  s|d   dk(  rt)        |d«      }n:|d   dk(  rt)        |d«      }n%|d   dk(  rt)        |d«      }n	 t)        ||d   «      }t-        |«      dk\  sŒ®t/        |d   «      }||   }ŒÂ dd d k(  rt)        |d«      }n|dk(  r|j1                  |«      }	 |j2                  |j2                  k(  s"J d!|j2                  › d"|j2                  › d#�«       ‚	 t        j                  d$|
› �«       t9        j:                  |«      |_        �Œý | S # t        $ r t        j                  d«       ‚ w xY w# t*        $ r+ t        j                  ddj%                  |
«      › �«       Y �Œ¸w xY w# t4        $ r1}|xj6                  |j2                  |j2                  fz  c_        ‚ d}~ww xY w)%z'Load tf checkpoints in a pytorch model.r   Nz™Loading a TensorFlow model in PyTorch, requires TensorFlow to be installed. Please see https://www.tensorflow.org/install/ for installation instructions.z&Converting TensorFlow checkpoint from zLoading TF weight z with shape Ú	ffn_layerÚffnÚFakeLayerNormÚ	LayerNormÚextra_output_weightszdense/kernelÚbertÚ
mobilebertú/c              3   ó$   K  — | ]  }|d v –— Œ
 y­w))Úadam_vÚadam_mÚAdamWeightDecayOptimizerÚAdamWeightDecayOptimizer_1Úglobal_stepN© )Ú.0Úns     úp/var/www/skyplay_api_hub/venv/lib/python3.12/site-packages/transformers/models/mobilebert/modeling_mobilebert.pyú	<genexpr>z0load_tf_weights_in_mobilebert.<locals>.<genexpr>q   s   è ø€ ò 
àð ÐnÔnñ
ùs   ‚z	Skipping z[A-Za-z]+_\d+z_(\d+)ÚkernelÚgammaÚweightÚoutput_biasÚbetaÚbiasÚoutput_weightsÚsquadÚ
classifieré   r   iõÿÿÿÚ_embeddingszPointer shape z and array shape z mismatchedzInitialize PyTorch weight )ÚreÚnumpyÚ
tensorflowÚImportErrorÚloggerÚerrorÚosÚpathÚabspathÚinfoÚtrainÚlist_variablesÚload_variableÚappendÚzipÚreplaceÚsplitÚanyÚjoinÚ	fullmatchÚgetattrÚAttributeErrorÚlenÚintÚ	transposeÚshapeÚAssertionErrorÚargsÚtorchÚ
from_numpyÚdata)ÚmodelÚconfigÚtf_checkpoint_pathrA   ÚnpÚtfÚtf_pathÚ	init_varsÚnamesÚarraysÚnamerZ   ÚarrayÚpointerÚm_nameÚscope_namesÚnumÚes                     r4   Úload_tf_weights_in_mobilebertrp   P   sh  € ð
ÛãÛô �g‰g�o‰oÐ0Ó1€GÜ
‡K�KÐ8¸¸	ÐBÔCà—‘×'Ñ'¨Ó0€IØ€EØ€FØ ò ‰ˆˆeÜ�‰Ð(¨¨¨l¸5¸'ÐBÔCØ—‘×&Ñ& w°Ó5ˆØ�‰�TÔØ�‰�eÕð	ô ˜5 &Ó)ó 1/‰ˆˆeØ�|‰|˜K¨Ó/ˆØ�|‰|˜O¨[Ó9ˆØ�|‰|Ð2°NÓCˆØ�|‰|˜F LÓ1ˆØ�z‰z˜#‹ˆô ñ 
àô
ô 
ô �K‰K˜) C§H¡H¨T£NÐ#3Ð4Ô5ØØˆØò 	'ˆFØ�|‰|Ð,¨fÔ5Ø Ÿh™h y°&Ó9‘à%˜h�Ø˜1‰~ Ò)¨[¸©^¸wÒ-FÜ! '¨8Ó4‘Ø˜Q‘ =Ò0°KÀ±NÀfÒ4LÜ! '¨6Ó2‘Ø˜Q‘Ð#3Ò3Ü! '¨8Ó4‘Ø˜Q‘ 7Ò*Ü! '¨<Ó8‘ðÜ% g¨{¸1©~Ó>�Gô �;Ó 1Ó$Ü˜+ a™.Ó)�Ø! #™,‘ð+	'ð, �#�$ˆ<˜=Ò(Ü˜g xÓ0‰GØ�xÒØ—L‘L Ó'ˆEð	Ø—=‘= E§K¡KÒ/ð Ø  §¡ Ð/@ÀÇÁÀÈ[ÐYóÑ/ô 	�‰Ð0°°Ð7Ô8Ü×'Ñ'¨Ó.ˆŽðc1/ðd €LøôI ò Ü�‰ðQô	
ð 	ðûôb &ò Ü—K‘K )¨C¯H©H°T«NÐ+;Ð <Ô=Úðûô ò 	Ø�FŠF�w—}‘} e§k¡kÐ2Ñ2�FØûð	ús5   ‚K ÈK6É ;L-Ë K3Ë60L*Ì)L*Ì-	M'Ì6,M"Í"M'c                   óX   ‡ — e Zd Zdˆ fd„	Zdej
                  dej
                  fd„Zˆ xZS )ÚNoNormc                 óÖ   •— t         ‰| �  «        t        j                  t	        j
                  |«      «      | _        t        j                  t	        j                  |«      «      | _        y ©N)	ÚsuperÚ__init__r   Ú	Parameterr]   Úzerosr;   Úonesr8   )ÚselfÚ	feat_sizeÚepsÚ	__class__s      €r4   rv   zNoNorm.__init__Ÿ   s@   ø€ Ü‰ÑÔÜ—L‘L¤§¡¨YÓ!7Ó8ˆŒ	Ü—l‘l¤5§:¡:¨iÓ#8Ó9ˆ�ó    Úinput_tensorÚreturnc                 ó:   — || j                   z  | j                  z   S rt   )r8   r;   )rz   r   s     r4   ÚforwardzNoNorm.forward¤   s   € Ø˜dŸk™kÑ)¨D¯I©IÑ5Ð5r~   rt   ©Ú__name__Ú
__module__Ú__qualname__rv   r]   ÚTensorr‚   Ú__classcell__©r}   s   @r4   rr   rr   ž   s#   ø„ õ:ð
6 E§L¡Lð 6°U·\±\÷ 6r~   rr   )Ú
layer_normÚno_normc                   óÄ   ‡ — e Zd ZdZˆ fd„Z	 	 	 	 d	deej                     deej                     deej                     deej                     dej                  f
d„Z
ˆ xZS )
ÚMobileBertEmbeddingszGConstruct the embeddings from word, position and token_type embeddings.c                 óX  •— t         ‰| �  «        |j                  | _        |j                  | _        |j                  | _        t        j                  |j                  |j                  |j                  ¬«      | _	        t        j                  |j                  |j                  «      | _        t        j                  |j                  |j                  «      | _        | j                  rdnd}| j                  |z  }t        j                  ||j                  «      | _        t!        |j"                     |j                  «      | _        t        j&                  |j(                  «      | _        | j-                  dt/        j0                  |j                  «      j3                  d«      d¬«       y )N)Úpadding_idxr   r   Úposition_ids)r   éÿÿÿÿF)Ú
persistent)ru   rv   Útrigram_inputÚembedding_sizeÚhidden_sizer   Ú	EmbeddingÚ
vocab_sizeÚpad_token_idÚword_embeddingsÚmax_position_embeddingsÚposition_embeddingsÚtype_vocab_sizeÚtoken_type_embeddingsÚLinearÚembedding_transformationÚNORM2FNÚnormalization_typer&   ÚDropoutÚhidden_dropout_probÚdropoutÚregister_bufferr]   ÚarangeÚexpand)rz   ra   Úembed_dim_multiplierÚembedded_input_sizer}   s       €r4   rv   zMobileBertEmbeddings.__init__®   sF  ø€ Ü‰ÑÔØ#×1Ñ1ˆÔØ$×3Ñ3ˆÔØ!×-Ñ-ˆÔä!Ÿ|™|¨F×,=Ñ,=¸v×?TÑ?TÐbh×buÑbuÔvˆÔÜ#%§<¡<°×0NÑ0NÐPV×PbÑPbÓ#cˆÔ Ü%'§\¡\°&×2HÑ2HÈ&×J\ÑJ\Ó%]ˆÔ"à$(×$6Ò$6™q¸AÐØ"×1Ñ1Ð4HÑHÐÜ(*¯	©	Ð2EÀv×GYÑGYÓ(ZˆÔ%ä  ×!:Ñ!:Ñ;¸F×<NÑ<NÓOˆŒÜ—z‘z &×"<Ñ"<Ó=ˆŒð 	×ÑØœEŸL™L¨×)GÑ)GÓH×OÑOÐPWÓXÐejð 	õ 	
r~   Ú	input_idsÚtoken_type_idsr�   Úinputs_embedsr€   c           
      ó$  — |�|j                  «       }n|j                  «       d d }|d   }|€| j                  d d …d |…f   }|€:t        j                  |t        j                  | j                  j
                  ¬«      }|€| j                  |«      }| j                  rpt        j                  t        j                  j                  |d d …dd …f   g d¢d¬«      |t        j                  j                  |d d …d d…f   g d¢d¬«      gd¬	«      }| j                  s| j                  | j                  k7  r| j                  |«      }| j                  |«      }| j!                  |«      }||z   |z   }	| j#                  |	«      }	| j%                  |	«      }	|	S )
Nr‘   r   ©ÚdtypeÚdevice)r   r   r   r   r   r   ç        )Úvalue)r   r   r   r   r   r   r?   ©Údim)Úsizer�   r]   rx   Úlongr°   r™   r“   Úcatr   Ú
functionalÚpadr”   r•   rŸ   r›   r�   r&   r¤   )
rz   rª   r«   r�   r¬   Úinput_shapeÚ
seq_lengthr›   r�   Ú
embeddingss
             r4   r‚   zMobileBertEmbeddings.forwardÄ   s‰  € ð Ð Ø#Ÿ.™.Ó*‰Kà'×,Ñ,Ó.¨s°Ð3ˆKà  ‘^ˆ
àÐØ×,Ñ,ªQ°°°¨^Ñ<ˆLàÐ!Ü"Ÿ[™[¨¼E¿J¹JÈt×O`ÑO`×OgÑOgÔhˆNØÐ Ø ×0Ñ0°Ó;ˆMà×Òô "ŸI™Iä—M‘M×%Ñ% m²A°q±r°EÑ&:Ò<NÐVYÐ%ÓZØ!Ü—M‘M×%Ñ% m²A°s¸°s°FÑ&;Ò=OÐWZÐ%Ó[ðð
 ôˆMð ×Ò ×!4Ñ!4¸×8HÑ8HÒ!HØ ×9Ñ9¸-ÓHˆMð #×6Ñ6°|ÓDÐØ $× :Ñ :¸>Ó JÐØ"Ð%8Ñ8Ð;PÑPˆ
Ø—^‘^ JÓ/ˆ
Ø—\‘\ *Ó-ˆ
ØÐr~   )NNNN)r„   r…   r†   Ú__doc__rv   r   r]   Ú
LongTensorÚFloatTensorr‡   r‚   rˆ   r‰   s   @r4   r�   r�   «   s~   ø„ ÙQô
ð0 15Ø59Ø37Ø59ñ0à˜E×,Ñ,Ñ-ð0ð ! ×!1Ñ!1Ñ2ð0ð ˜u×/Ñ/Ñ0ð	0ð
   × 1Ñ 1Ñ2ð0ð 
�‰÷0r~   r�   c                   óà   ‡ — e Zd Zˆ fd„Zd„ Z	 	 	 ddej                  dej                  dej                  deej                     deej                     dee	   d	e
ej                     fd
„Zˆ xZS )ÚMobileBertSelfAttentionc                 ó`  •— t         ‰| �  «        |j                  | _        t        |j                  |j                  z  «      | _        | j                  | j
                  z  | _        t        j                  |j                  | j                  «      | _	        t        j                  |j                  | j                  «      | _
        t        j                  |j                  r|j                  n|j                  | j                  «      | _        t        j                  |j                  «      | _        y rt   )ru   rv   Únum_attention_headsrX   Útrue_hidden_sizeÚattention_head_sizeÚall_head_sizer   rž   ÚqueryÚkeyÚuse_bottleneck_attentionr•   r²   r¢   Úattention_probs_dropout_probr¤   ©rz   ra   r}   s     €r4   rv   z MobileBertSelfAttention.__init__ø   sÙ   ø€ Ü‰ÑÔØ#)×#=Ñ#=ˆÔ Ü#& v×'>Ñ'>À×A[ÑA[Ñ'[Ó#\ˆÔ Ø!×5Ñ5¸×8PÑ8PÑPˆÔä—Y‘Y˜v×6Ñ6¸×8JÑ8JÓKˆŒ
Ü—9‘9˜V×4Ñ4°d×6HÑ6HÓIˆŒÜ—Y‘YØ'-×'FÒ'FˆF×#Ò#ÈF×L^ÑL^Ð`d×`rÑ`ró
ˆŒ
ô —z‘z &×"EÑ"EÓFˆ�r~   c                 ó¤   — |j                  «       d d | j                  | j                  fz   }|j                  |«      }|j	                  dddd«      S )Nr‘   r   r?   r   r   )rµ   rÃ   rÅ   ÚviewÚpermute)rz   ÚxÚnew_x_shapes      r4   Útranspose_for_scoresz,MobileBertSelfAttention.transpose_for_scores  sL   € Ø—f‘f“h˜s �m t×'?Ñ'?À×AYÑAYÐ&ZÑZˆØ�F‰F�;ÓˆØ�y‰y˜˜A˜q !Ó$Ð$r~   Úquery_tensorÚ
key_tensorÚvalue_tensorÚattention_maskÚ	head_maskÚoutput_attentionsr€   c                 óÌ  — | j                  |«      }| j                  |«      }| j                  |«      }	| j                  |«      }
| j                  |«      }| j                  |	«      }t	        j
                  |
|j                  dd«      «      }|t        j                  | j                  «      z  }|�||z   }t        j                  j                  |d¬«      }| j                  |«      }|�||z  }t	        j
                  ||«      }|j                  dddd«      j                  «       }|j!                  «       d d | j"                  fz   }|j%                  |«      }|r||f}|S |f}|S )Nr‘   éþÿÿÿr³   r   r?   r   r   )rÇ   rÈ   r²   rÑ   r]   ÚmatmulrY   ÚmathÚsqrtrÅ   r   r¸   Úsoftmaxr¤   rÎ   Ú
contiguousrµ   rÆ   rÍ   )rz   rÒ   rÓ   rÔ   rÕ   rÖ   r×   Úmixed_query_layerÚmixed_key_layerÚmixed_value_layerÚquery_layerÚ	key_layerÚvalue_layerÚattention_scoresÚattention_probsÚcontext_layerÚnew_context_layer_shapeÚoutputss                     r4   r‚   zMobileBertSelfAttention.forward
  sm  € ð !ŸJ™J |Ó4ÐØŸ(™( :Ó.ˆØ ŸJ™J |Ó4Ðà×/Ñ/Ð0AÓBˆØ×-Ñ-¨oÓ>ˆ	Ø×/Ñ/Ð0AÓBˆô !Ÿ<™<¨°Y×5HÑ5HÈÈRÓ5PÓQÐØ+¬d¯i©i¸×8PÑ8PÓ.QÑQÐØÐ%à/°.Ñ@ÐäŸ-™-×/Ñ/Ð0@ÀbÐ/ÓIˆð Ÿ,™, Ó7ˆàÐ Ø-°	Ñ9ˆOÜŸ™ _°kÓBˆØ%×-Ñ-¨a°°A°qÓ9×DÑDÓFˆØ"/×"4Ñ"4Ó"6°s¸Ð";¸t×?QÑ?QÐ>SÑ"SÐØ%×*Ñ*Ð+BÓCˆÙ6G�= /Ð2ˆØˆð O\ÐM]ˆØˆr~   ©NNN)r„   r…   r†   rv   rÑ   r]   r‡   r   r¿   Úboolr   r‚   rˆ   r‰   s   @r4   rÁ   rÁ   ÷   s‘   ø„ ôGò%ð 7;Ø15Ø,0ñ$à—l‘lð$ð —L‘Lð$ð —l‘lð	$ð
 ! ×!2Ñ!2Ñ3ð$ð ˜E×-Ñ-Ñ.ð$ð $ D™>ð$ð 
ˆu�|‰|Ñ	÷$r~   rÁ   c                   ón   ‡ — e Zd Zˆ fd„Zdej
                  dej
                  dej
                  fd„Zˆ xZS )ÚMobileBertSelfOutputc                 ój  •— t         ‰| �  «        |j                  | _        t        j                  |j
                  |j
                  «      | _        t        |j                     |j
                  |j                  ¬«      | _
        | j                  s%t        j                  |j                  «      | _        y y ©N©r|   )ru   rv   Úuse_bottleneckr   rž   rÄ   Údenser    r¡   Úlayer_norm_epsr&   r¢   r£   r¤   rË   s     €r4   rv   zMobileBertSelfOutput.__init__2  s„   ø€ Ü‰ÑÔØ$×3Ñ3ˆÔÜ—Y‘Y˜v×6Ñ6¸×8OÑ8OÓPˆŒ
Ü  ×!:Ñ!:Ñ;¸F×<SÑ<SÐY_×YnÑYnÔoˆŒØ×"Ò"ÜŸ:™: f×&@Ñ&@ÓAˆD�Lð #r~   Úhidden_statesÚresidual_tensorr€   c                 óŠ   — | j                  |«      }| j                  s| j                  |«      }| j                  ||z   «      }|S rt   )rò   rñ   r¤   r&   ©rz   rô   rõ   Úlayer_outputss       r4   r‚   zMobileBertSelfOutput.forward:  s@   € ØŸ
™
 =Ó1ˆØ×"Ò"Ø ŸL™L¨Ó7ˆMØŸ™ }°Ñ'FÓGˆØÐr~   rƒ   r‰   s   @r4   rí   rí   1  s2   ø„ ôBð U§\¡\ð ÀEÇLÁLð ÐUZ×UaÑUa÷ r~   rí   c                   óø   ‡ — e Zd Zˆ fd„Zd„ Z	 	 	 ddej                  dej                  dej                  dej                  deej                     deej                     d	ee	   d
e
ej                     fd„Zˆ xZS )ÚMobileBertAttentionc                 ó€   •— t         ‰| �  «        t        |«      | _        t	        |«      | _        t        «       | _        y rt   )ru   rv   rÁ   rz   rí   ÚoutputÚsetÚpruned_headsrË   s     €r4   rv   zMobileBertAttention.__init__C  s0   ø€ Ü‰ÑÔÜ+¨FÓ3ˆŒ	Ü*¨6Ó2ˆŒÜ›EˆÕr~   c                 ó>  — t        |«      dk(  ry t        || j                  j                  | j                  j                  | j
                  «      \  }}t        | j                  j                  |«      | j                  _        t        | j                  j                  |«      | j                  _        t        | j                  j                  |«      | j                  _	        t        | j                  j                  |d¬«      | j                  _        | j                  j                  t        |«      z
  | j                  _        | j                  j                  | j                  j                  z  | j                  _        | j
                  j                  |«      | _        y )Nr   r   r³   )rW   r   rz   rÃ   rÅ   rþ   r   rÇ   rÈ   r²   rü   rò   rÆ   Úunion)rz   ÚheadsÚindexs      r4   Úprune_headszMobileBertAttention.prune_headsI  s  € Üˆu‹:˜Š?ØÜ7Ø�4—9‘9×0Ñ0°$·)±)×2OÑ2OÐQU×QbÑQbó
‰ˆˆuô
 -¨T¯Y©Y¯_©_¸eÓDˆ�	‰	ŒÜ*¨4¯9©9¯=©=¸%Ó@ˆ�	‰	ŒÜ,¨T¯Y©Y¯_©_¸eÓDˆ�	‰	ŒÜ.¨t¯{©{×/@Ñ/@À%ÈQÔOˆ�‰Ôð )-¯	©	×(EÑ(EÌÈEË
Ñ(Rˆ�	‰	Ô%Ø"&§)¡)×"?Ñ"?À$Ç)Á)×B_ÑB_Ñ"_ˆ�	‰	ÔØ ×-Ñ-×3Ñ3°EÓ:ˆÕr~   rÒ   rÓ   rÔ   Úlayer_inputrÕ   rÖ   r×   r€   c                 ón   — | j                  ||||||«      }| j                  |d   |«      }	|	f|dd  z   }
|
S )Nr   r   )rz   rü   )rz   rÒ   rÓ   rÔ   r  rÕ   rÖ   r×   Úself_outputsÚattention_outputré   s              r4   r‚   zMobileBertAttention.forward[  sT   € ð —y‘yØØØØØØó
ˆð  Ÿ;™; |°A¡¸ÓDÐØ#Ð%¨°Q°RÐ(8Ñ8ˆØˆr~   rê   )r„   r…   r†   rv   r  r]   r‡   r   r¿   rë   r   r‚   rˆ   r‰   s   @r4   rú   rú   B  sž   ø„ ô"ò;ð0 7;Ø15Ø,0ñà—l‘lðð —L‘Lðð —l‘lð	ð
 —\‘\ðð ! ×!2Ñ!2Ñ3ðð ˜E×-Ñ-Ñ.ðð $ D™>ðð 
ˆu�|‰|Ñ	÷r~   rú   c                   óV   ‡ — e Zd Zˆ fd„Zdej
                  dej
                  fd„Zˆ xZS )ÚMobileBertIntermediatec                 ó  •— t         ‰| �  «        t        j                  |j                  |j
                  «      | _        t        |j                  t        «      rt        |j                     | _        y |j                  | _        y rt   )ru   rv   r   rž   rÄ   Úintermediate_sizerò   Ú
isinstanceÚ
hidden_actÚstrr   Úintermediate_act_fnrË   s     €r4   rv   zMobileBertIntermediate.__init__u  s]   ø€ Ü‰ÑÔÜ—Y‘Y˜v×6Ñ6¸×8PÑ8PÓQˆŒ
Ü�f×'Ñ'¬Ô-Ü'-¨f×.?Ñ.?Ñ'@ˆDÕ$à'-×'8Ñ'8ˆDÕ$r~   rô   r€   c                 óJ   — | j                  |«      }| j                  |«      }|S rt   )rò   r  ©rz   rô   s     r4   r‚   zMobileBertIntermediate.forward}  s&   € ØŸ
™
 =Ó1ˆØ×0Ñ0°Ó?ˆØÐr~   rƒ   r‰   s   @r4   r	  r	  t  s#   ø„ ô9ð U§\¡\ð °e·l±l÷ r~   r	  c                   ón   ‡ — e Zd Zˆ fd„Zdej
                  dej
                  dej
                  fd„Zˆ xZS )ÚOutputBottleneckc                 ó.  •— t         ‰| �  «        t        j                  |j                  |j
                  «      | _        t        |j                     |j
                  |j                  ¬«      | _
        t        j                  |j                  «      | _        y rï   )ru   rv   r   rž   rÄ   r•   rò   r    r¡   ró   r&   r¢   r£   r¤   rË   s     €r4   rv   zOutputBottleneck.__init__„  sh   ø€ Ü‰ÑÔÜ—Y‘Y˜v×6Ñ6¸×8JÑ8JÓKˆŒ
Ü  ×!:Ñ!:Ñ;¸F×<NÑ<NÐTZ×TiÑTiÔjˆŒÜ—z‘z &×"<Ñ"<Ó=ˆ�r~   rô   rõ   r€   c                 ór   — | j                  |«      }| j                  |«      }| j                  ||z   «      }|S rt   )rò   r¤   r&   r÷   s       r4   r‚   zOutputBottleneck.forwardŠ  s7   € ØŸ
™
 =Ó1ˆØŸ™ ]Ó3ˆØŸ™ }°Ñ'FÓGˆØÐr~   rƒ   r‰   s   @r4   r  r  ƒ  s1   ø„ ô>ð U§\¡\ð ÀEÇLÁLð ÐUZ×UaÑUa÷ r~   r  c                   ó†   ‡ — e Zd Zˆ fd„Zdej
                  dej
                  dej
                  dej
                  fd„Zˆ xZS )ÚMobileBertOutputc                 ór  •— t         ‰| �  «        |j                  | _        t        j                  |j
                  |j                  «      | _        t        |j                     |j                  «      | _
        | j                  s%t        j                  |j                  «      | _        y t        |«      | _        y rt   )ru   rv   rñ   r   rž   r  rÄ   rò   r    r¡   r&   r¢   r£   r¤   r  Ú
bottleneckrË   s     €r4   rv   zMobileBertOutput.__init__’  s‚   ø€ Ü‰ÑÔØ$×3Ñ3ˆÔÜ—Y‘Y˜v×7Ñ7¸×9PÑ9PÓQˆŒ
Ü  ×!:Ñ!:Ñ;¸F×<SÑ<SÓTˆŒØ×"Ò"ÜŸ:™: f×&@Ñ&@ÓAˆD�Lä.¨vÓ6ˆD�Or~   Úintermediate_statesÚresidual_tensor_1Úresidual_tensor_2r€   c                 óÚ   — | j                  |«      }| j                  s'| j                  |«      }| j                  ||z   «      }|S | j                  ||z   «      }| j	                  ||«      }|S rt   )rò   rñ   r¤   r&   r  )rz   r  r  r  Úlayer_outputs        r4   r‚   zMobileBertOutput.forwardœ  ss   € ð —z‘zÐ"5Ó6ˆØ×"Ò"ØŸ<™<¨Ó5ˆLØŸ>™>¨,Ð9JÑ*JÓKˆLð Ðð  Ÿ>™>¨,Ð9JÑ*JÓKˆLØŸ?™?¨<Ð9JÓKˆLØÐr~   rƒ   r‰   s   @r4   r  r  ‘  s?   ø„ ô7ð
Ø#(§<¡<ð
ØDIÇLÁLð
Øej×eqÑeqð
à	�‰÷
r~   r  c                   óV   ‡ — e Zd Zˆ fd„Zdej
                  dej
                  fd„Zˆ xZS )ÚBottleneckLayerc                 óæ   •— t         ‰| �  «        t        j                  |j                  |j
                  «      | _        t        |j                     |j
                  |j                  ¬«      | _
        y rï   )ru   rv   r   rž   r•   Úintra_bottleneck_sizerò   r    r¡   ró   r&   rË   s     €r4   rv   zBottleneckLayer.__init__ª  sR   ø€ Ü‰ÑÔÜ—Y‘Y˜v×1Ñ1°6×3OÑ3OÓPˆŒ
Ü  ×!:Ñ!:Ñ;¸F×<XÑ<XÐ^d×^sÑ^sÔtˆ�r~   rô   r€   c                 óJ   — | j                  |«      }| j                  |«      }|S rt   ©rò   r&   )rz   rô   r  s      r4   r‚   zBottleneckLayer.forward¯  s$   € Ø—j‘j Ó/ˆØ—n‘n [Ó1ˆØÐr~   rƒ   r‰   s   @r4   r   r   ©  s$   ø„ ôuð
 U§\¡\ð °e·l±l÷ r~   r   c                   ó\   ‡ — e Zd Zˆ fd„Zdej
                  deej
                     fd„Zˆ xZS )Ú
Bottleneckc                 óÀ   •— t         ‰| �  «        |j                  | _        |j                  | _        t	        |«      | _        | j                  rt	        |«      | _        y y rt   )ru   rv   Úkey_query_shared_bottleneckrÉ   r   ÚinputÚ	attentionrË   s     €r4   rv   zBottleneck.__init__¶  sP   ø€ Ü‰ÑÔØ+1×+MÑ+MˆÔ(Ø(.×(GÑ(GˆÔ%Ü$ VÓ,ˆŒ
Ø×+Ò+Ü,¨VÓ4ˆD�Nð ,r~   rô   r€   c                 óš   — | j                  |«      }| j                  r|fdz  S | j                  r| j                  |«      }||||fS ||||fS )Né   )r)  rÉ   r(  r*  )rz   rô   Úbottlenecked_hidden_statesÚshared_attention_inputs       r4   r‚   zBottleneck.forward¾  sc   € ð" &*§Z¡Z°Ó%>Ð"Ø×(Ò(Ø.Ð0°1Ñ4Ð4Ø×-Ò-Ø%)§^¡^°MÓ%BÐ"Ø*Ð,BÀMÐSmÐnÐnà! =°-ÐA[Ð\Ð\r~   ©	r„   r…   r†   rv   r]   r‡   r   r‚   rˆ   r‰   s   @r4   r&  r&  µ  s+   ø„ ô5ð] U§\¡\ð ]°e¸E¿L¹LÑ6I÷ ]r~   r&  c                   ón   ‡ — e Zd Zˆ fd„Zdej
                  dej
                  dej
                  fd„Zˆ xZS )Ú	FFNOutputc                 óæ   •— t         ‰| �  «        t        j                  |j                  |j
                  «      | _        t        |j                     |j
                  |j                  ¬«      | _
        y rï   )ru   rv   r   rž   r  rÄ   rò   r    r¡   ró   r&   rË   s     €r4   rv   zFFNOutput.__init__Ú  sR   ø€ Ü‰ÑÔÜ—Y‘Y˜v×7Ñ7¸×9PÑ9PÓQˆŒ
Ü  ×!:Ñ!:Ñ;¸F×<SÑ<SÐY_×YnÑYnÔoˆ�r~   rô   rõ   r€   c                 óP   — | j                  |«      }| j                  ||z   «      }|S rt   r$  r÷   s       r4   r‚   zFFNOutput.forwardß  s)   € ØŸ
™
 =Ó1ˆØŸ™ }°Ñ'FÓGˆØÐr~   rƒ   r‰   s   @r4   r1  r1  Ù  s2   ø„ ôpð
 U§\¡\ð ÀEÇLÁLð ÐUZ×UaÑUa÷ r~   r1  c                   óV   ‡ — e Zd Zˆ fd„Zdej
                  dej
                  fd„Zˆ xZS )ÚFFNLayerc                 ób   •— t         ‰| �  «        t        |«      | _        t	        |«      | _        y rt   )ru   rv   r	  Úintermediater1  rü   rË   s     €r4   rv   zFFNLayer.__init__æ  s'   ø€ Ü‰ÑÔÜ2°6Ó:ˆÔÜ Ó'ˆ�r~   rô   r€   c                 óL   — | j                  |«      }| j                  ||«      }|S rt   )r7  rü   )rz   rô   Úintermediate_outputrø   s       r4   r‚   zFFNLayer.forwardë  s*   € Ø"×/Ñ/°Ó>ÐØŸ™Ð$7¸ÓGˆØÐr~   rƒ   r‰   s   @r4   r5  r5  å  s#   ø„ ô(ð
 U§\¡\ð °e·l±l÷ r~   r5  c                   óª   ‡ — e Zd Zˆ fd„Z	 	 	 ddej
                  deej                     deej                     dee   de	ej
                     f
d„Z
ˆ xZS )	ÚMobileBertLayerc                 ó¶  •— t         ‰| �  «        |j                  | _        |j                  | _        t	        |«      | _        t        |«      | _        t        |«      | _	        | j                  rt        |«      | _        |j                  dkD  rHt        j                  t        |j                  dz
  «      D �cg c]  }t        |«      ‘Œ c}«      | _        y y c c}w ©Nr   )ru   rv   rñ   Únum_feedforward_networksrú   r*  r	  r7  r  rü   r&  r  r   Ú
ModuleListÚranger5  r$   ©rz   ra   Ú_r}   s      €r4   rv   zMobileBertLayer.__init__ò  s«   ø€ Ü‰ÑÔØ$×3Ñ3ˆÔØ(.×(GÑ(GˆÔ%ä,¨VÓ4ˆŒÜ2°6Ó:ˆÔÜ& vÓ.ˆŒØ×ÒÜ(¨Ó0ˆDŒOØ×*Ñ*¨QÒ.Ü—}‘}ÄÀf×FeÑFeÐhiÑFiÓ@jÖ%k¸1¤h¨vÕ&6Ò%kÓlˆD�Hð /ùÚ%ks   Â6Crô   rÕ   rÖ   r×   r€   c           	      ó´  — | j                   r| j                  |«      \  }}}}n|gdz  \  }}}}| j                  |||||||¬«      }	|	d   }
|
f}|	dd  }| j                  dk7  r+t	        | j
                  «      D ]  \  }} ||
«      }
||
fz  }Œ | j                  |
«      }| j                  ||
|«      }|f|z   t        j                  d«      |||||
|fz   |z   }|S )Nr,  )r×   r   r   iè  )
rñ   r  r*  r>  Ú	enumerater$   r7  rü   r]   Útensor)rz   rô   rÕ   rÖ   r×   rÒ   rÓ   rÔ   r  Úself_attention_outputsr  Úsré   ÚiÚ
ffn_moduler9  r  s                    r4   r‚   zMobileBertLayer.forwardÿ  s>  € ð ×ÒØBFÇ/Á/ÐR_ÓB`Ñ?ˆL˜* l±KàCPÀ/ÐTUÑBUÑ?ˆL˜* l°Kà!%§¡ØØØØØØØ/ð "0ó "
Ðð 2°!Ñ4ÐØÐˆØ(¨¨Ð,ˆà×(Ñ(¨AÒ-Ü!*¨4¯8©8Ó!4ò )‘��:Ù#-Ð.>Ó#?Ð ØÐ&Ð(Ñ(‘ð)ð #×/Ñ/Ð0@ÓAÐØ—{‘{Ð#6Ð8HÈ-ÓXˆàˆOØñô —‘˜TÓ"ØØØØØ Ø#ðñ
ð ñð 	ð ˆr~   rê   )r„   r…   r†   rv   r]   r‡   r   r¿   rë   r   r‚   rˆ   r‰   s   @r4   r;  r;  ñ  sp   ø„ ômð  7;Ø15Ø,0ñ.à—|‘|ð.ð ! ×!2Ñ!2Ñ3ð.ð ˜E×-Ñ-Ñ.ð	.ð
 $ D™>ð.ð 
ˆu�|‰|Ñ	÷.r~   r;  c                   ó²   ‡ — e Zd Zˆ fd„Z	 	 	 	 	 d
dej
                  deej                     deej                     dee   dee   dee   de	e
ef   fd	„Zˆ xZS )ÚMobileBertEncoderc                 ó´   •— t         ‰| �  «        t        j                  t	        |j
                  «      D �cg c]  }t        |«      ‘Œ c}«      | _        y c c}w rt   )ru   rv   r   r?  r@  Únum_hidden_layersr;  ÚlayerrA  s      €r4   rv   zMobileBertEncoder.__init__1  s<   ø€ Ü‰ÑÔÜ—]‘]ÄUÈ6×KcÑKcÓEdÖ#eÀ¤O°FÕ$;Ò#eÓfˆ�
ùÒ#es   ¶Arô   rÕ   rÖ   r×   Úoutput_hidden_statesÚreturn_dictr€   c                 óü   — |rdnd }|rdnd }t        | j                  «      D ],  \  }	}
|r||fz   } |
||||	   |«      }|d   }|sŒ$||d   fz   }Œ. |r||fz   }|st        d„ |||fD «       «      S t        |||¬«      S )Nr1   r   r   c              3   ó&   K  — | ]	  }|€Œ|–— Œ y ­wrt   r1   )r2   Úvs     r4   r5   z,MobileBertEncoder.forward.<locals>.<genexpr>T  s   è ø€ Òh˜qÐZ[ÑZgœÑhùs   ‚Š)Úlast_hidden_staterô   Ú
attentions)rD  rN  Útupler   )rz   rô   rÕ   rÖ   r×   rO  rP  Úall_hidden_statesÚall_attentionsrH  Úlayer_modulerø   s               r4   r‚   zMobileBertEncoder.forward5  sÆ   € ñ #7™B¸DÐÙ0™°dˆÜ(¨¯©Ó4ò 	F‰OˆAˆ|Ù#Ø$5¸Ð8HÑ$HÐ!á(ØØØ˜!‘Ø!ó	ˆMð *¨!Ñ,ˆMâ Ø!/°=ÀÑ3CÐ2EÑ!E‘ð	Fñ   Ø 1°]Ð4DÑ DÐáÜÑh ]Ð4EÀ~Ð$VÔhÓhÐhÜØ+Ð;LÐYgô
ð 	
r~   )NNFFT)r„   r…   r†   rv   r]   r‡   r   r¿   rë   r   r   r   r‚   rˆ   r‰   s   @r4   rK  rK  0  s“   ø„ ôgð 7;Ø15Ø,1Ø/4Ø&*ñ"
à—|‘|ð"
ð ! ×!2Ñ!2Ñ3ð"
ð ˜E×-Ñ-Ñ.ð	"
ð
 $ D™>ð"
ð ' t™nð"
ð ˜d‘^ð"
ð 
ˆu�oÐ%Ñ	&÷"
r~   rK  c                   óV   ‡ — e Zd Zˆ fd„Zdej
                  dej
                  fd„Zˆ xZS )ÚMobileBertPoolerc                 ó¼   •— t         ‰| �  «        |j                  | _        | j                  r0t	        j
                  |j                  |j                  «      | _        y y rt   )ru   rv   Úclassifier_activationÚdo_activater   rž   r•   rò   rË   s     €r4   rv   zMobileBertPooler.__init__[  sH   ø€ Ü‰ÑÔØ!×7Ñ7ˆÔØ×ÒÜŸ™ 6×#5Ñ#5°v×7IÑ7IÓJˆD�Jð r~   rô   r€   c                 ó€   — |d d …df   }| j                   s|S | j                  |«      }t        j                  |«      }|S )Nr   )r^  rò   r]   Útanh)rz   rô   Úfirst_token_tensorÚpooled_outputs       r4   r‚   zMobileBertPooler.forwarda  sE   € ð +ª1¨a¨4Ñ0ÐØ×ÒØ%Ð%à ŸJ™JÐ'9Ó:ˆMÜ!ŸJ™J }Ó5ˆMØ Ð r~   rƒ   r‰   s   @r4   r[  r[  Z  s$   ø„ ôKð	! U§\¡\ð 	!°e·l±l÷ 	!r~   r[  c                   óV   ‡ — e Zd Zˆ fd„Zdej
                  dej
                  fd„Zˆ xZS )Ú!MobileBertPredictionHeadTransformc                 óZ  •— t         ‰| �  «        t        j                  |j                  |j                  «      | _        t        |j                  t        «      rt        |j                     | _
        n|j                  | _
        t        d   |j                  |j                  ¬«      | _        y )NrŠ   rð   )ru   rv   r   rž   r•   rò   r  r  r  r   Útransform_act_fnr    ró   r&   rË   s     €r4   rv   z*MobileBertPredictionHeadTransform.__init__n  s|   ø€ Ü‰ÑÔÜ—Y‘Y˜v×1Ñ1°6×3EÑ3EÓFˆŒ
Ü�f×'Ñ'¬Ô-Ü$*¨6×+<Ñ+<Ñ$=ˆDÕ!à$*×$5Ñ$5ˆDÔ!Ü  Ñ.¨v×/AÑ/AÀv×G\ÑG\Ô]ˆ�r~   rô   r€   c                 ól   — | j                  |«      }| j                  |«      }| j                  |«      }|S rt   )rò   rf  r&   r  s     r4   r‚   z)MobileBertPredictionHeadTransform.forwardw  s4   € ØŸ
™
 =Ó1ˆØ×-Ñ-¨mÓ<ˆØŸ™ }Ó5ˆØÐr~   rƒ   r‰   s   @r4   rd  rd  m  s$   ø„ ô^ð U§\¡\ð °e·l±l÷ r~   rd  c                   ó^   ‡ — e Zd Zˆ fd„Zdd„Zdej                  dej                  fd„Zˆ xZS )ÚMobileBertLMPredictionHeadc                 óÄ  •— t         ‰| �  «        t        |«      | _        t	        j
                  |j                  |j                  |j                  z
  d¬«      | _	        t	        j
                  |j                  |j                  d¬«      | _
        t	        j                  t        j                  |j                  «      «      | _        | j                  | j                  _        y )NF)r;   )ru   rv   rd  Ú	transformr   rž   r—   r•   r”   rò   Údecoderrw   r]   rx   r;   rË   s     €r4   rv   z#MobileBertLMPredictionHead.__init__  s˜   ø€ Ü‰ÑÔÜ:¸6ÓBˆŒô —Y‘Y˜v×0Ñ0°&×2DÑ2DÀv×G\ÑG\Ñ2\ÐchÔiˆŒ
Ü—y‘y ×!6Ñ!6¸×8IÑ8IÐPUÔVˆŒÜ—L‘L¤§¡¨V×->Ñ->Ó!?Ó@ˆŒ	à ŸI™Iˆ�‰Õr~   r€   c                 ó:   — | j                   | j                  _         y rt   )r;   rl  ©rz   s    r4   Ú_tie_weightsz'MobileBertLMPredictionHead._tie_weightsŠ  s   € Ø ŸI™Iˆ�‰Õr~   rô   c                 ó  — | j                  |«      }|j                  t        j                  | j                  j
                  j                  «       | j                  j
                  gd¬«      «      }|| j                  j                  z  }|S )Nr   r³   )	rk  rÚ   r]   r·   rl  r8   Útrò   r;   r  s     r4   r‚   z"MobileBertLMPredictionHead.forward�  sk   € ØŸ™ }Ó5ˆØ%×,Ñ,¬U¯Y©Y¸¿¹×8KÑ8K×8MÑ8MÓ8OÐQU×Q[ÑQ[×QbÑQbÐ7cÐijÔ-kÓlˆØ˜Ÿ™×*Ñ*Ñ*ˆØÐr~   )r€   N)	r„   r…   r†   rv   ro  r]   r‡   r‚   rˆ   r‰   s   @r4   ri  ri  ~  s(   ø„ ô	&ó&ð U§\¡\ð °e·l±l÷ r~   ri  c                   óV   ‡ — e Zd Zˆ fd„Zdej
                  dej
                  fd„Zˆ xZS )ÚMobileBertOnlyMLMHeadc                 óB   •— t         ‰| �  «        t        |«      | _        y rt   )ru   rv   ri  ÚpredictionsrË   s     €r4   rv   zMobileBertOnlyMLMHead.__init__•  s   ø€ Ü‰ÑÔÜ5°fÓ=ˆÕr~   Úsequence_outputr€   c                 ó(   — | j                  |«      }|S rt   )ru  )rz   rv  Úprediction_scoress      r4   r‚   zMobileBertOnlyMLMHead.forward™  s   € Ø ×,Ñ,¨_Ó=ÐØ Ð r~   rƒ   r‰   s   @r4   rs  rs  ”  s#   ø„ ô>ð! u§|¡|ð !¸¿¹÷ !r~   rs  c                   ót   ‡ — e Zd Zˆ fd„Zdej
                  dej
                  deej
                     fd„Zˆ xZS )ÚMobileBertPreTrainingHeadsc                 óŒ   •— t         ‰| �  «        t        |«      | _        t	        j
                  |j                  d«      | _        y ©Nr?   )ru   rv   ri  ru  r   rž   r•   Úseq_relationshiprË   s     €r4   rv   z#MobileBertPreTrainingHeads.__init__Ÿ  s4   ø€ Ü‰ÑÔÜ5°fÓ=ˆÔÜ "§	¡	¨&×*<Ñ*<¸aÓ @ˆÕr~   rv  rb  r€   c                 óN   — | j                  |«      }| j                  |«      }||fS rt   )ru  r}  )rz   rv  rb  rx  Úseq_relationship_scores        r4   r‚   z"MobileBertPreTrainingHeads.forward¤  s0   € Ø ×,Ñ,¨_Ó=ÐØ!%×!6Ñ!6°}Ó!EÐØ Ð"8Ð8Ð8r~   r/  r‰   s   @r4   rz  rz  ž  s8   ø„ ôAð
9 u§|¡|ð 9ÀEÇLÁLð 9ÐUZÐ[`×[gÑ[gÑUh÷ 9r~   rz  c                   ó"   — e Zd ZdZeZeZdZd„ Z	y)ÚMobileBertPreTrainedModelz†
    An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained
    models.
    r)   c                 óx  — t        |t        j                  «      rm|j                  j                  j                  d| j                  j                  ¬«       |j                  �%|j                  j                  j                  «        yyt        |t        j                  «      rz|j                  j                  j                  d| j                  j                  ¬«       |j                  �2|j                  j                  |j                     j                  «        yyt        |t        j                  t        f«      rJ|j                  j                  j                  «        |j                  j                  j                  d«       yt        |t        «      r%|j                  j                  j                  «        yy)zInitialize the weightsr±   )ÚmeanÚstdNg      ð?)r  r   rž   r8   r_   Únormal_ra   Úinitializer_ranger;   Úzero_r–   r�   r&   rr   Úfill_ri  )rz   Úmodules     r4   Ú_init_weightsz'MobileBertPreTrainedModel._init_weights´  s,  € ä�fœbŸi™iÔ(ð �M‰M×Ñ×&Ñ&¨C°T·[±[×5RÑ5RÐ&ÔSØ�{‰{Ð&Ø—‘× Ñ ×&Ñ&Õ(ð 'ä˜¤§¡Ô-Ø�M‰M×Ñ×&Ñ&¨C°T·[±[×5RÑ5RÐ&ÔSØ×!Ñ!Ð-Ø—‘×"Ñ" 6×#5Ñ#5Ñ6×<Ñ<Õ>ð .ä˜¤§¡¬vÐ 6Ô7Ø�K‰K×Ñ×"Ñ"Ô$Ø�M‰M×Ñ×$Ñ$ SÕ)Ü˜Ô :Ô;Ø�K‰K×Ñ×"Ñ"Õ$ð <r~   N)
r„   r…   r†   r½   r   Úconfig_classrp   Úload_tf_weightsÚbase_model_prefixrŠ  r1   r~   r4   r�  r�  ª  s   „ ñð
 $€LØ3€OØ$Ðó%r~   r�  c                   óæ   — e Zd ZU dZdZeej                     ed<   dZ	eej                     ed<   dZ
eej                     ed<   dZeeej                        ed<   dZeeej                        ed<   y)ÚMobileBertForPreTrainingOutputab  
    Output type of [`MobileBertForPreTraining`].

    Args:
        loss (*optional*, returned when `labels` is provided, `torch.FloatTensor` of shape `(1,)`):
            Total loss as the sum of the masked language modeling loss and the next sequence prediction
            (classification) loss.
        prediction_logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`):
            Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
        seq_relationship_logits (`torch.FloatTensor` of shape `(batch_size, 2)`):
            Prediction scores of the next sequence prediction (classification) head (scores of True/False continuation
            before SoftMax).
        hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
            Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
            shape `(batch_size, sequence_length, hidden_size)`.

            Hidden-states of the model at the output of each layer plus the initial embedding outputs.
        attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
            Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
            sequence_length)`.

            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
            heads.
    NÚlossÚprediction_logitsÚseq_relationship_logitsrô   rU  )r„   r…   r†   r½   r�  r   r]   r¿   Ú__annotations__r‘  r’  rô   r   rU  r1   r~   r4   r�  r�  Ç  s~   … ñð2 )-€Dˆ(�5×$Ñ$Ñ
%Ó,Ø59Ð�x × 1Ñ 1Ñ2Ó9Ø;?Ð˜X e×&7Ñ&7Ñ8Ó?Ø8<€M�8˜E %×"3Ñ"3Ñ4Ñ5Ó<Ø59€J�˜˜u×0Ñ0Ñ1Ñ2Ô9r~   r�  aD  

    This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
    library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
    etc.)

    This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
    Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
    and behavior.

    Parameters:
        config ([`MobileBertConfig`]): Model configuration class with all the parameters of the model.
            Initializing with a config file does not load the weights associated with the model, only the
            configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
a5
  
    Args:
        input_ids (`torch.LongTensor` of shape `({0})`):
            Indices of input sequence tokens in the vocabulary.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are input IDs?](../glossary#input-ids)
        attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*):
            Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:

            - 1 for tokens that are **not masked**,
            - 0 for tokens that are **masked**.

            [What are attention masks?](../glossary#attention-mask)
        token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*):
            Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
            1]`:

            - 0 corresponds to a *sentence A* token,
            - 1 corresponds to a *sentence B* token.

            [What are token type IDs?](../glossary#token-type-ids)
        position_ids (`torch.LongTensor` of shape `({0})`, *optional*):
            Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
            config.max_position_embeddings - 1]`.

            [What are position IDs?](../glossary#position-ids)
        head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*):
            Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`:

            - 1 indicates the head is **not masked**,
            - 0 indicates the head is **masked**.

        inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*):
            Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
            is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
            model's internal embedding lookup matrix.
        output_attentions (`bool`, *optional*):
            Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
            tensors for more detail.
        output_hidden_states (`bool`, *optional*):
            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
            more detail.
        return_dict (`bool`, *optional*):
            Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
zdThe bare MobileBert Model transformer outputting raw hidden-states without any specific head on top.c                   ó‚  ‡ — e Zd ZdZdˆ fd„	Zd„ Zd„ Zd„ Z ee	j                  d«      «       eeee¬«      	 	 	 	 	 	 	 	 	 ddeej"                     d	eej$                     d
eej"                     deej"                     deej$                     deej$                     dee   dee   dee   deeef   fd„«       «       Zˆ xZS )ÚMobileBertModelz.
    https://arxiv.org/pdf/2004.02984.pdf
    c                 óº   •— t         ‰| �  |«       || _        t        |«      | _        t        |«      | _        |rt        |«      nd | _        | j                  «        y rt   )
ru   rv   ra   r�   r¼   rK  Úencoderr[  ÚpoolerÚ	post_init)rz   ra   Úadd_pooling_layerr}   s      €r4   rv   zMobileBertModel.__init__4  sL   ø€ Ü‰Ñ˜Ô ØˆŒÜ.¨vÓ6ˆŒÜ(¨Ó0ˆŒá2CÔ& vÔ.ÈˆŒð 	�‰Õr~   c                 ó.   — | j                   j                  S rt   ©r¼   r™   rn  s    r4   Úget_input_embeddingsz$MobileBertModel.get_input_embeddings?  s   € Ø�‰×.Ñ.Ð.r~   c                 ó&   — || j                   _        y rt   rœ  )rz   r²   s     r4   Úset_input_embeddingsz$MobileBertModel.set_input_embeddingsB  s   € Ø*/ˆ�‰Õ'r~   c                 ó˜   — |j                  «       D ]7  \  }}| j                  j                  |   j                  j	                  |«       Œ9 y)z�
        Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base
        class PreTrainedModel
        N)Úitemsr—  rN  r*  r  )rz   Úheads_to_prunerN  r  s       r4   Ú_prune_headszMobileBertModel._prune_headsE  sE   € ð
 +×0Ñ0Ó2ò 	C‰LˆE�5Ø�L‰L×Ñ˜uÑ%×/Ñ/×;Ñ;¸EÕBñ	Cr~   úbatch_size, sequence_length©Ú
checkpointÚoutput_typer‹  rª   rÕ   r«   r�   rÖ   r¬   rO  r×   rP  r€   c
                 ól  — |�|n| j                   j                  }|�|n| j                   j                  }|	�|	n| j                   j                  }	|�|�t	        d«      ‚|�#| j                  ||«       |j                  «       }
n!|�|j                  «       d d }
nt	        d«      ‚|�|j                  n|j                  }|€t        j                  |
|¬«      }|€&t        j                  |
t        j                  |¬«      }| j                  ||
«      }| j                  || j                   j                  «      }| j                  ||||¬«      }| j!                  ||||||	¬«      }|d   }| j"                  �| j#                  |«      nd }|	s
||f|d	d  z   S t%        |||j&                  |j(                  ¬
«      S )NzDYou cannot specify both input_ids and inputs_embeds at the same timer‘   z5You have to specify either input_ids or inputs_embeds)r°   r®   )rª   r�   r«   r¬   )rÕ   rÖ   r×   rO  rP  r   r   )rT  Úpooler_outputrô   rU  )ra   r×   rO  Úuse_return_dictÚ
ValueErrorÚ%warn_if_padding_and_no_attention_maskrµ   r°   r]   ry   rx   r¶   Úget_extended_attention_maskÚget_head_maskrM  r¼   r—  r˜  r   rô   rU  )rz   rª   rÕ   r«   r�   rÖ   r¬   rO  r×   rP  rº   r°   Úextended_attention_maskÚembedding_outputÚencoder_outputsrv  rb  s                    r4   r‚   zMobileBertModel.forwardM  sÛ  € ð$ 2CÐ1NÑ-ÐTX×T_ÑT_×TqÑTqÐà$8Ð$DÑ È$Ï+É+×JjÑJjð 	ð &1Ð%<‘kÀ$Ç+Á+×B]ÑB]ˆàÐ  ]Ð%>ÜÐcÓdÐdØÐ"Ø×6Ñ6°yÀ.ÔQØ#Ÿ.™.Ó*‰KØÐ&Ø'×,Ñ,Ó.¨s°Ð3‰KäÐTÓUÐUà%.Ð%:�×!Ò!À×@TÑ@TˆàÐ!Ü"ŸZ™Z¨¸FÔCˆNØÐ!Ü"Ÿ[™[¨¼E¿J¹JÈvÔVˆNð 15×0PÑ0PÐQ_ÐalÓ0mÐð ×&Ñ& y°$·+±+×2OÑ2OÓPˆ	àŸ?™?Ø¨lÈ>Ðivð +ó 
Ðð Ÿ,™,ØØ2ØØ/Ø!5Ø#ð 'ó 
ˆð *¨!Ñ,ˆØ8<¿¹Ð8O˜Ÿ™ OÔ4ÐUYˆáØ# ]Ð3°oÀaÀbÐ6IÑIÐIä)Ø-Ø'Ø)×7Ñ7Ø&×1Ñ1ô	
ð 	
r~   )T)	NNNNNNNNN)r„   r…   r†   r½   rv   r�  rŸ  r£  r   ÚMOBILEBERT_INPUTS_DOCSTRINGÚformatr   Ú_CHECKPOINT_FOR_DOCr   Ú_CONFIG_FOR_DOCr   r]   r¾   r¿   rë   r   r   r‚   rˆ   r‰   s   @r4   r•  r•  +  s>  ø„ ñ
õ	ò/ò0òCñ +Ð+F×+MÑ+MÐNkÓ+lÓmÙØ&Ø.Ø$ôð 15Ø6:Ø59Ø37Ø15Ø59Ø/3Ø,0Ø&*ñD
à˜E×,Ñ,Ñ-ðD
ð ! ×!2Ñ!2Ñ3ðD
ð ! ×!1Ñ!1Ñ2ð	D
ð
 ˜u×/Ñ/Ñ0ðD
ð ˜E×-Ñ-Ñ.ðD
ð   × 1Ñ 1Ñ2ðD
ð ' t™nðD
ð $ D™>ðD
ð ˜d‘^ðD
ð 
ˆuÐ0Ð0Ñ	1òD
óó nôD
r~   r•  z®
    MobileBert Model with two heads on top as done during the pretraining: a `masked language modeling` head and a
    `next sentence prediction (classification)` head.
    c                   ó(  ‡ — e Zd ZddgZˆ fd„Zd„ Zd„ Zddee   de	j                  fˆ fd„Z eej                  d	«      «       eee¬
«      	 	 	 	 	 	 	 	 	 	 	 ddeej&                     deej(                     deej&                     deej&                     deej(                     deej(                     deej&                     deej&                     deej(                     deej(                     deej(                     deeef   fd„«       «       Zˆ xZS )ÚMobileBertForPreTrainingúcls.predictions.decoder.weightúcls.predictions.decoder.biasc                 ó„   •— t         ‰| �  |«       t        |«      | _        t	        |«      | _        | j                  «        y rt   )ru   rv   r•  r)   rz  Úclsr™  rË   s     €r4   rv   z!MobileBertForPreTraining.__init__¤  s4   ø€ Ü‰Ñ˜Ô Ü)¨&Ó1ˆŒÜ-¨fÓ5ˆŒð 	�‰Õr~   c                 óB   — | j                   j                  j                  S rt   ©r»  ru  rl  rn  s    r4   Úget_output_embeddingsz.MobileBertForPreTraining.get_output_embeddings¬  ó   € Ø�x‰x×#Ñ#×+Ñ+Ð+r~   c                 ó„   — || j                   j                  _        |j                  | j                   j                  _        y rt   ©r»  ru  rl  r;   ©rz   Únew_embeddingss     r4   Úset_output_embeddingsz.MobileBertForPreTraining.set_output_embeddings¯  ó,   € Ø'5ˆ�‰×ÑÔ$Ø$2×$7Ñ$7ˆ�‰×ÑÕ!r~   Únew_num_tokensr€   c                 óº   •— | j                  | j                  j                  j                  |d¬«      | j                  j                  _        t        ‰| �  |¬«      S ©NT)rÆ  Ú
transposed)rÆ  ©Ú_get_resized_lm_headr»  ru  rò   ru   Úresize_token_embeddings©rz   rÆ  r}   s     €r4   rÌ  z0MobileBertForPreTraining.resize_token_embeddings³  sR   ø€ à%)×%>Ñ%>Ø�H‰H× Ñ ×&Ñ&°~ÐRVð &?ó &
ˆ�‰×ÑÔ"ô ‰wÑ.¸nÐ.ÓMÐMr~   r¤  ©r§  r‹  rª   rÕ   r«   r�   rÖ   r¬   ÚlabelsÚnext_sentence_labelr×   rO  rP  c                 ó
  — |�|n| j                   j                  }| j                  |||||||	|
|¬«	      }|dd \  }}| j                  ||«      \  }}d}|�u|�st	        «       } ||j                  d| j                   j                  «      |j                  d«      «      } ||j                  dd«      |j                  d«      «      }||z   }|s||f|dd z   }|�|f|z   S |S t        ||||j                  |j                  ¬«      S )aÀ  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
            config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
            loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
        next_sentence_label (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the next sequence prediction (classification) loss. Input should be a sequence pair
            (see `input_ids` docstring) Indices should be in `[0, 1]`:

            - 0 indicates sequence B is a continuation of sequence A,
            - 1 indicates sequence B is a random sequence.

        Returns:

        Examples:

        ```python
        >>> from transformers import AutoTokenizer, MobileBertForPreTraining
        >>> import torch

        >>> tokenizer = AutoTokenizer.from_pretrained("google/mobilebert-uncased")
        >>> model = MobileBertForPreTraining.from_pretrained("google/mobilebert-uncased")

        >>> input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True)).unsqueeze(0)
        >>> # Batch size 1
        >>> outputs = model(input_ids)

        >>> prediction_logits = outputs.prediction_logits
        >>> seq_relationship_logits = outputs.seq_relationship_logits
        ```N©rÕ   r«   r�   rÖ   r¬   r×   rO  rP  r?   r‘   )r�  r‘  r’  rô   rU  )
ra   rª  r)   r»  r	   rÍ   r—   r�  rô   rU  )rz   rª   rÕ   r«   r�   rÖ   r¬   rÏ  rÐ  r×   rO  rP  ré   rv  rb  rx  r  Ú
total_lossÚloss_fctÚmasked_lm_lossÚnext_sentence_lossrü   s                         r4   r‚   z MobileBertForPreTraining.forward»  sG  € ð\ &1Ð%<‘kÀ$Ç+Á+×B]ÑB]ˆà—/‘/ØØ)Ø)Ø%ØØ'Ø/Ø!5Ø#ð "ó 

ˆð *1°°!¨Ñ&ˆ˜Ø48·H±H¸_ÈmÓ4\Ñ1ÐÐ1àˆ
ØÐÐ"5Ð"AÜ'Ó)ˆHÙ%Ð&7×&<Ñ&<¸RÀÇÁ×AWÑAWÓ&XÐZ`×ZeÑZeÐfhÓZiÓjˆNÙ!)Ð*@×*EÑ*EÀbÈ!Ó*LÐNa×NfÑNfÐgiÓNjÓ!kÐØ'Ð*<Ñ<ˆJáØ'Ð)?Ð@À7È1È2À;ÑNˆFØ/9Ð/E�Z�M FÑ*ÐQÈ6ÐQä-ØØ/Ø$:Ø!×/Ñ/Ø×)Ñ)ô
ð 	
r~   rt   ©NNNNNNNNNNN)r„   r…   r†   Ú_tied_weights_keysrv   r¾  rÄ  r   rX   r   r–   rÌ  r   r²  r³  r   r�  rµ  r]   r¾   r¿   r   r   r‚   rˆ   r‰   s   @r4   r·  r·  š  s¢  ø„ ð ;Ð<ZÐ[Ðôò,ò8ñN°h¸s±mð NÈrÏ|É|õ Nñ +Ð+F×+MÑ+MÐNkÓ+lÓmÙÐ+IÐXgÔhð 15Ø6:Ø59Ø37Ø15Ø59Ø-1Ø:>Ø9=Ø<@Ø37ñM
à˜E×,Ñ,Ñ-ðM
ð ! ×!2Ñ!2Ñ3ðM
ð ! ×!1Ñ!1Ñ2ð	M
ð
 ˜u×/Ñ/Ñ0ðM
ð ˜E×-Ñ-Ñ.ðM
ð   × 1Ñ 1Ñ2ðM
ð ˜×)Ñ)Ñ*ðM
ð & e×&6Ñ&6Ñ7ðM
ð $ E×$5Ñ$5Ñ6ðM
ð ' u×'8Ñ'8Ñ9ðM
ð ˜e×/Ñ/Ñ0ðM
ð 
ˆuÐ4Ð4Ñ	5òM
ó ió nôM
r~   r·  z8MobileBert Model with a `language modeling` head on top.c                   óÒ  ‡ — e Zd ZddgZˆ fd„Zd„ Zd„ Zddee   de	j                  fˆ fd„Z eej                  d	«      «       eeeed
d¬«      	 	 	 	 	 	 	 	 	 	 ddeej(                     deej*                     deej(                     deej(                     deej*                     deej*                     deej(                     dee   dee   dee   deeef   fd„«       «       Zˆ xZS )ÚMobileBertForMaskedLMr¸  r¹  c                 ó–   •— t         ‰| �  |«       t        |d¬«      | _        t	        |«      | _        || _        | j                  «        y ©NF)rš  )ru   rv   r•  r)   rs  r»  ra   r™  rË   s     €r4   rv   zMobileBertForMaskedLM.__init__  s=   ø€ Ü‰Ñ˜Ô Ü)¨&ÀEÔJˆŒÜ(¨Ó0ˆŒØˆŒð 	�‰Õr~   c                 óB   — | j                   j                  j                  S rt   r½  rn  s    r4   r¾  z+MobileBertForMaskedLM.get_output_embeddings  r¿  r~   c                 ó„   — || j                   j                  _        |j                  | j                   j                  _        y rt   rÁ  rÂ  s     r4   rÄ  z+MobileBertForMaskedLM.set_output_embeddings  rÅ  r~   rÆ  r€   c                 óº   •— | j                  | j                  j                  j                  |d¬«      | j                  j                  _        t        ‰| �  |¬«      S rÈ  rÊ  rÍ  s     €r4   rÌ  z-MobileBertForMaskedLM.resize_token_embeddings!  sR   ø€ à%)×%>Ñ%>Ø�H‰H× Ñ ×&Ñ&°~ÐRVð &?ó &
ˆ�‰×ÑÔ"ô ‰wÑ.¸nÐ.ÓMÐMr~   r¤  z'paris'g=
×£p=â?©r¦  r§  r‹  Úexpected_outputÚexpected_lossrª   rÕ   r«   r�   rÖ   r¬   rÏ  r×   rO  rP  c                 óš  — |
�|
n| j                   j                  }
| j                  ||||||||	|
¬«	      }|d   }| j                  |«      }d}|�Ft	        «       } ||j                  d| j                   j                  «      |j                  d«      «      }|
s|f|dd z   }|�|f|z   S |S t        |||j                  |j                  ¬«      S )a¢  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
            config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
            loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
        NrÒ  r   r‘   r?   ©r�  Úlogitsrô   rU  )
ra   rª  r)   r»  r	   rÍ   r—   r   rô   rU  )rz   rª   rÕ   r«   r�   rÖ   r¬   rÏ  r×   rO  rP  ré   rv  rx  rÕ  rÔ  rü   s                    r4   r‚   zMobileBertForMaskedLM.forward(  sú   € ð6 &1Ð%<‘kÀ$Ç+Á+×B]ÑB]ˆà—/‘/ØØ)Ø)Ø%ØØ'Ø/Ø!5Ø#ð "ó 

ˆð " !™*ˆØ ŸH™H _Ó5ÐàˆØÐÜ'Ó)ˆHÙ%Ð&7×&<Ñ&<¸RÀÇÁ×AWÑAWÓ&XÐZ`×ZeÑZeÐfhÓZiÓjˆNáØ'Ð)¨G°A°B¨KÑ7ˆFØ3AÐ3M�^Ð%¨Ñ.ÐYÐSYÐYäØØ$Ø!×/Ñ/Ø×)Ñ)ô	
ð 	
r~   rt   ©
NNNNNNNNNN)r„   r…   r†   rØ  rv   r¾  rÄ  r   rX   r   r–   rÌ  r   r²  r³  r   r´  r   rµ  r]   r¾   r¿   rë   r   r   r‚   rˆ   r‰   s   @r4   rÚ  rÚ    sn  ø„ à:Ð<ZÐ[Ðôò,ò8ñN°h¸s±mð NÈrÏ|É|õ Nñ +Ð+F×+MÑ+MÐNkÓ+lÓmÙØ&Ø"Ø$Ø!Øôð 15Ø6:Ø59Ø37Ø15Ø59Ø-1Ø,0Ø/3Ø&*ñ2
à˜E×,Ñ,Ñ-ð2
ð ! ×!2Ñ!2Ñ3ð2
ð ! ×!1Ñ!1Ñ2ð	2
ð
 ˜u×/Ñ/Ñ0ð2
ð ˜E×-Ñ-Ñ.ð2
ð   × 1Ñ 1Ñ2ð2
ð ˜×)Ñ)Ñ*ð2
ð $ D™>ð2
ð ' t™nð2
ð ˜d‘^ð2
ð 
ˆu�nÐ$Ñ	%ò2
óó nô2
r~   rÚ  c                   óV   ‡ — e Zd Zˆ fd„Zdej
                  dej
                  fd„Zˆ xZS )ÚMobileBertOnlyNSPHeadc                 ól   •— t         ‰| �  «        t        j                  |j                  d«      | _        y r|  )ru   rv   r   rž   r•   r}  rË   s     €r4   rv   zMobileBertOnlyNSPHead.__init__f  s'   ø€ Ü‰ÑÔÜ "§	¡	¨&×*<Ñ*<¸aÓ @ˆÕr~   rb  r€   c                 ó(   — | j                  |«      }|S rt   )r}  )rz   rb  r  s      r4   r‚   zMobileBertOnlyNSPHead.forwardj  s   € Ø!%×!6Ñ!6°}Ó!EÐØ%Ð%r~   rƒ   r‰   s   @r4   rè  rè  e  s$   ø„ ôAð& U§\¡\ð &°e·l±l÷ &r~   rè  zPMobileBert Model with a `next sentence prediction (classification)` head on top.c                   óˆ  ‡ — e Zd Zˆ fd„Z eej                  d«      «       eee	¬«      	 	 	 	 	 	 	 	 	 	 dde
ej                     de
ej                     de
ej                     de
ej                     de
ej                     d	e
ej                     d
e
ej                     de
e   de
e   de
e   deeef   fd„«       «       Zˆ xZS )Ú#MobileBertForNextSentencePredictionc                 ó„   •— t         ‰| �  |«       t        |«      | _        t	        |«      | _        | j                  «        y rt   )ru   rv   r•  r)   rè  r»  r™  rË   s     €r4   rv   z,MobileBertForNextSentencePrediction.__init__t  s4   ø€ Ü‰Ñ˜Ô ä)¨&Ó1ˆŒÜ(¨Ó0ˆŒð 	�‰Õr~   r¤  rÎ  rª   rÕ   r«   r�   rÖ   r¬   rÏ  r×   rO  rP  r€   c                 óÐ  — d|v r+t        j                  dt        «       |j                  d«      }|
�|
n| j                  j
                  }
| j                  ||||||||	|
¬«	      }|d   }| j                  |«      }d}|�2t        «       } ||j                  dd«      |j                  d«      «      }|
s|f|dd z   }|�|f|z   S |S t        |||j                  |j                  ¬«      S )	aœ  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the next sequence prediction (classification) loss. Input should be a sequence pair
            (see `input_ids` docstring) Indices should be in `[0, 1]`.

            - 0 indicates sequence B is a continuation of sequence A,
            - 1 indicates sequence B is a random sequence.

        Returns:

        Examples:

        ```python
        >>> from transformers import AutoTokenizer, MobileBertForNextSentencePrediction
        >>> import torch

        >>> tokenizer = AutoTokenizer.from_pretrained("google/mobilebert-uncased")
        >>> model = MobileBertForNextSentencePrediction.from_pretrained("google/mobilebert-uncased")

        >>> prompt = "In Italy, pizza served in formal settings, such as at a restaurant, is presented unsliced."
        >>> next_sentence = "The sky is blue due to the shorter wavelength of blue light."
        >>> encoding = tokenizer(prompt, next_sentence, return_tensors="pt")

        >>> outputs = model(**encoding, labels=torch.LongTensor([1]))
        >>> loss = outputs.loss
        >>> logits = outputs.logits
        ```rÐ  zoThe `next_sentence_label` argument is deprecated and will be removed in a future version, use `labels` instead.NrÒ  r   r‘   r?   rä  )ÚwarningsÚwarnÚFutureWarningÚpopra   rª  r)   r»  r	   rÍ   r   rô   rU  )rz   rª   rÕ   r«   r�   rÖ   r¬   rÏ  r×   rO  rP  Úkwargsré   rb  r  rÖ  rÔ  rü   s                     r4   r‚   z+MobileBertForNextSentencePrediction.forward}  s  € ðX ! FÑ*Ü�M‰Mð%äôð
 —Z‘ZÐ 5Ó6ˆFà%0Ð%<‘kÀ$Ç+Á+×B]ÑB]ˆà—/‘/ØØ)Ø)Ø%ØØ'Ø/Ø!5Ø#ð "ó 

ˆð   ™
ˆØ!%§¡¨-Ó!8Ðà!ÐØÐÜ'Ó)ˆHÙ!)Ð*@×*EÑ*EÀbÈ!Ó*LÈfÏkÉkÐZ\ËoÓ!^ÐáØ,Ð.°¸¸°Ñ<ˆFØ7IÐ7UÐ'Ð)¨FÑ2ÐaÐ[aÐaä*Ø#Ø)Ø!×/Ñ/Ø×)Ñ)ô	
ð 	
r~   ræ  )r„   r…   r†   rv   r   r²  r³  r   r   rµ  r   r]   r¾   r¿   rë   r   r   r‚   rˆ   r‰   s   @r4   rì  rì  o  s>  ø„ ô
ñ +Ð+F×+MÑ+MÐNkÓ+lÓmÙÐ+FÐUdÔeð 15Ø6:Ø59Ø37Ø15Ø59Ø-1Ø,0Ø/3Ø&*ñQ
à˜E×,Ñ,Ñ-ðQ
ð ! ×!2Ñ!2Ñ3ðQ
ð ! ×!1Ñ!1Ñ2ð	Q
ð
 ˜u×/Ñ/Ñ0ðQ
ð ˜E×-Ñ-Ñ.ðQ
ð   × 1Ñ 1Ñ2ðQ
ð ˜×)Ñ)Ñ*ðQ
ð $ D™>ðQ
ð ' t™nðQ
ð ˜d‘^ðQ
ð 
ˆuÐ1Ð1Ñ	2òQ
ó fó nôQ
r~   rì  z¢
    MobileBert Model transformer with a sequence classification/regression head on top (a linear layer on top of the
    pooled output) e.g. for GLUE tasks.
    c                   ó¨  ‡ — e Zd Zˆ fd„Z eej                  d«      «       eee	e
ee¬«      	 	 	 	 	 	 	 	 	 	 ddeej                     deej                     deej                     deej                     deej                     d	eej                     d
eej                     dee   dee   dee   deeej                     e	f   fd„«       «       Zˆ xZS )Ú#MobileBertForSequenceClassificationc                 ón  •— t         ‰| �  |«       |j                  | _        || _        t	        |«      | _        |j                  �|j                  n|j                  }t        j                  |«      | _
        t        j                  |j                  |j                  «      | _        | j                  «        y rt   )ru   rv   Ú
num_labelsra   r•  r)   Úclassifier_dropoutr£   r   r¢   r¤   rž   r•   r>   r™  ©rz   ra   rø  r}   s      €r4   rv   z,MobileBertForSequenceClassification.__init__Ü  s�   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒØˆŒä)¨&Ó1ˆŒà)/×)BÑ)BÐ)NˆF×%Ò%ÐTZ×TnÑTnð 	ô —z‘zÐ"4Ó5ˆŒÜŸ)™) F×$6Ñ$6¸×8IÑ8IÓJˆŒð 	�‰Õr~   r¤  rà  rª   rÕ   r«   r�   rÖ   r¬   rÏ  r×   rO  rP  r€   c                 ó@  — |
�|
n| j                   j                  }
| j                  ||||||||	|
¬«	      }|d   }| j                  |«      }| j	                  |«      }d}|��‡| j                   j
                  €�| j                  dk(  rd| j                   _        nl| j                  dkD  rL|j                  t        j                  k(  s|j                  t        j                  k(  rd| j                   _        nd| j                   _        | j                   j
                  dk(  rIt        «       }| j                  dk(  r& ||j                  «       |j                  «       «      }nŒ |||«      }n‚| j                   j
                  dk(  r=t        «       } ||j                  d| j                  «      |j                  d«      «      }n,| j                   j
                  dk(  rt        «       } |||«      }|
s|f|dd z   }|�|f|z   S |S t!        |||j"                  |j$                  ¬	«      S )
a�  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        NrÒ  r   Ú
regressionÚsingle_label_classificationÚmulti_label_classificationr‘   r?   rä  )ra   rª  r)   r¤   r>   Úproblem_typer÷  r¯   r]   r¶   rX   r
   Úsqueezer	   rÍ   r   r   rô   rU  )rz   rª   rÕ   r«   r�   rÖ   r¬   rÏ  r×   rO  rP  ré   rb  rå  r�  rÔ  rü   s                    r4   r‚   z+MobileBertForSequenceClassification.forwardë  sè  € ð6 &1Ð%<‘kÀ$Ç+Á+×B]ÑB]ˆà—/‘/ØØ)Ø)Ø%ØØ'Ø/Ø!5Ø#ð "ó 

ˆð   ™
ˆàŸ™ ]Ó3ˆØ—‘ Ó/ˆàˆØÑØ�{‰{×'Ñ'Ð/Ø—?‘? aÒ'Ø/;�D—K‘KÕ,Ø—_‘_ qÒ(¨f¯l©l¼e¿j¹jÒ.HÈFÏLÉLÔ\a×\eÑ\eÒLeØ/L�D—K‘KÕ,à/K�D—K‘KÔ,à�{‰{×'Ñ'¨<Ò7Ü"›9�Ø—?‘? aÒ'Ù# F§N¡NÓ$4°f·n±nÓ6FÓG‘Dá# F¨FÓ3‘DØ—‘×)Ñ)Ð-JÒJÜ+Ó-�Ù §¡¨B°·±Ó @À&Ç+Á+ÈbÃ/ÓR‘Ø—‘×)Ñ)Ð-IÒIÜ,Ó.�Ù ¨Ó/�ÙØ�Y ¨¨ Ñ,ˆFØ)-Ð)9�T�G˜fÑ$ÐE¸vÐEä'ØØØ!×/Ñ/Ø×)Ñ)ô	
ð 	
r~   ræ  )r„   r…   r†   rv   r   r²  r³  r   Ú'_CHECKPOINT_FOR_SEQUENCE_CLASSIFICATIONr   rµ  Ú_SEQ_CLASS_EXPECTED_OUTPUTÚ_SEQ_CLASS_EXPECTED_LOSSr   r]   r‡   rë   r   r   r‚   rˆ   r‰   s   @r4   rõ  rõ  Ó  sC  ø„ ôñ +Ð+F×+MÑ+MÐNkÓ+lÓmÙØ:Ø,Ø$Ø2Ø.ôð -1Ø15Ø15Ø/3Ø,0Ø04Ø)-Ø,0Ø/3Ø&*ñE
à˜EŸL™LÑ)ðE
ð ! §¡Ñ.ðE
ð ! §¡Ñ.ð	E
ð
 ˜uŸ|™|Ñ,ðE
ð ˜EŸL™LÑ)ðE
ð   §¡Ñ-ðE
ð ˜Ÿ™Ñ&ðE
ð $ D™>ðE
ð ' t™nðE
ð ˜d‘^ðE
ð 
ˆu�U—\‘\Ñ"Ð$<Ð<Ñ	=òE
óó nôE
r~   rõ  zã
    MobileBert Model with a span classification head on top for extractive question-answering tasks like SQuAD (a
    linear layers on top of the hidden-states output to compute `span start logits` and `span end logits`).
    c                   óÌ  ‡ — e Zd Zˆ fd„Z eej                  d«      «       eee	e
eeee¬«      	 	 	 	 	 	 	 	 	 	 	 ddeej"                     deej"                     deej"                     deej"                     deej"                     d	eej"                     d
eej"                     deej"                     dee   dee   dee   deeej"                     e	f   fd„«       «       Zˆ xZS )ÚMobileBertForQuestionAnsweringc                 óè   •— t         ‰| �  |«       |j                  | _        t        |d¬«      | _        t        j                  |j                  |j                  «      | _        | j                  «        y rÜ  )
ru   rv   r÷  r•  r)   r   rž   r•   Ú
qa_outputsr™  rË   s     €r4   rv   z'MobileBertForQuestionAnswering.__init__D  sU   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒä)¨&ÀEÔJˆŒÜŸ)™) F×$6Ñ$6¸×8IÑ8IÓJˆŒð 	�‰Õr~   r¤  )r¦  r§  r‹  Úqa_target_start_indexÚqa_target_end_indexrá  râ  rª   rÕ   r«   r�   rÖ   r¬   Ústart_positionsÚend_positionsr×   rO  rP  r€   c                 ó(  — |�|n| j                   j                  }| j                  |||||||	|
|¬«	      }|d   }| j                  |«      }|j	                  dd¬«      \  }}|j                  d«      j                  «       }|j                  d«      j                  «       }d}|�·|�µt        |j                  «       «      dkD  r|j                  d«      }t        |j                  «       «      dkD  r|j                  d«      }|j                  d«      }|j                  d|«      }|j                  d|«      }t        |¬«      } |||«      } |||«      }||z   dz  }|s||f|dd z   }|�|f|z   S |S t        ||||j                  |j                  ¬	«      S )
a  
        start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for position (index) of the start of the labelled span for computing the token classification loss.
            Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
            are not taken into account for computing the loss.
        end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for position (index) of the end of the labelled span for computing the token classification loss.
            Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
            are not taken into account for computing the loss.
        NrÒ  r   r   r‘   r³   )Úignore_indexr?   )r�  Ústart_logitsÚ
end_logitsrô   rU  )ra   rª  r)   r  rQ   rÿ  rÞ   rW   rµ   Úclampr	   r   rô   rU  )rz   rª   rÕ   r«   r�   rÖ   r¬   r	  r
  r×   rO  rP  ré   rv  rå  r  r  rÓ  Úignored_indexrÔ  Ú
start_lossÚend_lossrü   s                          r4   r‚   z&MobileBertForQuestionAnswering.forwardN  sÃ  € ðD &1Ð%<‘kÀ$Ç+Á+×B]ÑB]ˆà—/‘/ØØ)Ø)Ø%ØØ'Ø/Ø!5Ø#ð "ó 

ˆð " !™*ˆà—‘ Ó1ˆØ#)§<¡<°°r <Ó#:Ñ ˆ�jØ#×+Ñ+¨BÓ/×:Ñ:Ó<ˆØ×'Ñ'¨Ó+×6Ñ6Ó8ˆ
àˆ
ØÐ&¨=Ð+Dä�?×'Ñ'Ó)Ó*¨QÒ.Ø"1×"9Ñ"9¸"Ó"=�Ü�=×%Ñ%Ó'Ó(¨1Ò,Ø -× 5Ñ 5°bÓ 9�à(×-Ñ-¨aÓ0ˆMØ-×3Ñ3°A°}ÓEˆOØ)×/Ñ/°°=ÓAˆMä'°]ÔCˆHÙ! ,°Ó@ˆJÙ 
¨MÓ:ˆHØ$ xÑ/°1Ñ4ˆJáØ" JÐ/°'¸!¸"°+Ñ=ˆFØ/9Ð/E�Z�M FÑ*ÐQÈ6ÐQä+ØØ%Ø!Ø!×/Ñ/Ø×)Ñ)ô
ð 	
r~   r×  )r„   r…   r†   rv   r   r²  r³  r   Ú_CHECKPOINT_FOR_QAr   rµ  Ú_QA_TARGET_START_INDEXÚ_QA_TARGET_END_INDEXÚ_QA_EXPECTED_OUTPUTÚ_QA_EXPECTED_LOSSr   r]   r‡   rë   r   r   r‚   rˆ   r‰   s   @r4   r  r  ;  s`  ø„ ôñ +Ð+F×+MÑ+MÐNkÓ+lÓmÙØ%Ø0Ø$Ø4Ø0Ø+Ø'ôð -1Ø15Ø15Ø/3Ø,0Ø04Ø26Ø04Ø,0Ø/3Ø&*ñH
à˜EŸL™LÑ)ðH
ð ! §¡Ñ.ðH
ð ! §¡Ñ.ð	H
ð
 ˜uŸ|™|Ñ,ðH
ð ˜EŸL™LÑ)ðH
ð   §¡Ñ-ðH
ð " %§,¡,Ñ/ðH
ð   §¡Ñ-ðH
ð $ D™>ðH
ð ' t™nðH
ð ˜d‘^ðH
ð 
ˆu�U—\‘\Ñ"Ð$@Ð@Ñ	AòH
óó nôH
r~   r  z«
    MobileBert Model with a multiple choice classification head on top (a linear layer on top of the pooled output and
    a softmax) e.g. for RocStories/SWAG tasks.
    c                   ó¤  ‡ — e Zd Zˆ fd„Z eej                  d«      «       eee	e
¬«      	 	 	 	 	 	 	 	 	 	 ddeej                     deej                     deej                     deej                     deej                     d	eej                     d
eej                     dee   dee   dee   deeej                     e	f   fd„«       «       Zˆ xZS )ÚMobileBertForMultipleChoicec                 ó*  •— t         ‰| �  |«       t        |«      | _        |j                  �|j                  n|j
                  }t        j                  |«      | _        t        j                  |j                  d«      | _        | j                  «        y r=  )ru   rv   r•  r)   rø  r£   r   r¢   r¤   rž   r•   r>   r™  rù  s      €r4   rv   z$MobileBertForMultipleChoice.__init__¬  su   ø€ Ü‰Ñ˜Ô ä)¨&Ó1ˆŒà)/×)BÑ)BÐ)NˆF×%Ò%ÐTZ×TnÑTnð 	ô —z‘zÐ"4Ó5ˆŒÜŸ)™) F×$6Ñ$6¸Ó:ˆŒð 	�‰Õr~   z(batch_size, num_choices, sequence_lengthr¥  rª   rÕ   r«   r�   rÖ   r¬   rÏ  r×   rO  rP  r€   c                 óL  — |
�|
n| j                   j                  }
|�|j                  d   n|j                  d   }|�!|j                  d|j	                  d«      «      nd}|�!|j                  d|j	                  d«      «      nd}|�!|j                  d|j	                  d«      «      nd}|�!|j                  d|j	                  d«      «      nd}|�1|j                  d|j	                  d«      |j	                  d«      «      nd}| j                  ||||||||	|
¬«	      }|d   }| j                  |«      }| j                  |«      }|j                  d|«      }d}|�t        «       } |||«      }|
s|f|dd z   }|�|f|z   S |S t        |||j                  |j                  ¬«      S )aJ  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the multiple choice classification loss. Indices should be in `[0, ...,
            num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See
            `input_ids` above)
        Nr   r‘   rÙ   rÒ  r?   rä  )ra   rª  rZ   rÍ   rµ   r)   r¤   r>   r	   r   rô   rU  )rz   rª   rÕ   r«   r�   rÖ   r¬   rÏ  r×   rO  rP  Únum_choicesré   rb  rå  Úreshaped_logitsr�  rÔ  rü   s                      r4   r‚   z#MobileBertForMultipleChoice.forward¹  sÝ  € ð6 &1Ð%<‘kÀ$Ç+Á+×B]ÑB]ˆØ,5Ð,A�i—o‘o aÒ(À}×GZÑGZÐ[\ÑG]ˆà>GÐ>S�I—N‘N 2 y§~¡~°bÓ'9Ô:ÐY]ˆ	ØM[ÐMg˜×,Ñ,¨R°×1DÑ1DÀRÓ1HÔIÐmqˆØM[ÐMg˜×,Ñ,¨R°×1DÑ1DÀRÓ1HÔIÐmqˆØGSÐG_�|×(Ñ(¨¨\×->Ñ->¸rÓ-BÔCÐeiˆð Ð(ð ×Ñ˜r =×#5Ñ#5°bÓ#9¸=×;MÑ;MÈbÓ;QÔRàð 	ð —/‘/ØØ)Ø)Ø%ØØ'Ø/Ø!5Ø#ð "ó 

ˆð   ™
ˆàŸ™ ]Ó3ˆØ—‘ Ó/ˆØ Ÿ+™+ b¨+Ó6ˆàˆØÐÜ'Ó)ˆHÙ˜O¨VÓ4ˆDáØ%Ð'¨'°!°"¨+Ñ5ˆFØ)-Ð)9�T�G˜fÑ$ÐE¸vÐEä(ØØ"Ø!×/Ñ/Ø×)Ñ)ô	
ð 	
r~   ræ  )r„   r…   r†   rv   r   r²  r³  r   r´  r   rµ  r   r]   r‡   rë   r   r   r‚   rˆ   r‰   s   @r4   r  r  £  s@  ø„ ôñ +Ø#×*Ñ*Ð+UÓVóñ  Ø&Ø-Ø$ôð -1Ø15Ø15Ø/3Ø,0Ø04Ø)-Ø,0Ø/3Ø&*ñ@
à˜EŸL™LÑ)ð@
ð ! §¡Ñ.ð@
ð ! §¡Ñ.ð	@
ð
 ˜uŸ|™|Ñ,ð@
ð ˜EŸL™LÑ)ð@
ð   §¡Ñ-ð@
ð ˜Ÿ™Ñ&ð@
ð $ D™>ð@
ð ' t™nð@
ð ˜d‘^ð@
ð 
ˆu�U—\‘\Ñ"Ð$=Ð=Ñ	>ò@
óóô@
r~   r  z©
    MobileBert Model with a token classification head on top (a linear layer on top of the hidden-states output) e.g.
    for Named-Entity-Recognition (NER) tasks.
    c                   ó¨  ‡ — e Zd Zˆ fd„Z eej                  d«      «       eee	e
ee¬«      	 	 	 	 	 	 	 	 	 	 ddeej                     deej                     deej                     deej                     deej                     d	eej                     d
eej                     dee   dee   dee   deeej                     e	f   fd„«       «       Zˆ xZS )Ú MobileBertForTokenClassificationc                 ód  •— t         ‰| �  |«       |j                  | _        t        |d¬«      | _        |j
                  �|j
                  n|j                  }t        j                  |«      | _	        t        j                  |j                  |j                  «      | _        | j                  «        y rÜ  )ru   rv   r÷  r•  r)   rø  r£   r   r¢   r¤   rž   r•   r>   r™  rù  s      €r4   rv   z)MobileBertForTokenClassification.__init__  sŠ   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒä)¨&ÀEÔJˆŒà)/×)BÑ)BÐ)NˆF×%Ò%ÐTZ×TnÑTnð 	ô —z‘zÐ"4Ó5ˆŒÜŸ)™) F×$6Ñ$6¸×8IÑ8IÓJˆŒð 	�‰Õr~   r¤  rà  rª   rÕ   r«   r�   rÖ   r¬   rÏ  r×   rO  rP  r€   c                 ó¨  — |
�|
n| j                   j                  }
| j                  ||||||||	|
¬«	      }|d   }| j                  |«      }| j	                  |«      }d}|�<t        «       } ||j                  d| j                  «      |j                  d«      «      }|
s|f|dd z   }|�|f|z   S |S t        |||j                  |j                  ¬«      S )zÛ
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`.
        NrÒ  r   r‘   r?   rä  )ra   rª  r)   r¤   r>   r	   rÍ   r÷  r   rô   rU  )rz   rª   rÕ   r«   r�   rÖ   r¬   rÏ  r×   rO  rP  ré   rv  rå  r�  rÔ  rü   s                    r4   r‚   z(MobileBertForTokenClassification.forward  sö   € ð2 &1Ð%<‘kÀ$Ç+Á+×B]ÑB]ˆà—/‘/ØØ)Ø)Ø%ØØ'Ø/Ø!5Ø#ð "ó 

ˆð " !™*ˆàŸ,™, Ó7ˆØ—‘ Ó1ˆàˆØÐÜ'Ó)ˆHÙ˜FŸK™K¨¨D¯O©OÓ<¸f¿k¹kÈ"»oÓNˆDáØ�Y ¨¨ Ñ,ˆFØ)-Ð)9�T�G˜fÑ$ÐE¸vÐEä$ØØØ!×/Ñ/Ø×)Ñ)ô	
ð 	
r~   ræ  )r„   r…   r†   rv   r   r²  r³  r   Ú$_CHECKPOINT_FOR_TOKEN_CLASSIFICATIONr   rµ  Ú_TOKEN_CLASS_EXPECTED_OUTPUTÚ_TOKEN_CLASS_EXPECTED_LOSSr   r]   r‡   rë   r   r   r‚   rˆ   r‰   s   @r4   r  r    s6  ø„ ôñ +Ð+F×+MÑ+MÐNkÓ+lÓmÙØ7Ø)Ø$Ø4Ø0ôð -1Ø15Ø15Ø/3Ø,0Ø04Ø)-Ø,0Ø/3Ø&*ñ2
à˜EŸL™LÑ)ð2
ð ! §¡Ñ.ð2
ð ! §¡Ñ.ð	2
ð
 ˜uŸ|™|Ñ,ð2
ð ˜EŸL™LÑ)ð2
ð   §¡Ñ-ð2
ð ˜Ÿ™Ñ&ð2
ð $ D™>ð2
ð ' t™nð2
ð ˜d‘^ð2
ð 
ˆu�U—\‘\Ñ"Ð$9Ð9Ñ	:ò2
óó nô2
r~   r  )rÚ  r  rì  r·  r  rõ  r  r;  r•  r�  rp   )]rÛ   rG   rï  Údataclassesr   Útypingr   r   r   r]   r   Útorch.nnr   r	   r
   Úactivationsr   Úmodeling_outputsr   r   r   r   r   r   r   r   Úmodeling_utilsr   Úpytorch_utilsr   r   Úutilsr   r   r   r   r   r   Úconfiguration_mobilebertr   Ú
get_loggerr„   rE   r´  rµ  r"  r#  r$  r  r  r  r  r  r   r  r  rp   ÚModulerr   r&   r    r�   rÁ   rí   rú   r	  r  r  r   r&  r1  r5  r;  rK  r[  rd  ri  rs  rz  r�  r�  ÚMOBILEBERT_START_DOCSTRINGr²  r•  r·  rÚ  rè  rì  rõ  r  r  r  Ú__all__r1   r~   r4   ú<module>r2     sÃ  ðó. Û 	Û Ý !ß )Ñ )ã Ý ß AÑ Aå !÷	÷ 	ó 	õ .ß Q÷÷ õ 7ð 
ˆ×	Ñ	˜HÓ	%€à1Ð Ø$€ð (JÐ $ØlÐ Ø!Ð ð ;Ð Ø'Ð ØÐ ØÐ ØÐ ð +DÐ 'Ø'Ð Ø!Ð òKô\6ˆR�Y‰Yô 6ð Ÿ™°&Ñ
9€ôI˜2Ÿ9™9ô IôX7˜bŸi™iô 7ôt˜2Ÿ9™9ô ô"/˜"Ÿ)™)ô /ôd˜RŸY™Yô ô�r—y‘yô ô�r—y‘yô ô0	�b—i‘iô 	ô!]�—‘ô !]ôH	�—	‘	ô 	ô	ˆr�y‰yô 	ô<�b—i‘iô <ô~'
˜Ÿ	™	ô '
ôT!�r—y‘yô !ô&¨¯	©	ô ô" §¡ô ô,!˜BŸI™Iô !ô	9 §¡ô 	9ô% ô %ð: ô: [ó :ó ð:ðBÐ ð /Ð ñd ØjØóôh
Ð/ó h
ó	ðh
ñV ðð óôi
Ð8ó i
óði
ñX ÐTÐVpÓqôT
Ð5ó T
ó rðT
ôn&˜BŸI™Iô &ñ ØZØóô]
Ð*Có ]
ó	ð]
ñ@ ðð óô]
Ð*Có ]
óð]
ñ@ ðð óô]
Ð%>ó ]
óð]
ñ@ ðð óôV
Ð";ó V
óðV
ñr ðð óôI
Ð'@ó I
óðI
òX�r~   