Ë
    T^(hy ã                  ó’  — d Z ddlmZ ddlZddlmZmZmZmZ ddl	Z
ddlZddlmZ ddlmZmZ ddlmZmZmZmZmZ dd	lmZmZ dd
lmZmZmZmZ ddl m!Z!  ejD                  e#«      Z$dZ%dZ&d„ Z'd„ Z(	 d@	 	 	 	 	 	 	 	 	 dAd„Z)dBdCd„Z* G d„ dejV                  jX                  «      Z- G d„ dejV                  j\                  «      Z/ G d„ dejV                  jX                  «      Z0 G d„ dejV                  jX                  «      Z1 G d„ dejV                  jX                  «      Z2 G d„ dejV                  jX                  «      Z3 G d„ d ejV                  jX                  «      Z4 G d!„ d"ejV                  jX                  «      Z5 G d#„ d$e5«      Z6 G d%„ d&ejV                  jX                  «      Z7 G d'„ d(ejV                  jX                  «      Z8 G d)„ d*ejV                  jX                  «      Z9 G d+„ d,ejV                  jX                  «      Z: G d-„ d.ejV                  jX                  «      Z; G d/„ d0ejV                  jX                  «      Z< G d1„ d2ejV                  jX                  «      Z=e G d3„ d4ejV                  jX                  «      «       Z> G d5„ d6e«      Z?d7Z@d8ZA ed9e@«       G d:„ d;e?«      «       ZB ed<e@«       G d=„ d>e?«      «       ZCg d?¢ZDy)DzTensorFlow Hubert model.é    )ÚannotationsN)ÚAnyÚOptionalÚTupleÚUnioné   )Úget_tf_activation)ÚTFBaseModelOutputÚTFCausalLMOutput)ÚTFPreTrainedModelÚget_initializerÚkerasÚkeras_serializableÚunpack_inputs)Ú
shape_listÚstable_softmax)Úadd_start_docstringsÚ%add_start_docstrings_to_model_forwardÚloggingÚreplace_return_docstringsé   )ÚHubertConfigr   g    „×—Ác                óâ   — t         j                  j                  t         j                  j	                  t        | «      dd«      «       }t         j                  j                  | |z   |«      \  }}|S )zÂ
    Categorical sampling without replacement is currently not implemented. The gumbel-max trick will do for now - see
    https://github.com/tensorflow/tensorflow/issues/9260 for more info
    r   r   )ÚtfÚmathÚlogÚrandomÚuniformr   ÚnnÚtop_k)ÚdistributionÚnum_samplesÚzÚ_Úindicess        úk/var/www/skyplay_api_hub/venv/lib/python3.12/site-packages/transformers/models/hubert/modeling_tf_hubert.pyÚ_sample_without_replacementr'   5   sS   € ô
 
�‰�‰”R—Y‘Y×&Ñ&¤z°,Ó'?ÀÀAÓFÓ	GÐG€AÜ—‘—‘˜\¨AÑ-¨{Ó;�J€A€wØ€Nó    c           
     ó   — t        |«      }t        j                  t        j                  t        j                  t        j
                  |d   «      d¬«      |«      ddg«      }t        j                  t        j                  |t        j                  |ddg«      gd«      «      }t        j                  |t        j                  | dg«      |«      S )zT
    Scatter function as in PyTorch with indices in format (batch_dim, indixes)
    r   éÿÿÿÿ©Úaxisr   )	r   r   ÚreshapeÚbroadcast_toÚexpand_dimsÚrangeÚ	transposeÚconcatÚ
scatter_nd)ÚvaluesÚbatch_indicesÚoutput_shapeÚindices_shapeÚbroad_casted_batch_dimsÚpair_indicess         r&   Ú _scatter_values_on_batch_indicesr:   @   s¡   € ô ˜}Ó-€Mä Ÿj™jÜ
�‰œŸ™¤r§x¡x°¸aÑ0@Ó'AÈÔKÈ]Ó[Ð^_ÐacÐ]dóÐô —<‘<¤§	¡	Ð+BÄBÇJÁJÈ}Ð_`ÐbdÐ^eÓDfÐ*gÐijÓ kÓl€Lä�=‰=˜¤r§z¡z°&¸2¸$Ó'?ÀÓNÐNr(   c           	     óx  — | \  }}|dk  rt        d«      ‚t        j                  j                  ||d|› d|› d�¬«       |t        j                  |t        j
                  «      z  |z  t        j                  j                  d«      z   }t        j                  ||«      }t        j                  |t        j                  «      }t        j                  j                  ||z  |«      }t        j                  |«      }t        j                  ||ft        j                  ¬«      }t        j                  |||dz
  z
  f«      }t        ||«      }	t        j                   |	d	«      }	t        j"                  |	dd|f«      }	t        j$                  |	|||z  f«      }	t        j&                  |«      t        j(                  t        j(                  d
d
…f   }
t        j"                  |
||df«      }
t        j$                  |
|||z  f«      }
|	|
z   }	t+        t        j,                  |	«      |	t        j.                  |«      «      }|S )aš  
    Computes random mask spans for a given shape

    Args:
        shape: the shape for which to compute masks.
            should be of size 2 where first element is batch size and 2nd is timesteps
        attention_mask: optional padding mask of the same size as shape, which will prevent masking padded elements
        mask_prob:
            probability for each token to be chosen as start of the span to be masked. this will be multiplied by
            number of timesteps divided by length of mask span to mask approximately this percentage of all elements.
            however due to overlaps, the actual number will be smaller (unless no_overlap is True)
        mask_length: size of the mask
        min_masks: minimum number of masked spans

    Adapted from [fairseq's
    data_utils.py](https://github.com/pytorch/fairseq/blob/e0788f7007a8473a76db573985031f3c94201e79/fairseq/data/data_utils.py#L376).
    r   z&`mask_length` has to be bigger than 0.zO`mask_length` has to be smaller than `sequence_length`, but got `mask_length`: z and `sequence_length`: ú`©Úmessage©r   ©Údtyper*   N)Ú
ValueErrorr   Ú	debuggingÚassert_lessÚcastÚfloat32r   r   ÚmaximumÚint32r   ÚminimumÚsqueezeÚzerosÚonesr'   r/   Útiler-   r0   Únewaxisr:   Ú	ones_likeÚshape)rP   Ú	mask_probÚmask_lengthÚ	min_masksÚ
batch_sizeÚsequence_lengthÚnum_masked_spansÚspec_aug_maskÚuniform_distÚspec_aug_mask_idxsÚoffsetss              r&   Ú_compute_mask_indicesr[   P   s  € ð. #(Ñ€J�à�Q‚ÜÐAÓBÐBä‡L�L×ÑØØà]Ð^iÐ]jð k#Ø#2Ð"3°1ð6ð	 ô ð !¤2§7¡7¨?¼B¿J¹JÓ#GÑGÈ+ÑUÔXZ×XaÑXa×XiÑXiÐjnÓXoÑoÐÜ—z‘zÐ"2°IÓ>ÐÜ—w‘wÐ/´·±Ó:Ðô —w‘w—‘ ¸+Ñ'EÐGWÓXÐÜ—z‘zÐ"2Ó3Ðô —H‘H˜j¨/Ð:Ä"Ç(Á(ÔK€Mô —7‘7˜J¨¸;È¹?Ñ(KÐLÓM€Lô 5°\ÐCSÓTÐô Ÿ™Ð(:¸BÓ?ÐÜŸ™Ð!3°a¸¸KÐ5HÓIÐÜŸ™Ð$6¸ÐEUÐXcÑEcÐ8dÓeÐä�h‰h�{Ó#¤B§J¡J´·
±
ºAÐ$=Ñ>€GÜ�g‰g�g 
Ð,<¸aÐ@ÓA€GÜ�j‰j˜ :Ð/?À+Ñ/MÐ"NÓO€Gà+¨gÑ5Ðô 5Ü
�‰Ð'Ó(Ð*<¼b¿h¹hÀ}Ó>Uó€Mð Ðr(   c                óø   — t        | «      d   }|�|n|}t        j                  d«      }t        j                  | |j                  ¬«      } t        j
                  | dd…dddd…f   dd|df«      }||z
  t        z  S )z_
    Expands attention_mask from `[bsz, seq_len]` to `[bsz, 1, tgt_seq_len, src_seq_len]`.
    r   Ng      ð?r@   )r   r   ÚconstantrE   rA   rM   ÚLARGE_NEGATIVE)ÚmaskÚtgt_lenÚsrc_lenÚone_cstÚexpanded_masks        r&   Ú_expand_maskrd   ›   sx   € ô ˜Ó˜qÑ!€GØ Ð,‰g°'€GÜ�k‰k˜#Ó€GÜ�7‰7�4˜wŸ}™}Ô-€DÜ—G‘G˜D¢ D¨$²Ð!1Ñ2°Q¸¸7ÀAÐ4FÓG€Mà�mÑ#¤~Ñ5Ð5r(   c                  óÈ   ‡ — e Zd ZdZ	 	 	 	 	 	 	 	 	 	 	 d	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dˆ fd„Zˆ fd„Zd„ Zˆ fd„Zd„ Zd„ Z	d„ Z
d	„ Zd
„ Zd„ Zd„ Zd„ Zd„ Zd„ Zd„ Zd„ Zˆ xZS )ÚTFHubertGroupNormzp
    From tensorflow-addons https://www.tensorflow.org/addons/api_docs/python/tfa/layers/GroupNormalization
    c                óH  •— t        ‰| �  di |¤Ž d| _        || _        || _        || _        || _        || _        t        j                  j                  |«      | _        t        j                  j                  |«      | _        t        j                  j                  |«      | _        t        j                  j                  |	«      | _        t        j                   j                  |
«      | _        t        j                   j                  |«      | _        | j'                  «        y )NT© )ÚsuperÚ__init__Úsupports_maskingÚgroupsr,   ÚepsilonÚcenterÚscaler   ÚinitializersÚgetÚbeta_initializerÚgamma_initializerÚregularizersÚbeta_regularizerÚgamma_regularizerÚconstraintsÚbeta_constraintÚgamma_constraintÚ_check_axis)Úselfrl   r,   rm   rn   ro   rr   rs   ru   rv   rx   ry   ÚkwargsÚ	__class__s                €r&   rj   zTFHubertGroupNorm.__init__®   sç   ø€ ô 	‰ÑÑ"˜6Ò"Ø $ˆÔØˆŒØˆŒ	ØˆŒØˆŒØˆŒ
Ü %× 2Ñ 2× 6Ñ 6Ð7GÓ HˆÔÜ!&×!3Ñ!3×!7Ñ!7Ð8IÓ!JˆÔÜ %× 2Ñ 2× 6Ñ 6Ð7GÓ HˆÔÜ!&×!3Ñ!3×!7Ñ!7Ð8IÓ!JˆÔÜ$×0Ñ0×4Ñ4°_ÓEˆÔÜ %× 1Ñ 1× 5Ñ 5Ð6FÓ GˆÔØ×ÑÕr(   c                óþ   •— | j                  |«       | j                  |«       | j                  |«       | j                  |«       | j	                  |«       | j                  |«       d| _        t        ‰| �!  |«       y ©NT)	Ú_check_if_input_shape_is_noneÚ'_set_number_of_groups_for_instance_normÚ_check_size_of_dimensionsÚ_create_input_specÚ_add_gamma_weightÚ_add_beta_weightÚbuiltri   Úbuild©r{   Úinput_shaper}   s     €r&   r‡   zTFHubertGroupNorm.buildÌ   sj   ø€ Ø×*Ñ*¨;Ô7Ø×4Ñ4°[ÔAØ×&Ñ& {Ô3Ø×Ñ Ô,à×Ñ˜{Ô+Ø×Ñ˜kÔ*ØˆŒ
Ü‰‰�kÕ"r(   c                ó4  — t         j                  j                  |«      }t        j                  |«      }| j                  |||«      \  }}| j                  ||«      }|| j                     | j                  z  dk(  }|st        j                  ||«      }|S |}|S ©Nr   )
r   ÚbackendÚ	int_shaper   rP   Ú_reshape_into_groupsÚ_apply_normalizationr,   rl   r-   )	r{   Úinputsr‰   Útensor_input_shapeÚreshaped_inputsÚgroup_shapeÚnormalized_inputsÚis_instance_normÚoutputss	            r&   ÚcallzTFHubertGroupNorm.call×   s—   € Ü—m‘m×-Ñ-¨fÓ5ˆÜŸX™X fÓ-Ðà'+×'@Ñ'@ÀÈÐVhÓ'iÑ$ˆ˜à ×5Ñ5°oÀ{ÓSÐà'¨¯	©	Ñ2°d·k±kÑAÀaÑGÐÙÜ—j‘jÐ!2Ð4FÓGˆGð ˆð (ˆGàˆr(   c                ó€  •— | j                   | j                  | j                  | j                  | j                  t
        j                  j                  | j                  «      t
        j                  j                  | j                  «      t
        j                  j                  | j                  «      t
        j                  j                  | j                  «      t
        j                  j                  | j                  «      t
        j                  j                  | j                  «      dœ}t         ‰| �E  «       }i |¥|¥S )N)rl   r,   rm   rn   ro   rr   rs   ru   rv   rx   ry   )rl   r,   rm   rn   ro   r   rp   Ú	serializerr   rs   rt   ru   rv   rw   rx   ry   ri   Ú
get_config)r{   ÚconfigÚbase_configr}   s      €r&   rš   zTFHubertGroupNorm.get_configç   sè   ø€ à—k‘kØ—I‘IØ—|‘|Ø—k‘kØ—Z‘ZÜ %× 2Ñ 2× <Ñ <¸T×=RÑ=RÓ SÜ!&×!3Ñ!3×!=Ñ!=¸d×>TÑ>TÓ!UÜ %× 2Ñ 2× <Ñ <¸T×=RÑ=RÓ SÜ!&×!3Ñ!3×!=Ñ!=¸d×>TÑ>TÓ!UÜ$×0Ñ0×:Ñ:¸4×;OÑ;OÓPÜ %× 1Ñ 1× ;Ñ ;¸D×<QÑ<QÓ Rñ
ˆô ‘gÑ(Ó*ˆØ(�+Ð( Ð(Ð(r(   c                ó   — |S ©Nrh   ©r{   r‰   s     r&   Úcompute_output_shapez&TFHubertGroupNorm.compute_output_shapeø   s   € ØÐr(   c                óš  — t        t        |«      «      D �cg c]  }||   ‘Œ	 }}|| j                     | j                  z  dk(  }|s~|| j                     | j                  z  || j                  <   |j	                  | j                  | j                  «       t        j                  |«      }t        j                  ||«      }||fS ||fS c c}w r‹   )r0   Úlenr,   rl   Úinsertr   Ústackr-   )r{   r�   r‰   r‘   Úir“   r•   r’   s           r&   rŽ   z&TFHubertGroupNorm._reshape_into_groupsû   s·   € Ü6;¼CÀÓ<LÓ6MÖN°Ð)¨!Ó,ÐNˆÐNØ'¨¯	©	Ñ2°d·k±kÑAÀaÑGÐÙØ%0°·±Ñ%;¸t¿{¹{Ñ%JˆK˜Ÿ	™	Ñ"Ø×Ñ˜tŸy™y¨$¯+©+Ô6ÜŸ(™( ;Ó/ˆKÜ Ÿj™j¨°Ó=ˆOØ" KÐ/Ð/à˜;Ð&Ð&ùò Os   —Cc                ó6  — t         j                  j                  |«      }t        t	        dt        |«      «      «      }|| j                     | j                  z  dk(  }|s!| j                  dk(  rdn| j                  dz
  }n | j                  dk(  rdn| j                  dz
  }|j                  |«       t        j                  j                  ||d¬«      \  }}| j                  |«      \  }	}
t        j                  j                  ||||	|
| j                  ¬«      }|S )Nr   r*   éþÿÿÿT)Úkeepdims)ÚmeanÚvariancero   ÚoffsetÚvariance_epsilon)r   rŒ   r�   Úlistr0   r¢   r,   rl   Úpopr   r   ÚmomentsÚ_get_reshaped_weightsÚbatch_normalizationrm   )r{   r’   r‰   r“   Úgroup_reduction_axesr•   r,   r©   rª   ÚgammaÚbetar”   s               r&   r�   z&TFHubertGroupNorm._apply_normalization  s÷   € Ü—m‘m×-Ñ-¨oÓ>ˆÜ#¤E¨!¬S°Ó-=Ó$>Ó?ÐØ'¨¯	©	Ñ2°d·k±kÑAÀaÑGÐÙØŸ™ bš‘2¨d¯i©i¸!©m‰DàŸ™ bš‘2¨d¯i©i¸!©mˆDØ× Ñ  Ô&äŸ™Ÿ™ Ð8LÐW[˜Ó\‰ˆˆhà×0Ñ0°Ó=‰ˆˆtÜŸE™E×5Ñ5ØØØØØØ!Ÿ\™\ð 6ó 
Ðð !Ð r(   c                óä   — | j                  |«      }d }d }| j                  r t        j                  | j                  |«      }| j
                  r t        j                  | j                  |«      }||fS rž   )Ú_create_broadcast_shapero   r   r-   r³   rn   r´   )r{   r‰   Úbroadcast_shaper³   r´   s        r&   r°   z'TFHubertGroupNorm._get_reshaped_weights  s\   € Ø×6Ñ6°{ÓCˆØˆØˆØ�:Š:Ü—J‘J˜tŸz™z¨?Ó;ˆEà�;Š;Ü—:‘:˜dŸi™i¨Ó9ˆDØ�dˆ{Ðr(   c                óŒ   — || j                      }|€3t        dt        | j                   «      z   dz   t        |«      z   dz   «      ‚y )NzAxis z\ of input tensor should have a defined dimension but the layer received an input with shape ú.)r,   rB   Ústr©r{   r‰   Údims      r&   r€   z/TFHubertGroupNorm._check_if_input_shape_is_none)  s\   € Ø˜$Ÿ)™)Ñ$ˆØˆ;ÜØÜ�d—i‘i“.ñ!àpñqô �kÓ"ñ#ð ñ	óð ð r(   c                óP   — || j                      }| j                  dk(  r|| _        y y ©Nr*   )r,   rl   r»   s      r&   r�   z9TFHubertGroupNorm._set_number_of_groups_for_instance_norm4  s(   € Ø˜$Ÿ)™)Ñ$ˆà�;‰;˜"ÒØˆD�Kð r(   c                ó0  — || j                      }|| j                  k  r3t        dt        | j                  «      z   dz   t        |«      z   dz   «      ‚|| j                  z  dk7  r3t        dt        | j                  «      z   dz   t        |«      z   dz   «      ‚y )NzNumber of groups (z.) cannot be more than the number of channels (ú).r   z0) must be a multiple of the number of channels ()r,   rl   rB   rº   r»   s      r&   r‚   z+TFHubertGroupNorm._check_size_of_dimensions:  s¸   € Ø˜$Ÿ)™)Ñ$ˆØ�—‘ÒÜØ$Ü�d—k‘kÓ"ñ#àBñCô �c“(ñð ñ	óð ð �—‘Ñ Ò!ÜØ$Ü�d—k‘kÓ"ñ#àDñEô �c“(ñð ñ	óð ð "r(   c                ó8   — | j                   dk(  rt        d«      ‚y )Nr   zdYou are trying to normalize your batch axis. Do you want to use tf.layer.batch_normalization instead)r,   rB   ©r{   s    r&   rz   zTFHubertGroupNorm._check_axisN  s"   € Ø�9‰9˜Š>ÜØvóð ð r(   c                ó˜   — || j                      }t        j                  j                  t	        |«      | j                   |i¬«      | _        y )N)ÚndimÚaxes)r,   r   ÚlayersÚ	InputSpecr¢   Ú
input_specr»   s      r&   rƒ   z$TFHubertGroupNorm._create_input_specT  s:   € Ø˜$Ÿ)™)Ñ$ˆÜŸ,™,×0Ñ0´c¸+Ó6FÈdÏiÉiÐY\ÐM]Ð0Ó^ˆ�r(   c                óÂ   — || j                      }|f}| j                  r:| j                  |d| j                  | j                  | j
                  ¬«      | _        y d | _        y )Nr³   ©rP   ÚnameÚinitializerÚregularizerÚ
constraint)r,   ro   Ú
add_weightrs   rv   ry   r³   ©r{   r‰   r¼   rP   s       r&   r„   z#TFHubertGroupNorm._add_gamma_weightX  s]   € Ø˜$Ÿ)™)Ñ$ˆØ�ˆà�:Š:ØŸ™ØØØ ×2Ñ2Ø ×2Ñ2Ø×0Ñ0ð )ó ˆD�Jð ˆD�Jr(   c                óÂ   — || j                      }|f}| j                  r:| j                  |d| j                  | j                  | j
                  ¬«      | _        y d | _        y )Nr´   rÊ   )r,   rn   rÏ   rr   ru   rx   r´   rÐ   s       r&   r…   z"TFHubertGroupNorm._add_beta_weightg  s]   € Ø˜$Ÿ)™)Ñ$ˆØ�ˆà�;Š;ØŸ™ØØØ ×1Ñ1Ø ×1Ñ1Ø×/Ñ/ð (ó ˆD�Ið ˆD�Ir(   c                ó:  — dgt        |«      z  }|| j                     | j                  z  dk(  }|sQ|| j                     | j                  z  || j                  <   |j                  | j                  | j                  «       |S | j                  || j                  <   |S r‹   )r¢   r,   rl   r£   )r{   r‰   r·   r•   s       r&   r¶   z)TFHubertGroupNorm._create_broadcast_shapev  s�   € Ø˜#¤ KÓ 0Ñ0ˆØ'¨¯	©	Ñ2°d·k±kÑAÀaÑGÐÙØ)4°T·Y±YÑ)?À4Ç;Á;Ñ)NˆO˜DŸI™IÑ&Ø×"Ñ" 4§9¡9¨d¯k©kÔ:ð Ðð *.¯©ˆO˜DŸI™IÑ&ØÐr(   )é    r*   gü©ñÒMbP?TTrK   rL   NNNN)rl   Úintr,   rÔ   rm   Úfloatrn   Úboolro   rÖ   rr   úkeras.initializers.Initializerrs   r×   ru   úkeras.regularizers.Regularizerrv   rØ   rx   úkeras.constraints.Constraintry   rÙ   )Ú__name__Ú
__module__Ú__qualname__Ú__doc__rj   r‡   r—   rš   r    rŽ   r�   r°   r€   r�   r‚   rz   rƒ   r„   r…   r¶   Ú__classcell__©r}   s   @r&   rf   rf   ©   sè   ø„ ñð ØØØØØ;BØ<BØ;?Ø<@Ø8<Ø9=ðàðð ðð ð	ð
 ðð ðð 9ðð :ðð 9ðð :ðð 6ðð 7õô<	#òô )ò"ò
'ò!ò.	ò	òòò(ò_òòör(   rf   c                  óB   ‡ — e Zd ZdZˆ fd„Zd„ Zd„ Zˆ fd„Zˆ fd„Zˆ xZ	S )ÚTFHubertWeightNormConv1DzeAdapted from https://www.tensorflow.org/probability/api_docs/python/tfp/layers/weight_norm/WeightNormc           
     ó†   •— t        ‰| �  d|||ddddœ|¤Ž || _        d| _        t	        j
                  ddg«      | _        y )	NÚvalidTÚ	he_normal)ÚfiltersÚkernel_sizerl   ÚpaddingÚuse_biasÚbias_initializeré   r   r   rh   )ri   rj   Úexplicit_paddingÚfilter_axisr   r]   Úkernel_norm_axes)r{   rå   ræ   rl   rë   r|   r}   s         €r&   rj   z!TFHubertWeightNormConv1D.__init__…  sX   ø€ Ü‰Ñð 	
ØØ#ØØØØ(ñ	
ð ò	
ð !1ˆÔØˆÔÜ "§¡¨Q°¨FÓ 3ˆÕr(   c                ó$  — t        j                  t        j                  t        j                  | j                  «      | j
                  ¬«      «      }| j                  j                  |dd…t         j                  t         j                  f   «       y)z"Set the norm of the weight vector.r+   N)	r   ÚsqrtÚ
reduce_sumÚsquareÚweight_vrí   Úweight_gÚassignrN   )r{   Úkernel_norms     r&   Ú
_init_normz#TFHubertWeightNormConv1D._init_norm“  sT   € ä—g‘gœbŸm™m¬B¯I©I°d·m±mÓ,DÈ4×K`ÑK`ÔaÓbˆØ�‰×Ñ˜[ª¬B¯J©J¼¿
¹
Ð)BÑCÕDr(   c                óâ   — t         j                  j                  | j                  | j                  ¬«      t        j
                  | j                  «      z  }t        j
                  |«      | _        y)zGenerate normalized weights.r+   N)r   r   Úl2_normalizerò   rí   r1   ró   Úkernel)r{   rù   s     r&   Ú_normalize_kernelz*TFHubertWeightNormConv1D._normalize_kernel˜  sM   € ä—‘×#Ñ# D§M¡M¸×8MÑ8MÐ#ÓNÔQS×Q]ÑQ]Ð^b×^kÑ^kÓQlÑlˆÜ—l‘l 6Ó*ˆ�r(   c                óö  •— | j                   sìt        ‰| �	  |«       t        j                  t        j
                  | j                  «      dd¬«      | _        | j                  | _        | j                  dt        | j                  j                  | j                     «      ddfd| j                  j                  d¬«      | _        | j                  «        | j                  d| j                  fd	d¬
«      | _        y y )Nrò   T)rË   Ú	trainableró   r   rL   )rË   rP   rÌ   rA   rü   ÚbiasrK   )rË   rP   rÌ   rü   )r†   ri   r‡   r   ÚVariabler1   rù   rò   rÏ   rÔ   rP   rì   rA   ró   rö   rå   rý   rˆ   s     €r&   r‡   zTFHubertWeightNormConv1D.build�  sÃ   ø€ Ø�zŠzÜ‰G‰M˜+Ô&äŸ+™+¤b§l¡l°4·;±;Ó&?ÀjÐ\`ÔaˆDŒKØ ŸK™KˆDŒMà ŸO™OØÜ˜4Ÿ=™=×.Ñ.¨t×/?Ñ/?Ñ@ÓAÀ1ÀaÐHØ"Ø—m‘m×)Ñ)Øð ,ó ˆDŒMð �O‰OÔØŸ™¨V¸D¿L¹L¸?ÐX_Ðko˜ÓpˆD�Ið r(   c                ó¤   •— | j                  «        t        j                  |d| j                  | j                  fdf«      }t        ‰| �  |«      }|S )N)r   r   )rú   r   Úpadrë   ri   r—   )r{   r�   Úpadded_inputsÚoutputr}   s       €r&   r—   zTFHubertWeightNormConv1D.call®  sM   ø€ ð 	×ÑÔ äŸ™˜v¨°×1FÑ1FÈ×H]ÑH]Ð0^Ð`fÐ'gÓhˆÜ‘‘˜mÓ,ˆàˆr(   )
rÚ   rÛ   rÜ   rÝ   rj   rö   rú   r‡   r—   rÞ   rß   s   @r&   rá   rá   ‚  s&   ø„ Ùoô4òEò
+ô
q÷"	ð 	r(   rá   c                  ó2   ‡ — e Zd Zddˆ fd„Zdd„Zdd„Zˆ xZS )ÚTFHubertNoLayerNormConvLayerc                ód  •— t        ‰| �  di |¤Ž |dkD  r|j                  |   nd| _        |j                  |   | _        t
        j                  j                  | j                  |j                  |   |j                  |   |j                  d¬«      | _        t        |j                  «      | _        y )Nr   r   Úconv©rå   ræ   Ústridesrè   rË   rh   )ri   rj   Úconv_dimÚin_conv_dimÚout_conv_dimr   rÆ   ÚConv1DÚconv_kernelÚconv_strideÚ	conv_biasr  r	   Úfeat_extract_activationÚ
activation©r{   r›   Úlayer_idr|   r}   s       €r&   rj   z%TFHubertNoLayerNormConvLayer.__init__¼  s�   ø€ Ü‰ÑÑ"˜6Ò"Ø8@À1º˜6Ÿ?™?¨8Ò4È!ˆÔØ"ŸO™O¨HÑ5ˆÔä—L‘L×'Ñ'Ø×%Ñ%Ø×*Ñ*¨8Ñ4Ø×&Ñ& xÑ0Ø×%Ñ%Øð (ó 
ˆŒ	ô ,¨F×,JÑ,JÓKˆ�r(   c                óJ   — | j                  |«      }| j                  |«      }|S rž   )r  r  ©r{   Úhidden_statess     r&   r—   z!TFHubertNoLayerNormConvLayer.callÊ  s$   € ØŸ	™	 -Ó0ˆØŸ™¨Ó6ˆØÐr(   c                ó  — | j                   ry d| _         t        | dd «      �[t        j                  | j                  j
                  «      5  | j                  j                  d d | j                  g«       d d d «       y y # 1 sw Y   y xY w©NTr  )r†   Úgetattrr   Ú
name_scoper  rË   r‡   r
  rŸ   s     r&   r‡   z"TFHubertNoLayerNormConvLayer.buildÏ  sw   € Ø�:Š:ØØˆŒ
Ü�4˜ Ó&Ð2Ü—‘˜tŸy™yŸ~™~Ó.ñ @Ø—	‘	—‘  t¨T×-=Ñ-=Ð >Ô?÷@ð @ð 3÷@ð @ús   Á)A>Á>B©r   ©r›   r   r  rÔ   r|   r   ÚreturnÚNone©r  ú	tf.Tensorr  r   rž   ©rÚ   rÛ   rÜ   rj   r—   r‡   rÞ   rß   s   @r&   r  r  »  s   ø„ öLó÷
@r(   r  c                  ó2   ‡ — e Zd Zddˆ fd„Zdd„Zdd„Zˆ xZS )ÚTFHubertLayerNormConvLayerc                óÄ  •— t        ‰| �  di |¤Ž |dkD  r|j                  |   nd| _        |j                  |   | _        t
        j                  j                  | j                  |j                  |   |j                  |   |j                  d¬«      | _        t
        j                  j                  d|j                  ¬«      | _        t        |j                   «      | _        y )Nr   r   r  r  Ú
layer_norm)rË   rm   rh   )ri   rj   r	  r
  r  r   rÆ   r  r  r  r  r  ÚLayerNormalizationÚlayer_norm_epsr%  r	   r  r  r  s       €r&   rj   z#TFHubertLayerNormConvLayer.__init__Ú  s¿   ø€ Ü‰ÑÑ"˜6Ò"Ø8@À1º˜6Ÿ?™?¨8Ò4È!ˆÔØ"ŸO™O¨HÑ5ˆÔä—L‘L×'Ñ'Ø×%Ñ%Ø×*Ñ*¨8Ñ4Ø×&Ñ& xÑ0Ø×%Ñ%Øð (ó 
ˆŒ	ô  Ÿ,™,×9Ñ9¸|ÐU[×UjÑUjÐ9ÓkˆŒÜ+¨F×,JÑ,JÓKˆ�r(   c                ól   — | j                  |«      }| j                  |«      }| j                  |«      }|S rž   ©r  r%  r  r  s     r&   r—   zTFHubertLayerNormConvLayer.callé  ó2   € ØŸ	™	 -Ó0ˆØŸ™¨Ó6ˆØŸ™¨Ó6ˆØÐr(   c                óú  — | j                   ry d| _         t        | dd «      �Zt        j                  | j                  j
                  «      5  | j                  j                  d d | j                  g«       d d d «       t        | dd «      �[t        j                  | j                  j
                  «      5  | j                  j                  d d | j                  g«       d d d «       y y # 1 sw Y   ŒrxY w# 1 sw Y   y xY w©NTr  r%  ©
r†   r  r   r  r  rË   r‡   r
  r%  r  rŸ   s     r&   r‡   z TFHubertLayerNormConvLayer.buildï  óÔ   € Ø�:Š:ØØˆŒ
Ü�4˜ Ó&Ð2Ü—‘˜tŸy™yŸ~™~Ó.ñ @Ø—	‘	—‘  t¨T×-=Ñ-=Ð >Ô?÷@ä�4˜ tÓ,Ð8Ü—‘˜tŸ™×3Ñ3Ó4ñ GØ—‘×%Ñ% t¨T°4×3DÑ3DÐ&EÔF÷Gð Gð 9÷@ð @ú÷Gð Gúó   Á)C%Â2)C1Ã%C.Ã1C:r  r  r  rž   r!  rß   s   @r&   r#  r#  Ù  s   ø„ öLó÷	Gr(   r#  c                  ó2   ‡ — e Zd Zddˆ fd„Zdd„Zdd„Zˆ xZS )ÚTFHubertGroupNormConvLayerc                ó²  •— t        ‰| �  di |¤Ž |dkD  r|j                  |   nd| _        |j                  |   | _        t
        j                  j                  | j                  |j                  |   |j                  |   |j                  d¬«      | _        t        |j                  «      | _        t        | j                  |j                   d¬«      | _        y )Nr   r   r  r  r%  )rl   rm   rË   rh   )ri   rj   r	  r
  r  r   rÆ   r  r  r  r  r  r	   r  r  rf   r'  r%  r  s       €r&   rj   z#TFHubertGroupNormConvLayer.__init__ý  sº   ø€ Ü‰ÑÑ"˜6Ò"Ø8@À1º˜6Ÿ?™?¨8Ò4È!ˆÔØ"ŸO™O¨HÑ5ˆÔä—L‘L×'Ñ'Ø×%Ñ%Ø×*Ñ*¨8Ñ4Ø×&Ñ& xÑ0Ø×%Ñ%Øð (ó 
ˆŒ	ô ,¨F×,JÑ,JÓKˆŒÜ+°4×3DÑ3DÈf×NcÑNcÐjvÔwˆ�r(   c                ól   — | j                  |«      }| j                  |«      }| j                  |«      }|S rž   r)  r  s     r&   r—   zTFHubertGroupNormConvLayer.call  r*  r(   c                óú  — | j                   ry d| _         t        | dd «      �Zt        j                  | j                  j
                  «      5  | j                  j                  d d | j                  g«       d d d «       t        | dd «      �[t        j                  | j                  j
                  «      5  | j                  j                  d d | j                  g«       d d d «       y y # 1 sw Y   ŒrxY w# 1 sw Y   y xY wr,  r-  rŸ   s     r&   r‡   z TFHubertGroupNormConvLayer.build  r.  r/  r  r  r  rž   r!  rß   s   @r&   r1  r1  ü  s   ø„ öxó÷	Gr(   r1  c                  ó0   ‡ — e Zd Zdˆ fd„Zdd„Zdd„Zˆ xZS )ÚTFHubertPositionalConvEmbeddingc                ó  •— t        ‰| �  di |¤Ž t        |j                  |j                  |j
                  |j                  dz  d¬«      | _        t        |j                  «      | _        t        |j                  «      | _        || _        y )Nrê   r  )rå   ræ   rl   rë   rË   rh   )ri   rj   rá   Úhidden_sizeÚnum_conv_pos_embeddingsÚnum_conv_pos_embedding_groupsr  ÚTFHubertSamePadLayerrç   r	   r  r  r›   ©r{   r›   r|   r}   s      €r&   rj   z(TFHubertPositionalConvEmbedding.__init__   sx   ø€ Ü‰ÑÑ"˜6Ò"Ü,Ø×&Ñ&Ø×6Ñ6Ø×7Ñ7Ø#×;Ñ;¸qÑ@Øô
ˆŒ	ô ,¨F×,JÑ,JÓKˆŒÜ+¨F×,JÑ,JÓKˆŒØˆ�r(   c                ól   — | j                  |«      }| j                  |«      }| j                  |«      }|S rž   )r  rç   r  r  s     r&   r—   z$TFHubertPositionalConvEmbedding.call-  s2   € ØŸ	™	 -Ó0ˆØŸ™ ]Ó3ˆØŸ™¨Ó6ˆØÐr(   c                ó(  — | j                   ry d| _         t        | dd «      �et        j                  | j                  j
                  «      5  | j                  j                  d d | j                  j                  g«       d d d «       y y # 1 sw Y   y xY wr  )	r†   r  r   r  r  rË   r‡   r›   r8  rŸ   s     r&   r‡   z%TFHubertPositionalConvEmbedding.build3  s{   € Ø�:Š:ØØˆŒ
Ü�4˜ Ó&Ð2Ü—‘˜tŸy™yŸ~™~Ó.ñ GØ—	‘	—‘  t¨T¯[©[×-DÑ-DÐ EÔF÷Gð Gð 3÷Gð Gús   Á3BÂB©r›   r   r|   r   r  r  r  rž   r!  rß   s   @r&   r6  r6    s   ø„ õó÷Gr(   r6  c                  ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )r;  c                óR   •— t        ‰| �  di |¤Ž |dz  dk(  rd| _        y d| _        y )Nrê   r   r   rh   )ri   rj   Únum_pad_remove)r{   r9  r|   r}   s      €r&   rj   zTFHubertSamePadLayer.__init__>  s.   ø€ Ü‰ÑÑ"˜6Ò"Ø#:¸QÑ#>À!Ò#C˜aˆÕÈˆÕr(   c                óV   — | j                   dkD  r|d d …d | j                    …d d …f   }|S )Nr   )rB  r  s     r&   r—   zTFHubertSamePadLayer.callB  s6   € Ø×Ñ Ò"Ø)ª!Ð-C°×0CÑ0CÐ/CÐ-CÂQÐ*FÑGˆMØÐr(   )rÚ   rÛ   rÜ   rj   r—   rÞ   rß   s   @r&   r;  r;  =  s   ø„ ôKör(   r;  c                  ó.   ‡ — e Zd Zdˆ fd„Zd„ Zdd„Zˆ xZS )ÚTFHubertFeatureEncoderc                óÂ  •— t        ‰| �  d	i |¤Ž |j                  dk(  rVt        |ddd› �¬«      gt	        |j
                  dz
  «      D �cg c]  }t        ||dz   d|dz   › �¬«      ‘Œ c}z   }|| _	        y |j                  dk(  r9t	        |j
                  «      D �cg c]  }t        ||d|› �¬«      ‘Œ }}|| _	        y t        d|j                  › d�«      ‚c c}w c c}w )
NÚgroupr   zconv_layers.)r  rË   r   Úlayerz`config.feat_extract_norm` is z), but has to be one of ['group', 'layer']rh   )
ri   rj   Úfeat_extract_normr1  r0   Únum_feat_extract_layersr  r#  rB   Úconv_layers)r{   r›   r|   r¥   rK  r}   s        €r&   rj   zTFHubertFeatureEncoder.__init__I  s  ø€ Ü‰ÑÑ"˜6Ò"à×#Ñ# wÒ.Ü5°fÀqÐQ]Ð^_Ð]`ÐOaÔbÐcä˜v×=Ñ=ÀÑAÓBögàô -¨V¸aÀ!¹eÈLÐYZÐ]^ÑY^ÐX_ÐJ`Öaògñ ˆKð 'ˆÕð ×%Ñ%¨Ò0ô ˜v×=Ñ=Ó>öàô +¨6¸AÀlÐSTÐRUÐDVÖWðˆKð ð 'ˆÕô Ø0°×1IÑ1IÐ0JÐJsÐtóð ùògùò
s   ÁCÂCc                ód   — t        j                  |d«      }| j                  D ]
  } ||«      }Œ |S r¾   )r   r/   rK  )r{   Úinput_valuesr  Ú
conv_layers       r&   r—   zTFHubertFeatureEncoder.call\  s7   € ÜŸ™ |°RÓ8ˆØ×*Ñ*ò 	6ˆJÙ& }Ó5‰Mð	6àÐr(   c                óØ   — | j                   ry d| _         | j                  D ];  }t        j                  |j                  «      5  |j                  d «       d d d «       Œ= y # 1 sw Y   ŒHxY wr   )r†   rK  r   r  rË   r‡   )r{   r‰   rN  s      r&   r‡   zTFHubertFeatureEncoder.buildb  s`   € Ø�:Š:ØØˆŒ
Ø×*Ñ*ò 	'ˆJÜ—‘˜zŸ™Ó/ñ 'Ø× Ñ  Ô&÷'ð 'ñ	'÷'ð 'ús   ÁA Á A)	r?  rž   r!  rß   s   @r&   rE  rE  H  s   ø„ õ'ò&÷'r(   rE  c                  ó   ‡ — e Zd Zˆ fd„Zˆ xZS )ÚTFHubertFeatureExtractorc                óÒ   •— t        ‰| �  |fi |¤Ž t        j                  d| j                  j
                  › d| j                  j                  d   j
                  › d�t        «       y )NzThe class `zD` has been depreciated and will be removed in Transformers v5. Use `r   z
` instead.)ri   rj   ÚwarningsÚwarnr}   rÚ   Ú	__bases__ÚFutureWarningr<  s      €r&   rj   z!TFHubertFeatureExtractor.__init__l  s`   ø€ Ü‰Ñ˜Ñ* 6Ò*Ü�‰Ø˜$Ÿ.™.×1Ñ1Ð2ð 3à—N‘N×,Ñ,¨QÑ/×8Ñ8Ð9¸ðEô õ		
r(   )rÚ   rÛ   rÜ   rj   rÞ   rß   s   @r&   rQ  rQ  k  s   ø„ ÷
ð 
r(   rQ  c                  ó2   ‡ — e Zd Zdˆ fd„Zddd„Zdd„Zˆ xZS )ÚTFHubertFeatureProjectionc                óz  •— t        ‰| �  di |¤Ž t        j                  j	                  |j
                  d¬«      | _        t        j                  j                  |j                  t        |j                  «      dd¬«      | _        t        j                  j                  |j                  ¬«      | _        || _        y )Nr%  ©rm   rË   rK   Ú
projection©ÚunitsÚkernel_initializerré   rË   )Úraterh   )ri   rj   r   rÆ   r&  r'  r%  ÚDenser8  r   Úinitializer_ranger[  ÚDropoutÚfeat_proj_dropoutÚdropoutr›   r<  s      €r&   rj   z"TFHubertFeatureProjection.__init__w  s•   ø€ Ü‰ÑÑ"˜6Ò"äŸ,™,×9Ñ9À&×BWÑBWÐ^jÐ9ÓkˆŒÜŸ,™,×,Ñ,Ø×$Ñ$Ü.¨v×/GÑ/GÓHØ$Øð	 -ó 
ˆŒô —|‘|×+Ñ+°×1IÑ1IÐ+ÓJˆŒØˆ�r(   c                óp   — | j                  |«      }| j                  |«      }| j                  ||¬«      }|S ©N©Útraining)r%  r[  rd  ©r{   r  rh  s      r&   r—   zTFHubertFeatureProjection.call„  s6   € ØŸ™¨Ó6ˆØŸ™¨Ó6ˆØŸ™ ]¸X˜ÓFˆØÐr(   c                ó.  — | j                   ry d| _         t        | dd «      �gt        j                  | j                  j
                  «      5  | j                  j                  d d | j                  j                  d   g«       d d d «       t        | dd «      �ht        j                  | j                  j
                  «      5  | j                  j                  d d | j                  j                  d   g«       d d d «       y y # 1 sw Y   ŒxY w# 1 sw Y   y xY w)NTr%  r*   r[  )
r†   r  r   r  r%  rË   r‡   r›   r	  r[  rŸ   s     r&   r‡   zTFHubertFeatureProjection.buildŠ  sê   € Ø�:Š:ØØˆŒ
Ü�4˜ tÓ,Ð8Ü—‘˜tŸ™×3Ñ3Ó4ñ NØ—‘×%Ñ% t¨T°4·;±;×3GÑ3GÈÑ3KÐ&LÔM÷Nä�4˜ tÓ,Ð8Ü—‘˜tŸ™×3Ñ3Ó4ñ NØ—‘×%Ñ% t¨T°4·;±;×3GÑ3GÈÑ3KÐ&LÔM÷Nð Nð 9÷Nð Nú÷Nð Nús   Á6C?Â?6DÃ?DÄD©r›   r   ©F©r  r   rh  rÖ   r  r   rž   r!  rß   s   @r&   rX  rX  v  s   ø„ õô÷	Nr(   rX  c                  ó|   ‡ — e Zd ZdZ	 	 	 d	 	 	 	 	 	 	 	 	 dˆ fd„Zdd„Z	 	 	 	 	 d		 	 	 	 	 	 	 	 	 	 	 	 	 d
d„Zdd„Zˆ xZS )ÚTFHubertAttentionz6Multi-headed attention from "Attention Is All You Needc                óz  •— t        ‰| �  d
i |¤Ž || _        || _        t        j
                  j                  |«      | _        ||z  | _        | j                  |z  | j                  k7  rt        d| j                  › d|› d�«      ‚| j                  dz  | _
        || _        t        j
                  j                  ||d¬«      | _        t        j
                  j                  ||d¬«      | _        t        j
                  j                  ||d¬«      | _        t        j
                  j                  ||d	¬«      | _        y )Nz;embed_dim must be divisible by num_heads (got `embed_dim`: z and `num_heads`: rÀ   g      à¿Úk_proj)rè   rË   Úq_projÚv_projÚout_projrh   )ri   rj   Ú	embed_dimÚ	num_headsr   rÆ   rb  rd  Úhead_dimrB   ÚscalingÚ
is_decoderr`  rq  rr  rs  rt  )r{   ru  rv  rd  ry  rý   r|   r}   s          €r&   rj   zTFHubertAttention.__init__š  s  ø€ ô 	‰ÑÑ"˜6Ò"Ø"ˆŒà"ˆŒÜ—|‘|×+Ñ+¨GÓ4ˆŒØ! YÑ.ˆŒØ�M‰M˜IÑ%¨$¯.©.Ò8ÜØMÈdÏnÉnÐM]Ø$ Y K¨rð3óð ð —}‘} dÑ*ˆŒØ$ˆŒä—l‘l×(Ñ(¨¸TÈÐ(ÓQˆŒÜ—l‘l×(Ñ(¨¸TÈÐ(ÓQˆŒÜ—l‘l×(Ñ(¨¸TÈÐ(ÓQˆŒÜŸ™×*Ñ*¨9¸tÈ*Ð*ÓUˆ�r(   c           	     ó†   — t        j                  t        j                  |||| j                  | j                  f«      d«      S )N©r   rê   r   r   )r   r1   r-   rv  rw  )r{   ÚtensorÚseq_lenÚbszs       r&   Ú_shapezTFHubertAttention._shape¶  s0   € Ü�|‰|œBŸJ™J v°°W¸d¿n¹nÈdÏmÉmÐ/\Ó]Ð_kÓlÐlr(   c           
     óÌ	  — |du}t        |«      \  }}	}
| j                  |«      | j                  z  }|r|�|d   }|d   }�n
|rE| j                  | j	                  |«      d|«      }| j                  | j                  |«      d|«      }nÃ|�}| j                  | j	                  |«      d|«      }| j                  | j                  |«      d|«      }t        j                  |d   |gd¬«      }t        j                  |d   |gd¬«      }nD| j                  | j	                  |«      d|«      }| j                  | j                  |«      d|«      }| j                  r||f}|| j                  z  d| j                  f}t        j                  | j                  ||	|«      |«      }t        j                  ||«      }t        j                  ||«      }t        |«      d   }t        j                  ||d¬«      }t        j                  j                  t        |«      || j                  z  |	|gd	|| j                  z  |	|f› d
t        |«      › �¬«       |�°t        j                  j                  t        |«      |d|	|gd|d|	|f› d
t        |«      › �¬«       t        j                  ||j                   ¬«      }t        j                  ||| j                  |	|f«      |z   }t        j                  ||| j                  z  |	|f«      }t#        |d¬«      }|�°t        j                  j                  t        |«      | j                  gd| j                  › d
t        |«      › �¬«       t        j                  |d«      t        j                  ||| j                  |	|f«      z  }t        j                  ||| j                  z  |	|f«      }| j%                  ||¬«      }t        j                  ||«      }t        j                  j                  t        |«      || j                  z  |	| j                  gd|| j                  |	| j                  f› d
t        |«      › �¬«       t        j&                  t        j                  ||| j                  |	| j                  f«      d«      }t        j                  |||	|
f«      }| j)                  |«      }t        j                  ||| j                  |	|f«      }|||fS )z#Input shape: Batch x Time x ChannelNr   r   r*   rê   r+   T)Útranspose_bz$Attention weights should be of size z	, but is r=   z!Attention mask should be of size r@   z/Head mask for a single layer should be of size )r   r*   r   r   rg  z `attn_output` should be of size r{  )r   rr  rx  r  rq  rs  r   r2   ry  rv  rw  r-   ÚmatmulrC   Úassert_equalrE   rA   r   rd  r1   rt  )r{   r  Úkey_value_statesÚpast_key_valueÚattention_maskÚlayer_head_maskrh  Úis_cross_attentionr~  r`   ru  Úquery_statesÚ
key_statesÚvalue_statesÚ
proj_shapera   Úattn_weightsÚ
attn_probsÚattn_outputs                      r&   r—   zTFHubertAttention.call¹  sž  € ð .°TÐ9ÐÜ",¨]Ó";ÑˆˆW�ið —{‘{ =Ó1°D·L±LÑ@ˆá .Ð"<à'¨Ñ*ˆJØ)¨!Ñ,ŠLÙàŸ™ T§[¡[Ð1AÓ%BÀBÈÓLˆJØŸ;™; t§{¡{Ð3CÓ'DÀbÈ#ÓN‰LØÐ'àŸ™ T§[¡[°Ó%?ÀÀSÓIˆJØŸ;™; t§{¡{°=Ó'AÀ2ÀsÓKˆLÜŸ™ N°1Ñ$5°zÐ#BÈÔKˆJÜŸ9™9 n°QÑ&7¸Ð%FÈQÔO‰Lð Ÿ™ T§[¡[°Ó%?ÀÀSÓIˆJØŸ;™; t§{¡{°=Ó'AÀ2ÀsÓKˆLà�?Š?ð )¨,Ð7ˆNà˜DŸN™NÑ*¨B°·±Ð>ˆ
Ü—z‘z $§+¡+¨l¸GÀSÓ"IÈ:ÓVˆÜ—Z‘Z 
¨JÓ7ˆ
Ü—z‘z ,°
Ó;ˆä˜ZÓ(¨Ñ+ˆÜ—y‘y ¨zÀtÔLˆä
�‰×!Ñ!Ü�|Ó$Ø�4—>‘>Ñ! 7¨GÐ4à6¸¸d¿n¹nÑ8LÈgÐW^Ð7_Ð6`ð aÜ˜|Ó,Ð-ð/ð	 	"ô 	
ð Ð%Ü�L‰L×%Ñ%Ü˜>Ó*Ø�a˜ 'Ð*à7¸¸aÀÈ'Ð8RÐ7Sð TÜ" >Ó2Ð3ð5ð	 &ô ô  ŸW™W ^¸<×;MÑ;MÔNˆNÜŸ:™: l°S¸$¿.¹.È'ÐSZÐ4[Ó\Ð_mÑmˆLÜŸ:™: l°S¸4¿>¹>Ñ5IÈ7ÐT[Ð4\Ó]ˆLä% l¸Ô<ˆàÐ&Ü�L‰L×%Ñ%Ü˜?Ó+Ø—‘Ð àEÀtÇ~Á~ÐEWð XÜ" ?Ó3Ð4ð6ð	 &ô ô Ÿ:™: o°}ÓEÌÏ
É
Ø˜s D§N¡N°G¸WÐEóIñ ˆLô Ÿ:™: l°S¸4¿>¹>Ñ5IÈ7ÐT[Ð4\Ó]ˆLà—\‘\ ,¸�\ÓBˆ
Ü—i‘i 
¨LÓ9ˆä
�‰×!Ñ!Ü�{Ó#Ø�4—>‘>Ñ! 7¨D¯M©MÐ:à2°C¸¿¹ÈÐRV×R_ÑR_Ð3`Ð2að bÜ˜{Ó+Ð,ð.ð	 	"ô 	
ô —l‘lÜ�J‰J�{ S¨$¯.©.¸'À4Ç=Á=Ð$QÓRÐT`ó
ˆô —j‘j ¨s°G¸YÐ.GÓHˆà—m‘m KÓ0ˆÜ"$§*¡*¨\¸CÀÇÁÐQXÐZaÐ;bÓ"cˆà˜L¨.Ð8Ð8r(   c                óÈ  — | j                   ry d| _         t        | dd «      �Zt        j                  | j                  j
                  «      5  | j                  j                  d d | j                  g«       d d d «       t        | dd «      �Zt        j                  | j                  j
                  «      5  | j                  j                  d d | j                  g«       d d d «       t        | dd «      �Zt        j                  | j                  j
                  «      5  | j                  j                  d d | j                  g«       d d d «       t        | dd «      �[t        j                  | j                  j
                  «      5  | j                  j                  d d | j                  g«       d d d «       y y # 1 sw Y   �ŒAxY w# 1 sw Y   ŒæxY w# 1 sw Y   Œ‹xY w# 1 sw Y   y xY w)NTrq  rr  rs  rt  )r†   r  r   r  rq  rË   r‡   ru  rr  rs  rt  rŸ   s     r&   r‡   zTFHubertAttention.build/  sŠ  € Ø�:Š:ØØˆŒ
Ü�4˜ 4Ó(Ð4Ü—‘˜tŸ{™{×/Ñ/Ó0ñ @Ø—‘×!Ñ! 4¨¨t¯~©~Ð">Ô?÷@ä�4˜ 4Ó(Ð4Ü—‘˜tŸ{™{×/Ñ/Ó0ñ @Ø—‘×!Ñ! 4¨¨t¯~©~Ð">Ô?÷@ä�4˜ 4Ó(Ð4Ü—‘˜tŸ{™{×/Ñ/Ó0ñ @Ø—‘×!Ñ! 4¨¨t¯~©~Ð">Ô?÷@ä�4˜ TÓ*Ð6Ü—‘˜tŸ}™}×1Ñ1Ó2ñ BØ—‘×#Ñ# T¨4°·±Ð$@ÔA÷Bð Bð 7÷@ñ @ú÷@ð @ú÷@ð @ú÷Bð Bús0   Á)F3Â2)G Ä)GÆ )GÆ3F=Ç G	ÇGÇG!)g        FT)
ru  rÔ   rv  rÔ   rd  rÕ   ry  rÖ   rý   rÖ   )r|  r   r}  rÔ   r~  rÔ   )NNNNF)r  r   r„  útf.Tensor | Noner…  zTuple[Tuple[tf.Tensor]] | Noner†  r‘  r‡  r‘  rh  úOptional[bool]r  z"Tuple[tf.Tensor, tf.Tensor | None]rž   )	rÚ   rÛ   rÜ   rÝ   rj   r  r—   r‡   rÞ   rß   s   @r&   ro  ro  —  s»   ø„ Ù@ð Ø ØðVàðVð ðVð ð	Vð
 ðVð õVó8mð .2Ø9=Ø+/Ø,0Ø#(ðt9à ðt9ð +ðt9ð 7ð	t9ð
 )ðt9ð *ðt9ð !ðt9ð 
,ót9÷lBr(   ro  c                  ó2   ‡ — e Zd Zdˆ fd„Zddd„Zdd„Zˆ xZS )ÚTFHubertFeedForwardc                ó2  •— t        ‰| �  di |¤Ž t        j                  j	                  |j
                  «      | _        t        j                  j                  |j                  t        |j                  «      dd¬«      | _        t        |j                  «      | _        t        j                  j                  |j                  t        |j                  «      dd¬«      | _        t        j                  j	                  |j"                  «      | _        || _        y )NrK   Úintermediate_denser\  Úoutput_denserh   )ri   rj   r   rÆ   rb  Úactivation_dropoutÚintermediate_dropoutr`  Úintermediate_sizer   ra  r–  r	   Ú
hidden_actÚintermediate_act_fnr8  r—  Úhidden_dropoutÚoutput_dropoutr›   r<  s      €r&   rj   zTFHubertFeedForward.__init__C  sÝ   ø€ Ü‰ÑÑ"˜6Ò"ä$)§L¡L×$8Ñ$8¸×9RÑ9RÓ$SˆÔ!ä"'§,¡,×"4Ñ"4Ø×*Ñ*Ü.¨v×/GÑ/GÓHØ$Ø%ð	 #5ó #
ˆÔô $5°V×5FÑ5FÓ#GˆÔ ä!ŸL™L×.Ñ.Ø×$Ñ$Ü.¨v×/GÑ/GÓHØ$Øð	 /ó 
ˆÔô $Ÿl™l×2Ñ2°6×3HÑ3HÓIˆÔØˆ�r(   c                ó¸   — | j                  |«      }| j                  |«      }| j                  ||¬«      }| j                  |«      }| j	                  ||¬«      }|S rf  )r–  rœ  r™  r—  rž  ri  s      r&   r—   zTFHubertFeedForward.callY  sb   € Ø×/Ñ/°Ó>ˆØ×0Ñ0°Ó?ˆØ×1Ñ1°-È(Ð1ÓSˆà×)Ñ)¨-Ó8ˆØ×+Ñ+¨MÀHÐ+ÓMˆØÐr(   c                ó"  — | j                   ry d| _         t        | dd «      �dt        j                  | j                  j
                  «      5  | j                  j                  d d | j                  j                  g«       d d d «       t        | dd «      �et        j                  | j                  j
                  «      5  | j                  j                  d d | j                  j                  g«       d d d «       y y # 1 sw Y   Œ|xY w# 1 sw Y   y xY w)NTr–  r—  )r†   r  r   r  r–  rË   r‡   r›   r8  r—  rš  rŸ   s     r&   r‡   zTFHubertFeedForward.buildb  sé   € Ø�:Š:ØØˆŒ
Ü�4Ð-¨tÓ4Ð@Ü—‘˜t×6Ñ6×;Ñ;Ó<ñ UØ×'Ñ'×-Ñ-¨t°T¸4¿;¹;×;RÑ;RÐ.SÔT÷Uä�4˜¨Ó.Ð:Ü—‘˜t×0Ñ0×5Ñ5Ó6ñ UØ×!Ñ!×'Ñ'¨¨t°T·[±[×5RÑ5RÐ(SÔT÷Uð Uð ;÷Uð Uú÷Uð Uús   Á3C9Â<3DÃ9DÄDrk  rl  rm  rž   r!  rß   s   @r&   r”  r”  B  s   ø„ õô,÷	Ur(   r”  c                  óJ   ‡ — e Zd Zdˆ fd„Z	 	 	 d	 	 	 	 	 	 	 	 	 dd„Zdd„Zˆ xZS )ÚTFHubertEncoderLayerc                óØ  •— t        ‰| �  d	i |¤Ž t        |j                  |j                  |j
                  dd¬«      | _        t        j                  j                  |j                  «      | _        t        j                  j                  |j                  d¬«      | _        t        |d¬«      | _        t        j                  j                  |j                  d¬«      | _        || _        y ©
NFÚ	attention)ru  rv  rd  ry  rË   r%  rZ  Úfeed_forward©rË   Úfinal_layer_normrh   ©ri   rj   ro  r8  Únum_attention_headsÚattention_dropoutr¥  r   rÆ   rb  r�  rd  r&  r'  r%  r”  r¦  r¨  r›   r<  s      €r&   rj   zTFHubertEncoderLayer.__init__p  ó¹   ø€ Ü‰ÑÑ"˜6Ò"Ü*Ø×(Ñ(Ø×0Ñ0Ø×,Ñ,ØØô
ˆŒô —|‘|×+Ñ+¨F×,AÑ,AÓBˆŒÜŸ,™,×9Ñ9À&×BWÑBWÐ^jÐ9ÓkˆŒÜ/°¸^ÔLˆÔÜ %§¡× ?Ñ ?È×H]ÑH]ÐdvÐ ?Ó wˆÔØˆ�r(   c                óì   — |}| j                  |||¬«      \  }}}| j                  ||¬«      }||z   }| j                  |«      }|| j                  |«      z   }| j	                  |«      }|f}|r||fz  }|S ©N)r†  rh  rg  )r¥  rd  r%  r¦  r¨  ©	r{   r  r†  Úoutput_attentionsrh  Úattn_residualr�  r$   r–   s	            r&   r—   zTFHubertEncoderLayer.call  s—   € ð &ˆØ)-¯©Ø¨.À8ð *8ó *
Ñ&ˆ�| Qð Ÿ™ ]¸X˜ÓFˆØ%¨Ñ5ˆàŸ™¨Ó6ˆØ%¨×(9Ñ(9¸-Ó(HÑHˆØ×-Ñ-¨mÓ<ˆà Ð"ˆáØ˜�Ñ&ˆGàˆr(   c                ó¼  — | j                   ry d| _         t        | dd «      �Mt        j                  | j                  j
                  «      5  | j                  j                  d «       d d d «       t        | dd «      �dt        j                  | j                  j
                  «      5  | j                  j                  d d | j                  j                  g«       d d d «       t        | dd «      �Mt        j                  | j                  j
                  «      5  | j                  j                  d «       d d d «       t        | dd «      �et        j                  | j                  j
                  «      5  | j                  j                  d d | j                  j                  g«       d d d «       y y # 1 sw Y   �ŒHxY w# 1 sw Y   ŒãxY w# 1 sw Y   Œ•xY w# 1 sw Y   y xY w©NTr¥  r%  r¦  r¨  ©r†   r  r   r  r¥  rË   r‡   r%  r›   r8  r¦  r¨  rŸ   s     r&   r‡   zTFHubertEncoderLayer.build˜  ó�  € Ø�:Š:ØØˆŒ
Ü�4˜ dÓ+Ð7Ü—‘˜tŸ~™~×2Ñ2Ó3ñ +Ø—‘×$Ñ$ TÔ*÷+ä�4˜ tÓ,Ð8Ü—‘˜tŸ™×3Ñ3Ó4ñ MØ—‘×%Ñ% t¨T°4·;±;×3JÑ3JÐ&KÔL÷Mä�4˜¨Ó.Ð:Ü—‘˜t×0Ñ0×5Ñ5Ó6ñ .Ø×!Ñ!×'Ñ'¨Ô-÷.ä�4Ð+¨TÓ2Ð>Ü—‘˜t×4Ñ4×9Ñ9Ó:ñ SØ×%Ñ%×+Ñ+¨T°4¸¿¹×9PÑ9PÐ,QÔR÷Sð Sð ?÷+ñ +ú÷Mð Mú÷.ð .ú÷Sð Súó0   ÁF-Â%3F:ÄGÅ03GÆ-F7Æ:GÇGÇGrk  ©NFF©
r  r   r†  r‘  r°  r’  rh  rÖ   r  zTuple[tf.Tensor]rž   r!  rß   s   @r&   r¢  r¢  o  sN   ø„ õð$ ,0Ø,1Øðà ðð )ðð *ð	ð
 ðð 
ó÷2Sr(   r¢  c                  óJ   ‡ — e Zd Zdˆ fd„Z	 	 	 d	 	 	 	 	 	 	 	 	 dd„Zdd„Zˆ xZS )Ú#TFHubertEncoderLayerStableLayerNormc                óØ  •— t        ‰| �  d	i |¤Ž t        |j                  |j                  |j
                  dd¬«      | _        t        j                  j                  |j                  «      | _        t        j                  j                  |j                  d¬«      | _        t        |d¬«      | _        t        j                  j                  |j                  d¬«      | _        || _        y r¤  r©  r<  s      €r&   rj   z,TFHubertEncoderLayerStableLayerNorm.__init__¬  r¬  r(   c                óè   — |}| j                  |«      }| j                  |||¬«      \  }}}| j                  ||¬«      }||z   }|| j                  | j	                  |«      «      z   }|f}|r||fz  }|S r®  )r%  r¥  rd  r¦  r¨  r¯  s	            r&   r—   z(TFHubertEncoderLayerStableLayerNorm.call»  s“   € ð &ˆØŸ™¨Ó6ˆØ)-¯©Ø¨.À8ð *8ó *
Ñ&ˆ�| Qð Ÿ™ ]¸X˜ÓFˆØ%¨Ñ5ˆØ%¨×(9Ñ(9¸$×:OÑ:OÐP]Ó:^Ó(_Ñ_ˆà Ð"ˆáØ˜�Ñ&ˆGàˆr(   c                ó¼  — | j                   ry d| _         t        | dd «      �Mt        j                  | j                  j
                  «      5  | j                  j                  d «       d d d «       t        | dd «      �dt        j                  | j                  j
                  «      5  | j                  j                  d d | j                  j                  g«       d d d «       t        | dd «      �Mt        j                  | j                  j
                  «      5  | j                  j                  d «       d d d «       t        | dd «      �et        j                  | j                  j
                  «      5  | j                  j                  d d | j                  j                  g«       d d d «       y y # 1 sw Y   �ŒHxY w# 1 sw Y   ŒãxY w# 1 sw Y   Œ•xY w# 1 sw Y   y xY wr³  r´  rŸ   s     r&   r‡   z)TFHubertEncoderLayerStableLayerNorm.buildÒ  rµ  r¶  rk  r·  r¸  rž   r!  rß   s   @r&   rº  rº  «  sN   ø„ õð$ ,0Ø,1Øðà ðð )ðð *ð	ð
 ðð 
ó÷.Sr(   rº  c                  óV   ‡ — e Zd Zdˆ fd„Z	 	 	 	 	 d	 	 	 	 	 	 	 	 	 	 	 	 	 dd„Zdd„Zˆ xZS )ÚTFHubertEncoderc                óˆ  •— t        ‰| �  di |¤Ž || _        t        |d¬«      | _        t
        j                  j                  |j                  d¬«      | _	        t
        j                  j                  |j                  «      | _        t        |j                  «      D �cg c]  }t        |d|› �¬«      ‘Œ c}| _        y c c}w ©NÚpos_conv_embedr§  r%  rZ  zlayers.rh   )ri   rj   r›   r6  rÂ  r   rÆ   r&  r'  r%  rb  r�  rd  r0   Únum_hidden_layersr¢  rH  ©r{   r›   r|   r¥   r}   s       €r&   rj   zTFHubertEncoder.__init__æ  s—   ø€ Ü‰ÑÑ"˜6Ò"ØˆŒÜ=¸fÐK[Ô\ˆÔÜŸ,™,×9Ñ9À&×BWÑBWÐ^jÐ9ÓkˆŒÜ—|‘|×+Ñ+¨F×,AÑ,AÓBˆŒÜPUÐV\×VnÑVnÓPoÖpÈ1Ô*¨6¸'À!À¸ÖFÒpˆ�
ùÒpó   ÂB?c                ó6  — |rdnd }|rdnd }|�%|t        j                  |d«      z  }t        |«      }nd }| j                  |«      }	||	z   }| j	                  |«      }| j                  ||¬«      }t        | j                  «      D ]f  \  }
}|r||fz   }t        j                  j                  dd«      }|r|| j                  j                  k  rŒJ |||||¬«      }|d   }|sŒ^||d   fz   }Œh |r||fz   }|st        d„ |||fD «       «      S t        |||¬«      S )	Nrh   r*   rg  r   r   ©r  r†  r°  rh  c              3  ó&   K  — | ]	  }|€Œ|–— Œ y ­wrž   rh   ©Ú.0Úvs     r&   ú	<genexpr>z'TFHubertEncoder.call.<locals>.<genexpr>  ó   è ø€ Òm˜qÐ_`Ñ_lœÑmùó   ‚Š©Úlast_hidden_stater  Ú
attentions)r   r/   rd   rÂ  r%  rd  Ú	enumeraterH  Únpr   r   r›   Ú	layerdropÚtupler
   ©r{   r  r†  r°  Úoutput_hidden_statesÚreturn_dictrh  Úall_hidden_statesÚall_self_attentionsÚposition_embeddingsr¥   Úlayer_moduleÚdropout_probabilityÚlayer_outputss                 r&   r—   zTFHubertEncoder.callî  s\  € ñ #7™B¸DÐÙ$5™b¸4ÐàÐ%Ø)¬B¯N©N¸>È2Ó,NÑNˆMÜ)¨.Ó9‰Nà!ˆNà"×1Ñ1°-Ó@ÐØ%Ð(;Ñ;ˆØŸ™¨Ó6ˆØŸ™ ]¸X˜ÓFˆä(¨¯©Ó4ò 	P‰OˆAˆ|Ù#Ø$5¸Ð8HÑ$HÐ!ô #%§)¡)×"3Ñ"3°A°qÓ"9ÐÙÐ0°4·;±;×3HÑ3HÒHØá(Ø+Ø-Ø"3Ø!ô	ˆMð *¨!Ñ,ˆMâ Ø&9¸]È1Ñ=MÐ<OÑ&OÑ#ð%	Pñ*  Ø 1°]Ð4DÑ DÐáÜÑm ]Ð4EÐGZÐ$[ÔmÓmÐmÜ Ø+Ø+Ø*ô
ð 	
r(   c                óº  — | j                   ry d| _         t        | dd «      �Mt        j                  | j                  j
                  «      5  | j                  j                  d «       d d d «       t        | dd «      �dt        j                  | j                  j
                  «      5  | j                  j                  d d | j                  j                  g«       d d d «       t        | dd «      �K| j                  D ];  }t        j                  |j
                  «      5  |j                  d «       d d d «       Œ= y y # 1 sw Y   ŒÓxY w# 1 sw Y   ŒnxY w# 1 sw Y   ŒaxY w©NTrÂ  r%  rH  ©r†   r  r   r  rÂ  rË   r‡   r%  r›   r8  rH  ©r{   r‰   rH  s      r&   r‡   zTFHubertEncoder.build%  ó"  € Ø�:Š:ØØˆŒ
Ü�4Ð)¨4Ó0Ð<Ü—‘˜t×2Ñ2×7Ñ7Ó8ñ 0Ø×#Ñ#×)Ñ)¨$Ô/÷0ä�4˜ tÓ,Ð8Ü—‘˜tŸ™×3Ñ3Ó4ñ MØ—‘×%Ñ% t¨T°4·;±;×3JÑ3JÐ&KÔL÷Mä�4˜ $Ó'Ð3ØŸ™ò &�Ü—]‘] 5§:¡:Ó.ñ &Ø—K‘K Ô%÷&ð &ñ&ð 4÷0ð 0ú÷Mð Mú÷&ð &úó$   ÁD9Â%3EÄEÄ9EÅEÅE	rk  ©NFFTF©r  r   r†  r‘  r°  r’  r×  r’  rØ  r’  rh  r’  r  ú*Union[TFBaseModelOutput, Tuple[tf.Tensor]]rž   r!  rß   s   @r&   r¿  r¿  å  si   ø„ õqð ,0Ø,1Ø/4Ø&*Ø#(ð5
à ð5
ð )ð5
ð *ð	5
ð
 -ð5
ð $ð5
ð !ð5
ð 
4ó5
÷n&r(   r¿  c                  óV   ‡ — e Zd Zdˆ fd„Z	 	 	 	 	 d	 	 	 	 	 	 	 	 	 	 	 	 	 dd„Zdd„Zˆ xZS )ÚTFHubertEncoderStableLayerNormc                óˆ  •— t        ‰| �  di |¤Ž || _        t        |d¬«      | _        t
        j                  j                  |j                  d¬«      | _	        t
        j                  j                  |j                  «      | _        t        |j                  «      D �cg c]  }t        |d|› �¬«      ‘Œ c}| _        y c c}w rÁ  )ri   rj   r›   r6  rÂ  r   rÆ   r&  r'  r%  rb  r�  rd  r0   rÃ  rº  rH  rÄ  s       €r&   rj   z'TFHubertEncoderStableLayerNorm.__init__7  sž   ø€ Ü‰ÑÑ"˜6Ò"ØˆŒÜ=¸fÐK[Ô\ˆÔÜŸ,™,×9Ñ9À&×BWÑBWÐ^jÐ9ÓkˆŒÜ—|‘|×+Ñ+¨F×,AÑ,AÓBˆŒäUZÐ[a×[sÑ[sÓUtö
ØPQÔ/°¸wÀqÀc¸]ÖKò
ˆ�
ùò 
rÅ  c                ó6  — |rdnd }|rdnd }|�%|t        j                  |d«      z  }t        |«      }nd }| j                  |«      }	||	z   }| j	                  ||¬«      }t        | j                  «      D ]f  \  }
}|r||fz   }t        j                  j                  dd«      }|r|| j                  j                  k  rŒJ |||||¬«      }|d   }|sŒ^||d   fz   }Œh | j                  |«      }|r||fz   }|st        d„ |||fD «       «      S t        |||¬«      S )	Nrh   r*   rg  r   r   rÇ  c              3  ó&   K  — | ]	  }|€Œ|–— Œ y ­wrž   rh   rÉ  s     r&   rÌ  z6TFHubertEncoderStableLayerNorm.call.<locals>.<genexpr>q  rÍ  rÎ  rÏ  )r   r/   rd   rÂ  rd  rÒ  rH  rÓ  r   r   r›   rÔ  r%  rÕ  r
   rÖ  s                 r&   r—   z#TFHubertEncoderStableLayerNorm.callA  s\  € ñ #7™B¸DÐÙ$5™b¸4ÐàÐ%Ø)¬B¯N©N¸>È2Ó,NÑNˆMÜ)¨.Ó9‰Nà!ˆNà"×1Ñ1°-Ó@ÐØ%Ð(;Ñ;ˆØŸ™ ]¸X˜ÓFˆä(¨¯©Ó4ò 	P‰OˆAˆ|Ù#Ø$5¸Ð8HÑ$HÐ!ô #%§)¡)×"3Ñ"3°A°qÓ"9ÐÙÐ0°4·;±;×3HÑ3HÒHØá(Ø+Ø-Ø"3Ø!ô	ˆMð *¨!Ñ,ˆMâ Ø&9¸]È1Ñ=MÐ<OÑ&OÑ#ð%	Pð( Ÿ™¨Ó6ˆáØ 1°]Ð4DÑ DÐáÜÑm ]Ð4EÐGZÐ$[ÔmÓmÐmÜ Ø+Ø+Ø*ô
ð 	
r(   c                óº  — | j                   ry d| _         t        | dd «      �Mt        j                  | j                  j
                  «      5  | j                  j                  d «       d d d «       t        | dd «      �dt        j                  | j                  j
                  «      5  | j                  j                  d d | j                  j                  g«       d d d «       t        | dd «      �K| j                  D ];  }t        j                  |j
                  «      5  |j                  d «       d d d «       Œ= y y # 1 sw Y   ŒÓxY w# 1 sw Y   ŒnxY w# 1 sw Y   ŒaxY wrà  rá  râ  s      r&   r‡   z$TFHubertEncoderStableLayerNorm.buildx  rã  rä  rk  rå  ræ  rž   r!  rß   s   @r&   ré  ré  6  sh   ø„ õ
ð ,0Ø,1Ø/4Ø&*Ø#(ð5
à ð5
ð )ð5
ð *ð	5
ð
 -ð5
ð $ð5
ð !ð5
ð 
4ó5
÷n&r(   ré  c                  óŽ   ‡ — e Zd ZeZdˆ fd„Zdd„Zdd„Zdd	d„Ze		 	 	 	 	 	 	 	 	 d
	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dd„«       Z
ˆ xZS )ÚTFHubertMainLayerc                óÜ   •— t        ‰| �  di |¤Ž || _        t        |d¬«      | _        t        |d¬«      | _        |j                  rt        |d¬«      | _	        y t        |d¬«      | _	        y )NÚfeature_extractorr§  Úfeature_projectionÚencoderrh   )ri   rj   r›   rE  rñ  rX  rò  Údo_stable_layer_normré  ró  r¿  r<  s      €r&   rj   zTFHubertMainLayer.__init__Œ  s_   ø€ Ü‰ÑÑ"˜6Ò"ØˆŒÜ!7¸ÐEXÔ!YˆÔÜ";¸FÐI]Ô"^ˆÔà×&Ò&Ü9¸&ÀyÔQˆD�Lä*¨6¸	ÔBˆD�Lr(   c                óð  — | j                  | j                  j                  fddd¬«      | _        | j                  ry d| _        t        | dd «      �Mt        j                  | j                  j                  «      5  | j                  j                  d «       d d d «       t        | dd «      �Mt        j                  | j                  j                  «      5  | j                  j                  d «       d d d «       t        | dd «      �Nt        j                  | j                  j                  «      5  | j                  j                  d «       d d d «       y y # 1 sw Y   Œ¿xY w# 1 sw Y   ŒqxY w# 1 sw Y   y xY w)Nr   TÚmasked_spec_embed)rP   rÌ   rü   rË   rñ  rò  ró  )rÏ   r›   r8  rö  r†   r  r   r  rñ  rË   r‡   rò  ró  rŸ   s     r&   r‡   zTFHubertMainLayer.build—  s:  € Ø!%§¡Ø—;‘;×*Ñ*Ð,¸)ÈtÐZmð "1ó "
ˆÔð �:Š:ØØˆŒ
Ü�4Ð,¨dÓ3Ð?Ü—‘˜t×5Ñ5×:Ñ:Ó;ñ 3Ø×&Ñ&×,Ñ,¨TÔ2÷3ä�4Ð-¨tÓ4Ð@Ü—‘˜t×6Ñ6×;Ñ;Ó<ñ 4Ø×'Ñ'×-Ñ-¨dÔ3÷4ä�4˜ DÓ)Ð5Ü—‘˜tŸ|™|×0Ñ0Ó1ñ )Ø—‘×"Ñ" 4Ô(÷)ð )ð 6÷3ð 3ú÷4ð 4ú÷)ð )ús$   Á:EÃE Ä.E,ÅEÅ E)Å,E5c                ó˜   — d„ }t        | j                  j                  | j                  j                  «      D ]  \  }} ||||«      }Œ |S )zH
        Computes the output length of the convolutional layers
        c                ó   — | |z
  |z  dz   S r‹   rh   )Úinput_lengthræ   Ústrides      r&   Ú_conv_out_lengthzLTFHubertMainLayer._get_feat_extract_output_lengths.<locals>._conv_out_length®  s   € ð ! ;Ñ.°6Ñ9¸AÑ=Ð=r(   )Úzipr›   r  r  )r{   Úinput_lengthsrû  ræ   rú  s        r&   Ú _get_feat_extract_output_lengthsz2TFHubertMainLayer._get_feat_extract_output_lengths©  sP   € ò
	>ô
 $' t§{¡{×'>Ñ'>ÀÇÁ×@WÑ@WÓ#Xò 	QÑˆK˜Ù,¨]¸KÈÓP‰Mð	Qð Ðr(   c                óü  — t        |«      \  }}}t        | j                  dd«      s|S |�€t        j                  t        j
                  |dd…dd…t        j                  f   t        j                  «      | j                  t        j                  t        j                  dd…f   |«      }nÑ| j                  j                  dkD  r¸t        ||f| j                  j                  | j                  j                  d¬«      }t        j                  t        j
                  |dd…dd…t        j                  f   t        j                  «      | j                  t        j                  t        j                  dd…f   |«      }| j                  j                  dkD  rgt        ||f| j                  j                  | j                  j                  ¬«      }t        j                  |dd…t        j                  dd…f   |d«      }|S )zš
        Masks extracted features along time axis and/or along feature axis according to
        [SpecAugment](https://arxiv.org/abs/1904.08779).
        Úapply_spec_augmentTNr   rê   )rQ   rR   rS   )rQ   rR   )r   r  r›   r   ÚwhererE   rN   rÖ   rö  Úmask_time_probr[   Úmask_time_lengthÚmask_feature_probÚmask_feature_length)r{   r  Úmask_time_indicesrT   rU   r8  Úmask_feature_indicess          r&   Ú_mask_hidden_statesz%TFHubertMainLayer._mask_hidden_states¸  s‰  € ô
 4>¸mÓ3LÑ0ˆ
�O [ô �t—{‘{Ð$8¸$Ô?Ø Ð àÐ(äŸH™HÜ—‘Ð)ª!ªQ´·
±
Ð*:Ñ;¼R¿W¹WÓEØ×&Ñ&¤r§z¡z´2·:±:ºqÐ'@ÑAØó‰Mð �[‰[×'Ñ'¨!Ò+ä 5Ø˜_Ð-ØŸ+™+×4Ñ4Ø ŸK™K×8Ñ8Øô	!Ðô ŸH™HÜ—‘Ð)ª!ªQ´·
±
Ð*:Ñ;¼R¿W¹WÓEØ×&Ñ&¤r§z¡z´2·:±:ºqÐ'@ÑAØóˆMð �;‰;×(Ñ(¨1Ò,Ü#8Ø˜[Ð)ØŸ+™+×7Ñ7Ø ŸK™K×;Ñ;ô$Ð ô
 ŸH™HÐ%9º!¼R¿Z¹ZÊÐ:JÑ%KÈ]Ð\]Ó^ˆMàÐr(   c                ó  — | j                  t        j                  |t        j                  «      |
¬«      }|�S| j	                  t        j
                  |d«      «      }t        j                  |t        |«      d   |j                  ¬«      }| j                  ||
¬«      }|j                  dd «      }|
r| j                  ||¬«      }| j                  |||||	|
¬«      }|d   }|	s	|f|dd  z   S t        ||j                  |j                  ¬	«      S )
Nrg  r*   r   )ÚmaxlenrA   r  )r  )r†  r°  r×  rØ  rh  r   rÏ  )rñ  r   rE   rF   rþ  rð   Úsequence_maskr   rA   rò  rq   r  ró  r
   r  rÑ  )r{   rM  r†  Útoken_type_idsÚposition_idsÚ	head_maskÚinputs_embedsr°  r×  rØ  rh  r|   r  Úoutput_lengthsr  Úencoder_outputss                   r&   r—   zTFHubertMainLayer.callä  s   € ð ×.Ñ.¬r¯w©w°|ÄRÇZÁZÓ/PÐ[cÐ.ÓdˆàÐ%à!×BÑBÄ2Ç=Á=ÐQ_ÐacÓCdÓeˆNä×-Ñ-Ø¤z°-Ó'@ÀÑ'CÈ=×K^ÑK^ôˆNð ×/Ñ/°ÈÐ/ÓQˆà"ŸJ™JÐ':¸DÓAÐÙØ ×4Ñ4°]ÐVgÐ4ÓhˆMàŸ,™,ØØ)Ø/Ø!5Ø#Øð 'ó 
ˆð (¨Ñ*ˆáØ!Ð# o°a°bÐ&9Ñ9Ð9ä Ø+Ø)×7Ñ7Ø&×1Ñ1ô
ð 	
r(   rk  rž   )rý  r   )r  r   r  r‘  ©	NNNNNNNNF)rM  r   r†  r‘  r  r‘  r  r‘  r  r‘  r  r‘  r°  r‘  r×  r‘  rØ  r’  rh  rÖ   r|   r   )rÚ   rÛ   rÜ   r   Úconfig_classrj   r‡   rþ  r  r   r—   rÞ   rß   s   @r&   rï  rï  ˆ  s»   ø„ à€Lõ	Có)ó$ô*ðX ð ,0Ø+/Ø)-Ø&*Ø*.Ø.2Ø15Ø&*Øð/
àð/
ð )ð/
ð )ð	/
ð
 'ð/
ð $ð/
ð (ð/
ð ,ð/
ð /ð/
ð $ð/
ð ð/
ð ò/
ó ô/
r(   rï  c                  ó>   ‡ — e Zd ZdZeZdZdZed„ «       Z	ˆ fd„Z
ˆ xZS )ÚTFHubertPreTrainedModelz†
    An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained
    models.
    ÚhubertrM  c                óæ   — t        j                  dt         j                  d¬«      t        j                  dt         j                  d¬«      t        j                  dt         j                  d¬«      dœS )N)Ni€>  rM  r§  )NNr†  r  )rM  r†  r  )r   Ú
TensorSpecrF   rH   rÂ   s    r&   Úinput_signaturez'TFHubertPreTrainedModel.input_signature!  sL   € ô ŸM™M¨-¼¿¹È.ÔYÜ Ÿm™m¨L¼"¿(¹(ÐIYÔZÜ Ÿm™m¨L¼"¿(¹(ÐIYÔZñ
ð 	
r(   c                ó†   •— t        ‰| �  |g|¢­i |¤Ž t        j                  d| j                  j
                  › d�«       y )Nú
z� has backpropagation operations that are NOT supported on CPU. If you wish to train/fine-tune this model, you need a GPU or a TPU)ri   rj   ÚloggerÚwarningr}   rÚ   ©r{   r›   r�   r|   r}   s       €r&   rj   z TFHubertPreTrainedModel.__init__)  sD   ø€ Ü‰Ñ˜Ð3 &Ò3¨FÒ3Ü�‰Ø�—‘×(Ñ(Ð)ð *Eð Eõ	
r(   )rÚ   rÛ   rÜ   rÝ   r   r  Úbase_model_prefixÚmain_input_nameÚpropertyr  rj   rÞ   rß   s   @r&   r  r    s6   ø„ ñð
  €LØ ÐØ$€Oàñ
ó ð
÷
ð 
r(   r  aˆ	  

    This model inherits from [`TFPreTrainedModel`]. Check the superclass documentation for the generic methods the
    library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
    etc.)

    This model is also a [keras.Model](https://www.tensorflow.org/api_docs/python/tf/keras/Model) subclass. Use it
    as a regular TF 2.0 Keras Model and refer to the TF 2.0 documentation for all matter related to general usage and
    behavior.

    <Tip>

    TensorFlow models and layers in `transformers` accept two formats as input:

    - having all inputs as keyword arguments (like PyTorch models), or
    - having all inputs as a list, tuple or dict in the first positional argument.

    The reason the second format is supported is that Keras methods prefer this format when passing inputs to models
    and layers. Because of this support, when using methods like `model.fit()` things should "just work" for you - just
    pass your inputs and labels in any format that `model.fit()` supports! If, however, you want to use the second
    format outside of Keras methods like `fit()` and `predict()`, such as when creating your own layers or models with
    the Keras `Functional` API, there are three possibilities you can use to gather all the input Tensors in the first
    positional argument:

    - a single Tensor with `input_values` only and nothing else: `model(input_values)`
    - a list of varying length with one or several input Tensors IN THE ORDER given in the docstring:
    `model([input_values, attention_mask])` or `model([input_values, attention_mask, token_type_ids])`
    - a dictionary with one or several input Tensors associated to the input names given in the docstring:
    `model({"input_values": input_values, "token_type_ids": token_type_ids})`

    Note that when creating models and layers with
    [subclassing](https://keras.io/guides/making_new_layers_and_models_via_subclassing/) then you don't need to worry
    about any of this, as you can just pass inputs like you would to any other Python function!

    </Tip>

    Args:
        config ([`HubertConfig`]): Model configuration class with all the parameters of the model.
            Initializing with a config file does not load the weights associated with the model, only the
            configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
a	  
    Args:
        input_values (`np.ndarray`, `tf.Tensor`, `List[tf.Tensor]` `Dict[str, tf.Tensor]` or `Dict[str, np.ndarray]` and each example must have the shape `({0})`):
            Indices of input sequence tokens in the vocabulary.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.__call__`] and
            [`PreTrainedTokenizer.encode`] for details.

            [What are input IDs?](../glossary#input-ids)
        attention_mask (`np.ndarray` or `tf.Tensor` of shape `({0})`, *optional*):
            Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:

            - 1 for tokens that are **not masked**,
            - 0 for tokens that are **masked**.

            [What are attention masks?](../glossary#attention-mask)
        token_type_ids (`np.ndarray` or `tf.Tensor` of shape `({0})`, *optional*):
            Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
            1]`:

            - 0 corresponds to a *sentence A* token,
            - 1 corresponds to a *sentence B* token.

            [What are token type IDs?](../glossary#token-type-ids)
        position_ids (`np.ndarray` or `tf.Tensor` of shape `({0})`, *optional*):
            Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
            config.max_position_embeddings - 1]`.

            [What are position IDs?](../glossary#position-ids)
        head_mask (`np.ndarray` or `tf.Tensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*):
            Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`:

            - 1 indicates the head is **not masked**,
            - 0 indicates the head is **masked**.

        inputs_embeds (`np.ndarray` or `tf.Tensor` of shape `({0}, hidden_size)`, *optional*):
            Optionally, instead of passing `input_values` you can choose to directly pass an embedded representation.
            This is useful if you want more control over how to convert `input_values` indices into associated vectors
            than the model's internal embedding lookup matrix.
        output_attentions (`bool`, *optional*):
            Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
            tensors for more detail. This argument can be used only in eager mode, in graph mode the value in the
            config will be used instead.
        output_hidden_states (`bool`, *optional*):
            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
            more detail. This argument can be used only in eager mode, in graph mode the value in the config will be
            used instead.
        return_dict (`bool`, *optional*):
            Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. This argument can be used in
            eager mode, in graph mode the value will always be set to True.
        training (`bool`, *optional*, defaults to `False``):
            Whether or not to use the model in training mode (some modules like dropout modules have different
            behaviors between training and evaluation).
zaThe bare TFHubert Model transformer outputing raw hidden-states without any specific head on top.c                  ó¨   ‡ — e Zd Zdˆ fd„Z ee«       eee¬«      e		 	 	 	 	 	 	 	 	 d	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dd„«       «       «       Z
dd„Zˆ xZS )	ÚTFHubertModelc                ó^   •— t        ‰| �  |g|¢­i |¤Ž || _        t        |d¬«      | _        y )Nr  r§  )ri   rj   r›   rï  r  r  s       €r&   rj   zTFHubertModel.__init__˜  s/   ø€ Ü‰Ñ˜Ð3 &Ò3¨FÒ3ØˆŒÜ'¨°XÔ>ˆ�r(   ©Úoutput_typer  c                óØ   — |r|n| j                   j                  }|r|n| j                   j                  }|	r|	n| j                   j                  }	| j	                  |||||||||	|
¬«
      }|S )a^  

        Returns:

        Example:

        ```python
        >>> from transformers import AutoProcessor, TFHubertModel
        >>> from datasets import load_dataset
        >>> import soundfile as sf

        >>> processor = AutoProcessor.from_pretrained("facebook/hubert-large-ls960-ft")
        >>> model = TFHubertModel.from_pretrained("facebook/hubert-large-ls960-ft")


        >>> def map_to_array(batch):
        ...     speech, _ = sf.read(batch["file"])
        ...     batch["speech"] = speech
        ...     return batch


        >>> ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation")
        >>> ds = ds.map(map_to_array)

        >>> input_values = processor(ds["speech"][0], return_tensors="tf").input_values  # Batch size 1
        >>> hidden_states = model(input_values).last_hidden_state
        ```©
rM  r†  r  r  r  r  r°  r×  rØ  rh  )r›   r×  r°  rØ  r  )r{   rM  r†  r  r  r  r  r°  r×  rØ  rh  r–   s               r&   r—   zTFHubertModel.call�  s~   € ñX 8LÑ3ÐQU×Q\ÑQ\×QqÑQqÐÙ1BÑ-ÈÏÉ×HeÑHeÐÙ%0‘k°d·k±k×6MÑ6Mˆà—+‘+Ø%Ø)Ø)Ø%ØØ'Ø/Ø!5Ø#Øð ó 
ˆð ˆr(   c                óú   — | j                   ry d| _         t        | dd «      �Nt        j                  | j                  j
                  «      5  | j                  j                  d «       d d d «       y y # 1 sw Y   y xY w)NTr  )r†   r  r   r  r  rË   r‡   rŸ   s     r&   r‡   zTFHubertModel.buildÜ  si   € Ø�:Š:ØØˆŒ
Ü�4˜ 4Ó(Ð4Ü—‘˜tŸ{™{×/Ñ/Ó0ñ (Ø—‘×!Ñ! $Ô'÷(ð (ð 5÷(ð (ús   ÁA1Á1A:rk  r  )rM  r   r†  r‘  r  r‘  r  r‘  r  r‘  r  r‘  r°  r’  r×  r’  rØ  r’  rh  rÖ   r  rç  rž   )rÚ   rÛ   rÜ   rj   r   ÚHUBERT_INPUTS_DOCSTRINGr   r
   Ú_CONFIG_FOR_DOCr   r—   r‡   rÞ   rß   s   @r&   r#  r#  “  sË   ø„ õ
?ñ
 +Ð+BÓCÙÐ+<È?Ô[Øð ,0Ø+/Ø)-Ø&*Ø*.Ø,0Ø/3Ø&*Øð:àð:ð )ð:ð )ð	:ð
 'ð:ð $ð:ð (ð:ð *ð:ð -ð:ð $ð:ð ð:ð 
4ò:ó ó \ó Dð:÷x(r(   r#  zfTFHubert Model with a `language modeling` head on top for Connectionist Temporal Classification (CTC).c                  óº   ‡ — e Zd Zdˆ fd„Zd„ Zd„ Z ee«       ee	e
¬«      e	 	 	 	 	 	 	 	 	 	 d	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d	d„«       «       «       Zd
d„Zˆ xZS )ÚTFHubertForCTCc                ó‚  •— t        ‰| �  |g|¢­i |¤Ž t        |d¬«      | _        t        j
                  j                  |j                  «      | _        t        j
                  j                  |j                  d¬«      | _        t        |d«      r|j                  r|j                  | _        y |j                  | _        y )Nr  r§  Úlm_headÚadd_adapter)ri   rj   rï  r  r   rÆ   rb  Úfinal_dropoutrd  r`  Ú
vocab_sizer/  Úhasattrr0  Úoutput_hidden_sizer8  r  s       €r&   rj   zTFHubertForCTC.__init__ê  s˜   ø€ Ü‰Ñ˜Ð3 &Ò3¨FÒ3ä'¨°XÔ>ˆŒÜ—|‘|×+Ñ+¨F×,@Ñ,@ÓAˆŒÜ—|‘|×)Ñ)¨&×*;Ñ*;À)Ð)ÓLˆŒä)0°¸Ô)GÈF×L^ÒL^ˆF×%Ñ%ð 	ÕØdj×dvÑdvð 	Õr(   c                óX   — t        j                  dt        «       | j                  «        y)z©
        Calling this function will disable the gradient computation for the feature encoder so that its parameters will
        not be updated during training.
        zžThe method `freeze_feature_extractor` is deprecated and will be removed in Transformers v5. Please use the equivalent `freeze_feature_encoder` method instead.N)rS  rT  rV  Úfreeze_feature_encoderrÂ   s    r&   Úfreeze_feature_extractorz'TFHubertForCTC.freeze_feature_extractorô  s'   € ô
 	�‰ðQäô	
ð
 	×#Ñ#Õ%r(   c                ó:   — d| j                   j                  _        y)z¨
        Calling this function will disable the gradient computation for the feature encoder so that its parameter will
        not be updated during training.
        FN)r  rñ  rü   rÂ   s    r&   r6  z%TFHubertForCTC.freeze_feature_encoder   s   € ð
 38ˆ�‰×%Ñ%Õ/r(   r%  c                óV  — |�Nt        j                  |«      | j                  j                  k\  r"t	        d| j                  j                  › �«      ‚| j                  ||||||||	|
|¬«
      }|d   }| j                  ||¬«      }| j                  |«      }|��Y|�|n$t        j                  |t         j                  ¬«      }| j
                  j                  t        j                  |d¬«      «      }t        j                  |dk\  t         j                  «      }t        j                  |d¬«      }t         j                  j                  ||||| j                  j                   d	¬
«      }| j                  j"                  dk(  r+t        j                  |«      }t        j$                  |d«      }| j                  j"                  dk(  r.t        j&                  |«      }t        j$                  |d«      }nd}|
s|f|dd z   }|�|f|z   S |S t)        |||j*                  |j,                  ¬«      S )aÆ  
        labels (`tf.Tensor` or `np.ndarray` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
            config.vocab_size]` (see `input_values` docstring) Tokens with indices set to `-100` are ignored (masked),
            the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`

        Returns:

        Example:

        ```python
        >>> import tensorflow as tf
        >>> from transformers import AutoProcessor, TFHubertForCTC
        >>> from datasets import load_dataset
        >>> import soundfile as sf

        >>> processor = AutoProcessor.from_pretrained("facebook/hubert-large-ls960-ft")
        >>> model = TFHubertForCTC.from_pretrained("facebook/hubert-large-ls960-ft")


        >>> def map_to_array(batch):
        ...     speech, _ = sf.read(batch["file"])
        ...     batch["speech"] = speech
        ...     return batch


        >>> ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation")
        >>> ds = ds.map(map_to_array)

        >>> input_values = processor(ds["speech"][0], return_tensors="tf").input_values  # Batch size 1
        >>> logits = model(input_values).logits
        >>> predicted_ids = tf.argmax(logits, axis=-1)

        >>> transcription = processor.decode(predicted_ids[0])

        >>> # compute loss
        >>> target_transcription = "A MAN SAID TO THE UNIVERSE SIR I EXIST"

        >>> # Pass the transcription as text to encode labels
        >>> labels = processor(text=transcription, return_tensors="tf").input_values

        >>> loss = model(input_values, labels=labels).loss
        ```Nz$Label values must be <= vocab_size: r(  r   rg  r@   r*   r+   F)ÚlogitsÚlabelsÚlogit_lengthÚlabel_lengthÚblank_indexÚlogits_time_majorÚsumr?   r©   r   )Úlossr:  r  rÑ  )r   Ú
reduce_maxr›   r2  rB   r  rd  r/  rO   rF   rþ  rð   rE   rH   r   Úctc_lossÚpad_token_idÚctc_loss_reductionr-   Úreduce_meanr   r  rÑ  )r{   rM  r†  r  r  r  r  r°  r;  r×  rØ  rh  r–   r  r:  rý  Úlabels_maskÚtarget_lengthsrA  r  s                       r&   r—   zTFHubertForCTC.call  sþ  € ðx Ð¤"§-¡-°Ó"7¸4¿;¹;×;QÑ;QÒ"QÜÐCÀDÇKÁK×DZÑDZÐC[Ð\Ó]Ð]à—+‘+Ø%Ø)Ø)Ø%ØØ'Ø/Ø!5Ø#Øð ó 
ˆð   ™
ˆØŸ™ ]¸X˜ÓFˆà—‘˜mÓ,ˆàÑà"0Ð"<‘Ä"Ç,Á,È|Ôce×cmÑcmÔBnð ð !ŸK™K×HÑHÌÏÉÐWeÐlnÔIoÓpˆMô Ÿ'™' &¨A¡+¬r¯x©xÓ8ˆKÜŸ]™]¨;¸RÔ@ˆNä—5‘5—>‘>ØØØ*Ø+Ø ŸK™K×4Ñ4Ø"'ð "ó ˆDð �{‰{×-Ñ-°Ò6Ü—}‘} TÓ*�Ü—z‘z $¨Ó-�Ø�{‰{×-Ñ-°Ò7Ü—~‘~ dÓ+�Ü—z‘z $¨Ó-‘àˆDáØ�Y ¨¨ Ñ,ˆFØ)-Ð)9�T�G˜fÑ$ÐE¸vÐEäØØØ!×/Ñ/Ø×)Ñ)ô	
ð 	
r(   c                óà  — | j                   ry d| _         t        | dd «      �Mt        j                  | j                  j
                  «      5  | j                  j                  d «       d d d «       t        | dd «      �[t        j                  | j                  j
                  «      5  | j                  j                  d d | j                  g«       d d d «       y y # 1 sw Y   ŒrxY w# 1 sw Y   y xY w)NTr  r/  )	r†   r  r   r  r  rË   r‡   r/  r4  rŸ   s     r&   r‡   zTFHubertForCTC.build  sÇ   € Ø�:Š:ØØˆŒ
Ü�4˜ 4Ó(Ð4Ü—‘˜tŸ{™{×/Ñ/Ó0ñ (Ø—‘×!Ñ! $Ô'÷(ä�4˜ DÓ)Ð5Ü—‘˜tŸ|™|×0Ñ0Ó1ñ JØ—‘×"Ñ" D¨$°×0GÑ0GÐ#HÔI÷Jð Jð 6÷(ð (ú÷Jð Jús   ÁCÂ%)C$ÃC!Ã$C-rk  )
NNNNNNNNNF)rM  r   r†  r‘  r  r‘  r  r‘  r  r‘  r  r‘  r°  r’  r;  r‘  r×  r’  rØ  r’  rh  r’  r  z)Union[TFCausalLMOutput, Tuple[tf.Tensor]]rž   )rÚ   rÛ   rÜ   rj   r7  r6  r   r*  r   r   r+  r   r—   r‡   rÞ   rß   s   @r&   r-  r-  å  sñ   ø„ õ

ò
&ò8ñ +Ð+BÓCÙÐ+;È/ÔZØð ,0Ø+/Ø)-Ø&*Ø*.Ø,0Ø#'Ø/3Ø&*Ø#(ðs
àðs
ð )ðs
ð )ð	s
ð
 'ðs
ð $ðs
ð (ðs
ð *ðs
ð !ðs
ð -ðs
ð $ðs
ð !ðs
ð 
3òs
ó ó [ó Dðs
÷j	Jr(   r-  )r-  r#  r  r  )
rP   zTuple[int, int]rQ   rÕ   rR   rÔ   rS   rÔ   r  r   rž   )r_   r   r`   zOptional[int])ErÝ   Ú
__future__r   rS  Útypingr   r   r   r   ÚnumpyrÓ  Ú
tensorflowr   Úactivations_tfr	   Úmodeling_tf_outputsr
   r   Úmodeling_tf_utilsr   r   r   r   r   Útf_utilsr   r   Úutilsr   r   r   r   Úconfiguration_hubertr   Ú
get_loggerrÚ   r  r+  r^   r'   r:   r[   rd   rÆ   ÚLayerrf   r  rá   r  r#  r1  r6  r;  rE  rQ  rX  ro  r”  r¢  rº  r¿  ré  rï  r  ÚHUBERT_START_DOCSTRINGr*  r#  r-  Ú__all__rh   r(   r&   ú<module>rX     sÕ  ðñ å "ã ß .Ó .ã Û å /ß F÷õ ÷ 3÷ó õ /ð 
ˆ×	Ñ	˜HÓ	%€à €ð €òòOð( ð	GØðGàðGð ðGð ð	Gð
 óGôV
6ôU˜Ÿ™×*Ñ*ô Uôr5˜uŸ|™|×2Ñ2ô 5ôr@ 5§<¡<×#5Ñ#5ô @ô<G §¡×!3Ñ!3ô GôFG §¡×!3Ñ!3ô GôFG e§l¡l×&8Ñ&8ô Gô<˜5Ÿ<™<×-Ñ-ô ô '˜UŸ\™\×/Ñ/ô  'ôF
Ð5ô 
ôN §¡× 2Ñ 2ô NôBgB˜Ÿ™×*Ñ*ô gBôV)U˜%Ÿ,™,×,Ñ,ô )UôZ8S˜5Ÿ<™<×-Ñ-ô 8Sôx6S¨%¯,©,×*<Ñ*<ô 6SôtM&�e—l‘l×(Ñ(ô M&ôbO& U§\¡\×%7Ñ%7ô O&ðd ôK
˜Ÿ™×*Ñ*ó K
ó ðK
ô\
Ð/ô 
ð4(Ð ðT5Ð ñp ØgØóôK(Ð+ó K(ó	ðK(ñ\ ØpØóô_JÐ,ó _Jó	ð_JòD I�r(   