Ë
    S^(h� ã                   ó–  — d Z ddlZddlZddlZddlmZ ddlmZmZm	Z	 ddl
Z
ddlZ
ddl
mZ ddlmZmZmZ ddlmZ dd	lmZmZmZmZmZmZ dd
lmZ ddlmZmZmZ ddl m!Z!m"Z"m#Z#m$Z$m%Z% ddl&m'Z'  e$jP                  e)«      Z*dZ+dZ,g d¢Z-e G d„ de«      «       Z.d„ Z/ G d„ dej`                  «      Z1 G d„ dej`                  «      Z2 G d„ dej`                  «      Z3 G d„ dej`                  «      Z4 G d„ dej`                  «      Z5 G d„ d ej`                  «      Z6 G d!„ d"ej`                  «      Z7 G d#„ d$ej`                  «      Z8 G d%„ d&ej`                  «      Z9 G d'„ d(ej`                  «      Z: G d)„ d*ej`                  «      Z; G d+„ d,ej`                  «      Z< G d-„ d.ej`                  «      Z= G d/„ d0ej`                  «      Z> G d1„ d2e«      Z?d3Z@d4ZA e"d5e@«       G d6„ d7e?«      «       ZB e"d8e@«       G d9„ d:e?«      «       ZC e"d;e@«       G d<„ d=e?«      «       ZD e"d>e@«       G d?„ d@e?«      «       ZE e"dAe@«       G dB„ dCe?«      «       ZFg dD¢ZGy)EzPyTorch CANINE model.é    N)Ú	dataclass)ÚOptionalÚTupleÚUnion)Únn)ÚBCEWithLogitsLossÚCrossEntropyLossÚMSELossé   )ÚACT2FN)ÚBaseModelOutputÚModelOutputÚMultipleChoiceModelOutputÚQuestionAnsweringModelOutputÚSequenceClassifierOutputÚTokenClassifierOutput)ÚPreTrainedModel)Úapply_chunking_to_forwardÚ find_pruneable_heads_and_indicesÚprune_linear_layer)Úadd_code_sample_docstringsÚadd_start_docstringsÚ%add_start_docstrings_to_model_forwardÚloggingÚreplace_return_docstringsé   )ÚCanineConfigzgoogle/canine-sr   )é   é+   é;   é=   éI   éa   ég   éq   é‰   é•   é�   é­   éµ   éÁ   éÓ   éß   c                   ó¾   — e Zd ZU dZdZeej                     ed<   dZ	eej                     ed<   dZ
eeej                        ed<   dZeeej                        ed<   y)ÚCanineModelOutputWithPoolingaÿ  
    Output type of [`CanineModel`]. Based on [`~modeling_outputs.BaseModelOutputWithPooling`], but with slightly
    different `hidden_states` and `attentions`, as these also include the hidden states and attentions of the shallow
    Transformer encoders.

    Args:
        last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
            Sequence of hidden-states at the output of the last layer of the model (i.e. the output of the final
            shallow Transformer encoder).
        pooler_output (`torch.FloatTensor` of shape `(batch_size, hidden_size)`):
            Hidden-state of the first token of the sequence (classification token) at the last layer of the deep
            Transformer encoder, further processed by a Linear layer and a Tanh activation function. The Linear layer
            weights are trained from the next sentence prediction (classification) objective during pretraining.
        hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
            Tuple of `torch.FloatTensor` (one for the input to each encoder + one for the output of each layer of each
            encoder) of shape `(batch_size, sequence_length, hidden_size)` and `(batch_size, sequence_length //
            config.downsampling_rate, hidden_size)`. Hidden-states of the model at the output of each layer plus the
            initial input to each Transformer encoder. The hidden states of the shallow encoders have length
            `sequence_length`, but the hidden states of the deep encoder have length `sequence_length` //
            `config.downsampling_rate`.
        attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
            Tuple of `torch.FloatTensor` (one for each layer) of the 3 Transformer encoders of shape `(batch_size,
            num_heads, sequence_length, sequence_length)` and `(batch_size, num_heads, sequence_length //
            config.downsampling_rate, sequence_length // config.downsampling_rate)`. Attentions weights after the
            attention softmax, used to compute the weighted average in the self-attention heads.
    NÚlast_hidden_stateÚpooler_outputÚhidden_statesÚ
attentions)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r0   r   ÚtorchÚFloatTensorÚ__annotations__r1   r2   r   r3   © ó    úh/var/www/skyplay_api_hub/venv/lib/python3.12/site-packages/transformers/models/canine/modeling_canine.pyr/   r/   ;   sh   … ñð6 6:Ð�x × 1Ñ 1Ñ2Ó9Ø15€M�8˜E×-Ñ-Ñ.Ó5Ø8<€M�8˜E %×"3Ñ"3Ñ4Ñ5Ó<Ø59€J�˜˜u×0Ñ0Ñ1Ñ2Ô9r<   r/   c           	      óØ  — 	 ddl }ddl}ddl}t        j                  j                  |«      }t        j                  d|› �«       |j                  j                  |«      }g }g }	|D ]^  \  }
}t        j                  d|
› d|› �«       |j                  j                  ||
«      }|j                  |
«       |	j                  |«       Œ` t        ||	«      D �]4  \  }
}|
j                  d«      }
t!        d„ |
D «       «      r(t        j                  d	dj#                  |
«      › �«       ŒR|
d   d
k(  rd|
d<   nU|
d   dk(  r|
j%                  |
d   «       n8|
d   dk(  rd|
d<   n*|
d   dk(  r
dg|
dd z   }
n|
d   dk(  r|
d   dv r	dg|
dd z   }
| }|
D ]¯  }|j'                  d|«      rd|vr|j                  d|«      }n|g}|d   dk(  s|d   dk(  rt)        |d«      }nB|d   dk(  s|d   dk(  rt)        |d«      }n%|d   dk(  rt)        |d«      }n	 t)        ||d   «      }t-        |«      d k\  sŒ�t/        |d   «      }||   }Œ± d!d d"k(  rt)        |d«      }nD|d#d t1        d$«      D �cg c]  }d%|› �‘Œ	 c}v rt)        |d«      }n|dk(  r|j3                  |«      }|j4                  |j4                  k7  r&t7        d&|j4                  › d'|j4                  › d(�«      ‚t        j                  d)|
› �«       t9        j:                  |«      |_        �Œ7 | S # t        $ r t        j                  d«       ‚ w xY w# t*        $ r+ t        j                  d	dj#                  |
«      › �«       Y �Œ×w xY wc c}w )*z'Load tf checkpoints in a pytorch model.r   Nz™Loading a TensorFlow model in PyTorch, requires TensorFlow to be installed. Please see https://www.tensorflow.org/install/ for installation instructions.z&Converting TensorFlow checkpoint from zLoading TF weight z with shape ú/c              3   ó$   K  — | ]  }|d v –— Œ
 y­w))Úadam_vÚadam_mÚAdamWeightDecayOptimizerÚAdamWeightDecayOptimizer_1Úglobal_stepÚclsÚautoregressive_decoderÚchar_output_weightsNr;   )Ú.0Úns     r=   ú	<genexpr>z,load_tf_weights_in_canine.<locals>.<genexpr>|   s$   è ø€ ò 
ð ð ð	ô
ñ
ùs   ‚z	Skipping ÚbertÚencoderr   Ú
embeddingsÚsegment_embeddingsÚtoken_type_embeddingsÚinitial_char_encoderÚchars_to_moleculeséþÿÿÿÚfinal_char_encoder)Ú	LayerNormÚconvÚ
projectionz[A-Za-z]+_\d+ÚEmbedderz_(\d+)ÚkernelÚgammaÚweightÚoutput_biasÚbetaÚbiasÚoutput_weightsé   iõÿÿÿÚ_embeddingsiöÿÿÿé   Ú	Embedder_zPointer shape z and array shape z mismatchedzInitialize PyTorch weight )ÚreÚnumpyÚ
tensorflowÚImportErrorÚloggerÚerrorÚosÚpathÚabspathÚinfoÚtrainÚlist_variablesÚload_variableÚappendÚzipÚsplitÚanyÚjoinÚremoveÚ	fullmatchÚgetattrÚAttributeErrorÚlenÚintÚrangeÚ	transposeÚshapeÚ
ValueErrorr8   Ú
from_numpyÚdata)ÚmodelÚconfigÚtf_checkpoint_pathrd   ÚnpÚtfÚtf_pathÚ	init_varsÚnamesÚarraysÚnamer~   ÚarrayÚpointerÚm_nameÚscope_namesÚnumÚis                     r=   Úload_tf_weights_in_caniner’   ^   s¦  € ð
ÛãÛô �g‰g�o‰oÐ0Ó1€GÜ
‡K�KÐ8¸¸	ÐBÔCà—‘×'Ñ'¨Ó0€IØ€EØ€FØ ò ‰ˆˆeÜ�‰Ð(¨¨¨l¸5¸'ÐBÔCØ—‘×&Ñ& w°Ó5ˆØ�‰�TÔØ�‰�eÕð	ô ˜5 &Ó)ó D/‰ˆˆeØ�z‰z˜#‹ˆô ñ 
ð ô
ô 
ô �K‰K˜) C§H¡H¨T£NÐ#3Ð4Ô5Øà�‰7�fÒØˆD�ŠGà�!‰W˜Ò$Ø�K‰K˜˜Q™Õ à�!‰WÐ,Ò,Ø-ˆD�ŠGà�!‰WÐ.Ò.Ø(Ð)¨D°°¨IÑ5‰Dà�!‰WÐ,Ò,°°a±Ð<QÑ1QØ �> D¨¨ HÑ,ˆDØˆØò 	'ˆFØ—‘Ð-¨vÔ6¸JÈfÑ<TØ Ÿh™h y°&Ó9‘à%˜h�Ø˜1‰~ Ò)¨[¸©^¸wÒ-FÜ! '¨8Ó4‘Ø˜Q‘ =Ò0°KÀ±NÀfÒ4LÜ! '¨6Ó2‘Ø˜Q‘Ð#3Ò3Ü! '¨8Ó4‘ðÜ% g¨{¸1©~Ó>�Gô �;Ó 1Ó$Ü˜+ a™.Ó)�Ø! #™,‘ð'	'ð( �#�$ˆ<˜=Ò(Ü˜g xÓ0‰GØ�C�Dˆ\´u¸Q³xÖ@°! 	¨!¨šoÒ@Ñ@Ü˜g xÓ0‰GØ�xÒØ—L‘L Ó'ˆEà�=‰=˜EŸK™KÒ'Ü˜~¨g¯m©m¨_Ð<MÈeÏkÉkÈ]ÐZeÐfÓgÐgä�‰Ð0°°Ð7Ô8Ü×'Ñ'¨Ó.ˆŽðID/ðJ €Løôo ò Ü�‰ðQô	
ð 	ðûôJ &ò Ü—K‘K )¨C¯H©H°T«NÐ+;Ð <Ô=Úðüò As#   ‚L ÈL0É$M'Ì L-Ì00M$Í#M$c                   óè   ‡ — e Zd ZdZˆ fd„Zdedefd„Zdededefd„Z	 	 	 	 ddee	j                     d	ee	j                     d
ee	j                     dee	j                     de	j                  f
d„Zˆ xZS )ÚCanineEmbeddingsz<Construct the character, position and token_type embeddings.c           	      óþ  •— t         ‰| �  «        || _        |j                  |j                  z  }t        |j                  «      D ]2  }d|› �}t        | |t        j                  |j                  |«      «       Œ4 t        j                  |j                  |j                  «      | _
        t        j                  |j                  |j                  «      | _        t        j                  |j                  |j                  ¬«      | _        t        j                  |j                   «      | _        | j%                  dt'        j(                  |j*                  «      j-                  d«      d¬«       t/        |dd«      | _        y )	NÚHashBucketCodepointEmbedder_©ÚepsÚposition_ids)r   éÿÿÿÿF)Ú
persistentÚposition_embedding_typeÚabsolute)ÚsuperÚ__init__rƒ   Úhidden_sizeÚnum_hash_functionsr|   Úsetattrr   Ú	EmbeddingÚnum_hash_bucketsÚchar_position_embeddingsÚtype_vocab_sizerP   rU   Úlayer_norm_epsÚDropoutÚhidden_dropout_probÚdropoutÚregister_bufferr8   ÚarangeÚmax_position_embeddingsÚexpandrx   rœ   )Úselfrƒ   Úshard_embedding_sizer‘   r‹   Ú	__class__s        €r=   rŸ   zCanineEmbeddings.__init__Â   s7  ø€ Ü‰ÑÔàˆŒð  &×1Ñ1°V×5NÑ5NÑNÐÜ�v×0Ñ0Ó1ò 	]ˆAØ1°!°Ð5ˆDÜ�D˜$¤§¡¨V×-DÑ-DÐFZÓ [Õ\ð	]ô )+¯©°V×5LÑ5LÈf×N`ÑN`Ó(aˆÔ%Ü%'§\¡\°&×2HÑ2HÈ&×J\ÑJ\Ó%]ˆÔ"ô Ÿ™ f×&8Ñ&8¸f×>SÑ>SÔTˆŒÜ—z‘z &×"<Ñ"<Ó=ˆŒð 	×ÑØœEŸL™L¨×)GÑ)GÓH×OÑOÐPWÓXÐejð 	ô 	
ô (/¨vÐ7PÐR\Ó']ˆÕ$r<   Ú
num_hashesÚnum_bucketsc                 ó¼   — |t        t        «      kD  rt        dt        t        «      › �«      ‚t        d| }g }|D ]  }|dz   |z  |z  }|j                  |«       Œ  |S )a—  
        Converts ids to hash bucket ids via multiple hashing.

        Args:
            input_ids: The codepoints or other IDs to be hashed.
            num_hashes: The number of hash functions to use.
            num_buckets: The number of hash buckets (i.e. embeddings in each table).

        Returns:
            A list of tensors, each of which is the hash bucket IDs from one hash function.
        z`num_hashes` must be <= Nr   )rz   Ú_PRIMESr   rq   )r¯   Ú	input_idsr²   r³   ÚprimesÚresult_tensorsÚprimeÚhasheds           r=   Ú_hash_bucket_tensorsz%CanineEmbeddings._hash_bucket_tensorsÚ   sp   € ð œœG›Ò$ÜÐ7¼¼G»°~ÐFÓGÐGä˜˜*Ð%ˆàˆØò 	*ˆEØ  1‘}¨Ñ-°Ñ<ˆFØ×!Ñ! &Õ)ð	*ð Ðr<   Úembedding_sizec                 ó  — ||z  dk7  rt        d|› d|› d�«      ‚| j                  |||¬«      }g }t        |«      D ]-  \  }}d|› �}	 t        | |	«      |«      }
|j	                  |
«       Œ/ t        j                  |d¬«      S )	zDConverts IDs (e.g. codepoints) into embeddings via multiple hashing.r   zExpected `embedding_size` (z) % `num_hashes` (z) == 0)r²   r³   r–   rš   ©Údim)r   r»   Ú	enumeraterx   rq   r8   Úcat)r¯   r¶   r¼   r²   r³   Úhash_bucket_tensorsÚembedding_shardsr‘   Úhash_bucket_idsr‹   Úshard_embeddingss              r=   Ú_embed_hash_bucketsz$CanineEmbeddings._embed_hash_bucketsñ   s«   € à˜JÑ&¨!Ò+ÜÐ:¸>Ð:JÐJ\Ð]gÐ\hÐhnÐoÓpÐpà"×7Ñ7¸	ÈjÐfqÐ7ÓrÐØÐÜ"+Ð,?Ó"@ò 	6ÑˆAˆØ1°!°Ð5ˆDØ2œw t¨TÓ2°?ÓCÐØ×#Ñ#Ð$4Õ5ð	6ô
 �y‰yÐ)¨rÔ2Ð2r<   r¶   Útoken_type_idsr™   Úinputs_embedsÚreturnc                 ó`  — |�|j                  «       }n|j                  «       d d }|d   }|€| j                  d d …d |…f   }|€:t        j                  |t        j                  | j                  j
                  ¬«      }|€P| j                  || j                  j                  | j                  j                  | j                  j                  «      }| j                  |«      }||z   }| j                  dk(  r| j                  |«      }	||	z  }| j                  |«      }| j                  |«      }|S )Nrš   r   ©ÚdtypeÚdevicer�   )Úsizer™   r8   ÚzerosÚlongrÍ   rÆ   rƒ   r    r¡   r¤   rP   rœ   r¥   rU   rª   )
r¯   r¶   rÇ   r™   rÈ   Úinput_shapeÚ
seq_lengthrP   rN   Úposition_embeddingss
             r=   ÚforwardzCanineEmbeddings.forwardÿ   s$  € ð Ð Ø#Ÿ.™.Ó*‰Kà'×,Ñ,Ó.¨s°Ð3ˆKà  ‘^ˆ
àÐØ×,Ñ,ªQ°°°¨^Ñ<ˆLàÐ!Ü"Ÿ[™[¨¼E¿J¹JÈt×O`ÑO`×OgÑOgÔhˆNàÐ Ø ×4Ñ4Ø˜4Ÿ;™;×2Ñ2°D·K±K×4RÑ4RÐTX×T_ÑT_×TpÑTpóˆMð !%× :Ñ :¸>Ó JÐà"Ð%:Ñ:ˆ
à×'Ñ'¨:Ò5Ø"&×"?Ñ"?ÀÓ"MÐØÐ-Ñ-ˆJØ—^‘^ JÓ/ˆ
Ø—\‘\ *Ó-ˆ
ØÐr<   )NNNN)r4   r5   r6   r7   rŸ   r{   r»   rÆ   r   r8   Ú
LongTensorr9   rÔ   Ú__classcell__©r±   s   @r=   r”   r”   ¿   s¯   ø„ ÙFô^ð0¸#ð ÈCó ð.3¸Sð 3Ècð 3Ð`có 3ð  15Ø59Ø37Ø59ñ"à˜E×,Ñ,Ñ-ð"ð ! ×!1Ñ!1Ñ2ð"ð ˜u×/Ñ/Ñ0ð	"ð
   × 1Ñ 1Ñ2ð"ð 
×	Ñ	÷"r<   r”   c                   óZ   ‡ — e Zd ZdZˆ fd„Zdej                  dej                  fd„Zˆ xZS )ÚCharactersToMoleculeszeConvert character sequence to initial molecule sequence (i.e. downsample) using strided convolutions.c                 ó>  •— t         ‰| �  «        t        j                  |j                  |j                  |j
                  |j
                  ¬«      | _        t        |j                     | _	        t        j                  |j                  |j                  ¬«      | _
        y )N©Úin_channelsÚout_channelsÚkernel_sizeÚstrider—   )rž   rŸ   r   ÚConv1dr    Údownsampling_raterV   r   Ú
hidden_actÚ
activationrU   r§   ©r¯   rƒ   r±   s     €r=   rŸ   zCharactersToMolecules.__init__'  sv   ø€ Ü‰ÑÔä—I‘IØ×*Ñ*Ø×+Ñ+Ø×0Ñ0Ø×+Ñ+ô	
ˆŒ	ô ! ×!2Ñ!2Ñ3ˆŒô Ÿ™ f×&8Ñ&8¸f×>SÑ>SÔTˆ�r<   Úchar_encodingrÉ   c                 ó2  — |d d …dd…d d …f   }t        j                  |dd«      }| j                  |«      }t        j                  |dd«      }| j                  |«      }|d d …dd…d d …f   }t        j                  ||gd¬«      }| j                  |«      }|S )Nr   r   r`   rš   r¾   )r8   r}   rV   rã   rÁ   rU   )r¯   rå   Úcls_encodingÚdownsampledÚdownsampled_truncatedÚresults         r=   rÔ   zCharactersToMolecules.forward6  sš   € à$¢Q¨¨!¨ªQ YÑ/ˆô Ÿ™¨°q¸!Ó<ˆØ—i‘i Ó.ˆÜ—o‘o k°1°aÓ8ˆØ—o‘o kÓ2ˆð !,ªA¨q°¨t²Q¨JÑ 7Ðô —‘˜LÐ*?Ð@ÀaÔHˆà—‘ Ó'ˆàˆr<   )	r4   r5   r6   r7   rŸ   r8   ÚTensorrÔ   rÖ   r×   s   @r=   rÙ   rÙ   $  s'   ø„ ÙoôUð U§\¡\ð °e·l±l÷ r<   rÙ   c                   ó|   ‡ — e Zd ZdZˆ fd„Z	 ddej                  deej                     dej                  fd„Zˆ xZ	S )ÚConvProjectionz�
    Project representations from hidden_size*2 back to hidden_size across a window of w = config.upsampling_kernel_size
    characters.
    c                 ó†  •— t         ‰| �  «        || _        t        j                  |j
                  dz  |j
                  |j                  d¬«      | _        t        |j                     | _
        t        j                  |j
                  |j                  ¬«      | _        t        j                  |j                  «      | _        y )Nr`   r   rÛ   r—   )rž   rŸ   rƒ   r   rà   r    Úupsampling_kernel_sizerV   r   râ   rã   rU   r§   r¨   r©   rª   rä   s     €r=   rŸ   zConvProjection.__init__X  s’   ø€ Ü‰ÑÔØˆŒÜ—I‘IØ×*Ñ*¨QÑ.Ø×+Ñ+Ø×5Ñ5Øô	
ˆŒ	ô ! ×!2Ñ!2Ñ3ˆŒô Ÿ™ f×&8Ñ&8¸f×>SÑ>SÔTˆŒÜ—z‘z &×"<Ñ"<Ó=ˆ�r<   ÚinputsÚfinal_seq_char_positionsrÉ   c                 óŽ  — t        j                  |dd«      }| j                  j                  dz
  }|dz  }||z
  }t	        j
                  ||fd«      }| j                   ||«      «      }t        j                  |dd«      }| j                  |«      }| j                  |«      }| j                  |«      }|}|�t        d«      ‚|}	|	S )Nr   r`   r   z,CanineForMaskedLM is currently not supported)r8   r}   rƒ   rï   r   ÚConstantPad1drV   rã   rU   rª   ÚNotImplementedError)
r¯   rð   rñ   Ú	pad_totalÚpad_begÚpad_endÚpadrê   Úfinal_char_seqÚ	query_seqs
             r=   rÔ   zConvProjection.forwardg  sÈ   € ô —‘ ¨¨AÓ.ˆð
 —K‘K×6Ñ6¸Ñ:ˆ	Ø˜q‘.ˆØ˜gÑ%ˆä×Ñ ¨Ð1°1Ó5ˆà—‘™3˜v›;Ó'ˆÜ—‘ ¨¨AÓ.ˆØ—‘ Ó(ˆØ—‘ Ó'ˆØ—‘˜fÓ%ˆØˆà#Ð/ô
 &Ð&TÓUÐUà&ˆIàÐr<   ©N)
r4   r5   r6   r7   rŸ   r8   rë   r   rÔ   rÖ   r×   s   @r=   rí   rí   R  sE   ø„ ñô
>ð$ <@ñ"à—‘ð"ð #+¨5¯<©<Ñ"8ð"ð 
�‰÷	"r<   rí   c                   óæ   ‡ — e Zd Zˆ fd„Zd„ Z	 	 	 d
dej                  dej                  deej                     deej                     dee	   de
ej                  eej                     f   fd	„Zˆ xZS )ÚCanineSelfAttentionc                 óº  •— t         ‰| �  «        |j                  |j                  z  dk7  r2t	        |d«      s&t        d|j                  › d|j                  › d�«      ‚|j                  | _        t        |j                  |j                  z  «      | _        | j                  | j                  z  | _        t        j                  |j                  | j                  «      | _        t        j                  |j                  | j                  «      | _        t        j                  |j                  | j                  «      | _        t        j                  |j                  «      | _        t#        |dd«      | _        | j$                  dk(  s| j$                  d	k(  rG|j&                  | _        t        j(                  d
|j&                  z  dz
  | j                  «      | _        y y )Nr   r¼   zThe hidden size (z6) is not a multiple of the number of attention heads (ú)rœ   r�   Úrelative_keyÚrelative_key_queryr`   r   )rž   rŸ   r    Únum_attention_headsÚhasattrr   r{   Úattention_head_sizeÚall_head_sizer   ÚLinearÚqueryÚkeyÚvaluer¨   Úattention_probs_dropout_probrª   rx   rœ   r­   r£   Údistance_embeddingrä   s     €r=   rŸ   zCanineSelfAttention.__init__�  sŠ  ø€ Ü‰ÑÔØ×Ñ × :Ñ :Ñ:¸aÒ?ÌÐPVÐXhÔHiÜØ# F×$6Ñ$6Ð#7ð 8Ø ×4Ñ4Ð5°Qð8óð ð
 $*×#=Ñ#=ˆÔ Ü#& v×'9Ñ'9¸F×<VÑ<VÑ'VÓ#WˆÔ Ø!×5Ñ5¸×8PÑ8PÑPˆÔä—Y‘Y˜v×1Ñ1°4×3EÑ3EÓFˆŒ
Ü—9‘9˜V×/Ñ/°×1CÑ1CÓDˆŒÜ—Y‘Y˜v×1Ñ1°4×3EÑ3EÓFˆŒ
ä—z‘z &×"EÑ"EÓFˆŒÜ'.¨vÐ7PÐR\Ó']ˆÔ$Ø×'Ñ'¨>Ò9¸T×=YÑ=YÐ]qÒ=qØ+1×+IÑ+IˆDÔ(Ü&(§l¡l°1°v×7UÑ7UÑ3UÐXYÑ3YÐ[_×[sÑ[sÓ&tˆDÕ#ð >rr<   c                 ó    — |j                  «       d d | j                  | j                  fz   } |j                  |Ž }|j	                  dddd«      S )Nrš   r   r`   r   r   )rÎ   r  r  ÚviewÚpermute)r¯   ÚxÚnew_x_shapes      r=   Útranspose_for_scoresz(CanineSelfAttention.transpose_for_scores£  sN   € Ø—f‘f“h˜s �m t×'?Ñ'?À×AYÑAYÐ&ZÑZˆØˆA�F‰F�KÐ ˆØ�y‰y˜˜A˜q !Ó$Ð$r<   Úfrom_tensorÚ	to_tensorÚattention_maskÚ	head_maskÚoutput_attentionsrÉ   c                 óP  — | j                  |«      }| j                  | j                  |«      «      }| j                  | j                  |«      «      }| j                  |«      }	t	        j
                  |	|j                  dd«      «      }
| j                  dk(  s| j                  dk(  �rF|j                  «       d   }t	        j                  |t        j                  |j                  ¬«      j                  dd«      }t	        j                  |t        j                  |j                  ¬«      j                  dd«      }||z
  }| j                  || j                  z   dz
  «      }|j                  |	j                   ¬«      }| j                  dk(  rt	        j"                  d|	|«      }|
|z   }
nE| j                  dk(  r6t	        j"                  d|	|«      }t	        j"                  d	||«      }|
|z   |z   }
|
t%        j&                  | j(                  «      z  }
|�h|j*                  d
k(  rTt	        j,                  |d¬«      }d|j/                  «       z
  t	        j0                  |
j                   «      j2                  z  }|
|z   }
t4        j6                  j9                  |
d¬«      }| j;                  |«      }|�||z  }t	        j
                  ||«      }|j=                  dddd
«      j?                  «       }|j                  «       d d | j@                  fz   } |j                  |Ž }|r||f}|S |f}|S )Nrš   rS   r   r  r   rË   )rÌ   zbhld,lrd->bhlrzbhrd,lrd->bhlrr   r¾   ç      ð?r   r`   )!r  r  r  r	  r8   Úmatmulr}   rœ   rÎ   r¬   rÐ   rÍ   r  r  r­   ÚtorÌ   ÚeinsumÚmathÚsqrtr  ÚndimÚ	unsqueezeÚfloatÚfinfoÚminr   Ú
functionalÚsoftmaxrª   r  Ú
contiguousr  )r¯   r  r  r  r  r  Úmixed_query_layerÚ	key_layerÚvalue_layerÚquery_layerÚattention_scoresrÒ   Úposition_ids_lÚposition_ids_rÚdistanceÚpositional_embeddingÚrelative_position_scoresÚrelative_position_scores_queryÚrelative_position_scores_keyÚattention_probsÚcontext_layerÚnew_context_layer_shapeÚoutputss                          r=   rÔ   zCanineSelfAttention.forward¨  sý  € ð !ŸJ™J {Ó3Ðð ×-Ñ-¨d¯h©h°yÓ.AÓBˆ	Ø×/Ñ/°·
±
¸9Ó0EÓFˆà×/Ñ/Ð0AÓBˆô !Ÿ<™<¨°Y×5HÑ5HÈÈRÓ5PÓQÐà×'Ñ'¨>Ò9¸T×=YÑ=YÐ]qÓ=qØ$×)Ñ)Ó+¨AÑ.ˆJÜ"Ÿ\™\¨*¼E¿J¹JÈ{×OaÑOaÔb×gÑgÐhjÐlmÓnˆNÜ"Ÿ\™\¨*¼E¿J¹JÈ{×OaÑOaÔb×gÑgÐhiÐkmÓnˆNØ%¨Ñ6ˆHØ#'×#:Ñ#:¸8Àd×FbÑFbÑ;bÐefÑ;fÓ#gÐ Ø#7×#:Ñ#:À×ARÑARÐ#:Ó#SÐ à×+Ñ+¨~Ò=Ü+0¯<©<Ð8HÈ+ÐWkÓ+lÐ(Ø#3Ð6NÑ#NÑ Ø×-Ñ-Ð1EÒEÜ16·±Ð>NÐP[Ð]qÓ1rÐ.Ü/4¯|©|Ð<LÈiÐYmÓ/nÐ,Ø#3Ð6TÑ#TÐWsÑ#sÐ à+¬d¯i©i¸×8PÑ8PÓ.QÑQÐØÐ%Ø×"Ñ" aÒ'ä!&§¡°ÀQÔ!G�ð #&¨×(<Ñ(<Ó(>Ñ">Ä%Ç+Á+ÐN^×NdÑNdÓBe×BiÑBiÑ!i�à/°.Ñ@Ðô Ÿ-™-×/Ñ/Ð0@ÀbÐ/ÓIˆð Ÿ,™, Ó7ˆð Ð Ø-°	Ñ9ˆOäŸ™ _°kÓBˆà%×-Ñ-¨a°°A°qÓ9×DÑDÓFˆØ"/×"4Ñ"4Ó"6°s¸Ð";¸t×?QÑ?QÐ>SÑ"SÐØ*˜×*Ñ*Ð,CÐDˆá6G�= /Ð2ˆàˆð O\ÐM]ˆàˆr<   ©NNF)r4   r5   r6   rŸ   r  r8   rë   r   r9   Úboolr   rÔ   rÖ   r×   s   @r=   rý   rý   Œ  s˜   ø„ ôuò,%ð 7;Ø15Ø,1ñEà—\‘\ðEð —<‘<ðEð ! ×!2Ñ!2Ñ3ð	Eð
 ˜E×-Ñ-Ñ.ðEð $ D™>ðEð 
ˆu�|‰|˜X e§l¡lÑ3Ð3Ñ	4÷Er<   rý   c                   ó’   ‡ — e Zd Zˆ fd„Zdeej                     dej                  deej                  ej                  f   fd„Zˆ xZS )ÚCanineSelfOutputc                 ó(  •— t         ‰| �  «        t        j                  |j                  |j                  «      | _        t        j                  |j                  |j                  ¬«      | _        t        j                  |j                  «      | _
        y ©Nr—   )rž   rŸ   r   r  r    ÚdenserU   r§   r¨   r©   rª   rä   s     €r=   rŸ   zCanineSelfOutput.__init__ñ  s`   ø€ Ü‰ÑÔÜ—Y‘Y˜v×1Ñ1°6×3EÑ3EÓFˆŒ
ÜŸ™ f×&8Ñ&8¸f×>SÑ>SÔTˆŒÜ—z‘z &×"<Ñ"<Ó=ˆ�r<   r2   Úinput_tensorrÉ   c                 ór   — | j                  |«      }| j                  |«      }| j                  ||z   «      }|S rû   ©r<  rª   rU   ©r¯   r2   r=  s      r=   rÔ   zCanineSelfOutput.forward÷  s9   € ð Ÿ
™
 =Ó1ˆØŸ™ ]Ó3ˆØŸ™ }°|Ñ'CÓDˆØÐr<   ©	r4   r5   r6   rŸ   r   r8   r9   rÔ   rÖ   r×   s   @r=   r9  r9  ð  sL   ø„ ô>ðØ" 5×#4Ñ#4Ñ5ðØEJ×EVÑEVðà	ˆu× Ñ  %×"3Ñ"3Ð3Ñ	4÷r<   r9  c                   ó  ‡ — e Zd ZdZ	 	 	 	 	 	 	 ddedededededefˆ fd„Zd	„ Z	 	 	 dd
ee	j                     dee	j                     dee	j                     dee   dee	j                  ee	j                     f   f
d„Zˆ xZS )ÚCanineAttentionav  
    Additional arguments related to local attention:

        - **local** (`bool`, *optional*, defaults to `False`) -- Whether to apply local attention.
        - **always_attend_to_first_position** (`bool`, *optional*, defaults to `False`) -- Should all blocks be able to
          attend
        to the `to_tensor`'s first position (e.g. a [CLS] position)? - **first_position_attends_to_all** (`bool`,
        *optional*, defaults to `False`) -- Should the *from_tensor*'s first position be able to attend to all
        positions within the *from_tensor*? - **attend_from_chunk_width** (`int`, *optional*, defaults to 128) -- The
        width of each block-wise chunk in `from_tensor`. - **attend_from_chunk_stride** (`int`, *optional*, defaults to
        128) -- The number of elements to skip when moving to the next block in `from_tensor`. -
        **attend_to_chunk_width** (`int`, *optional*, defaults to 128) -- The width of each block-wise chunk in
        *to_tensor*. - **attend_to_chunk_stride** (`int`, *optional*, defaults to 128) -- The number of elements to
        skip when moving to the next block in `to_tensor`.
    Úalways_attend_to_first_positionÚfirst_position_attends_to_allÚattend_from_chunk_widthÚattend_from_chunk_strideÚattend_to_chunk_widthÚattend_to_chunk_stridec	                 ó"  •— t         ‰	| �  «        t        |«      | _        t	        |«      | _        t        «       | _        || _        ||k  rt        d«      ‚||k  rt        d«      ‚|| _
        || _        || _        || _        || _        || _        y )Nze`attend_from_chunk_width` < `attend_from_chunk_stride` would cause sequence positions to get skipped.z``attend_to_chunk_width` < `attend_to_chunk_stride`would cause sequence positions to get skipped.)rž   rŸ   rý   r¯   r9  ÚoutputÚsetÚpruned_headsÚlocalr   rD  rE  rF  rG  rH  rI  ©
r¯   rƒ   rN  rD  rE  rF  rG  rH  rI  r±   s
            €r=   rŸ   zCanineAttention.__init__  s¢   ø€ ô 	‰ÑÔÜ'¨Ó/ˆŒ	Ü& vÓ.ˆŒÜ›EˆÔð ˆŒ
Ø"Ð%=Ò=ÜØwóð ð !Ð#9Ò9ÜØróð ð 0OˆÔ,Ø-JˆÔ*Ø'>ˆÔ$Ø(@ˆÔ%Ø%:ˆÔ"Ø&<ˆÕ#r<   c                 ó>  — t        |«      dk(  ry t        || j                  j                  | j                  j                  | j
                  «      \  }}t        | j                  j                  |«      | j                  _        t        | j                  j                  |«      | j                  _        t        | j                  j                  |«      | j                  _	        t        | j                  j                  |d¬«      | j                  _        | j                  j                  t        |«      z
  | j                  _        | j                  j                  | j                  j                  z  | j                  _        | j
                  j                  |«      | _        y )Nr   r   r¾   )rz   r   r¯   r  r  rM  r   r  r  r	  rK  r<  r  Úunion)r¯   ÚheadsÚindexs      r=   Úprune_headszCanineAttention.prune_heads2  s  € Üˆu‹:˜Š?ØÜ7Ø�4—9‘9×0Ñ0°$·)±)×2OÑ2OÐQU×QbÑQbó
‰ˆˆuô
 -¨T¯Y©Y¯_©_¸eÓDˆ�	‰	ŒÜ*¨4¯9©9¯=©=¸%Ó@ˆ�	‰	ŒÜ,¨T¯Y©Y¯_©_¸eÓDˆ�	‰	ŒÜ.¨t¯{©{×/@Ñ/@À%ÈQÔOˆ�‰Ôð )-¯	©	×(EÑ(EÌÈEË
Ñ(Rˆ�	‰	Ô%Ø"&§)¡)×"?Ñ"?À$Ç)Á)×B_ÑB_Ñ"_ˆ�	‰	ÔØ ×-Ñ-×3Ñ3°EÓ:ˆÕr<   r2   r  r  r  rÉ   c                 óà  — | j                   s| j                  |||||«      }|d   }�n|j                  d   x}}|x}	}
g }| j                  r|j	                  d«       d}nd}t        ||| j                  «      D ].  }t        ||| j                  z   «      }|j	                  ||f«       Œ0 g }| j                  r|j	                  d|f«       t        d|| j                  «      D ].  }t        ||| j                  z   «      }|j	                  ||f«       Œ0 t        |«      t        |«      k7  rt        d|› d|› d�«      ‚g }g }t        ||«      D ]Ï  \  \  }}\  }}|	d d …||…d d …f   }|
d d …||…d d …f   }|d d …||…||…f   }| j                  rN|d d …||…dd…f   }t        j                   ||gd¬«      }|
d d …dd…d d …f   }t        j                   ||gd¬«      }| j                  |||||«      }|j	                  |d   «       |sŒ¼|j	                  |d   «       ŒÑ t        j                   |d¬«      }| j#                  ||«      }|f}| j                   s
|dd  z   }|S |t%        «      z   }|S )	Nr   r   )r   r   z/Expected to have same number of `from_chunks` (z) and `to_chunks` (z). Check strides.r`   r¾   )rN  r¯   r~   rE  rq   r|   rG  r"  rF  rI  rH  rz   r   rr   rD  r8   rÁ   rK  Útuple)r¯   r2   r  r  r  Úself_outputsÚattention_outputÚfrom_seq_lengthÚto_seq_lengthr  r  Úfrom_chunksÚ
from_startÚchunk_startÚ	chunk_endÚ	to_chunksÚattention_output_chunksÚattention_probs_chunksÚfrom_endÚto_startÚto_endÚfrom_tensor_chunkÚto_tensor_chunkÚattention_mask_chunkÚcls_attention_maskÚcls_positionÚattention_outputs_chunkr5  s                               r=   rÔ   zCanineAttention.forwardD  sú  € ð �zŠzØŸ9™9 ]°MÀ>ÐS\Ð^oÓpˆLØ+¨A™Òà.;×.AÑ.AÀ!Ñ.DÐDˆO˜mØ&3Ð3ˆK˜)ð ˆKØ×1Ò1Ø×"Ñ" 6Ô*ð ‘
à�
Ü$ Z°À$×B_ÑB_Ó`ò =�Ü °¸t×?[Ñ?[Ñ1[Ó\�	Ø×"Ñ" K°Ð#;Õ<ð=ð
 ˆIØ×1Ò1Ø× Ñ  ! ]Ð!3Ô4Ü$ Q¨°t×7RÑ7RÓSò ;�Ü ¨{¸T×=WÑ=WÑ/WÓX�	Ø× Ñ  +¨yÐ!9Õ:ð;ô �;Ó¤3 y£>Ò1Ü ØEÀkÀ]ð S$Ø$/ =Ð0AðCóð ð ')Ð#Ø%'Ð"Ü>AÀ+ÈyÓ>Yò NÑ:Ñ&�˜XÑ(:¨°6Ø$/²°:¸hÐ3FÊÐ0IÑ$JÐ!Ø"+ªA¨x¸¨ÂÐ,AÑ"B�ð (6²a¸ÀHÐ9LÈhÐW]ÈoÐ6]Ñ'^Ð$Ø×7Ò7Ø)7º¸:ÀhÐ;NÐPQÐRSÐPSÐ8SÑ)TÐ&Ü+0¯9©9Ð6HÐJ^Ð5_ÐefÔ+gÐ(à#,ªQ°°!°²Q¨YÑ#7�LÜ&+§i¡i°¸Ð0OÐUVÔ&W�Oà*.¯)©)Ø% Ð8LÈiÐYjó+Ð'ð (×.Ñ.Ð/FÀqÑ/IÔJÚ$Ø*×1Ñ1Ð2IÈ!Ñ2LÕMð%Nô(  %Ÿy™yÐ)@ÀaÔHÐàŸ;™;Ð'7¸ÓGÐØ#Ð%ˆØ�zŠzØ ¨Q¨RÐ 0Ñ0ˆGð ˆð ¤Ð&<Ó =Ñ=ˆGØˆr<   ©FFFé€   rl  rl  rl  r6  )r4   r5   r6   r7   r7  r{   rŸ   rT  r   r8   r9   r   rÔ   rÖ   r×   s   @r=   rC  rC     sì   ø„ ñð& Ø05Ø.3Ø'*Ø(+Ø%(Ø&)ñ=ð *.ð	=ð
 (,ð=ð "%ð=ð #&ð=ð  #ð=ð !$õ=òB;ð* 7;Ø15Ø,1ñHà˜U×.Ñ.Ñ/ðHð ! ×!2Ñ!2Ñ3ðHð ˜E×-Ñ-Ñ.ð	Hð
 $ D™>ðHð 
ˆu× Ñ  (¨5×+<Ñ+<Ñ"=Ð=Ñ	>÷Hr<   rC  c                   óV   ‡ — e Zd Zˆ fd„Zdej
                  dej
                  fd„Zˆ xZS )ÚCanineIntermediatec                 ó  •— t         ‰| �  «        t        j                  |j                  |j
                  «      | _        t        |j                  t        «      rt        |j                     | _        y |j                  | _        y rû   )rž   rŸ   r   r  r    Úintermediate_sizer<  Ú
isinstancerâ   Ústrr   Úintermediate_act_fnrä   s     €r=   rŸ   zCanineIntermediate.__init__�  s]   ø€ Ü‰ÑÔÜ—Y‘Y˜v×1Ñ1°6×3KÑ3KÓLˆŒ
Ü�f×'Ñ'¬Ô-Ü'-¨f×.?Ñ.?Ñ'@ˆDÕ$à'-×'8Ñ'8ˆDÕ$r<   r2   rÉ   c                 óJ   — | j                  |«      }| j                  |«      }|S rû   )r<  rs  ©r¯   r2   s     r=   rÔ   zCanineIntermediate.forward˜  s&   € ØŸ
™
 =Ó1ˆØ×0Ñ0°Ó?ˆØÐr<   )r4   r5   r6   rŸ   r8   r9   rÔ   rÖ   r×   s   @r=   rn  rn  �  s'   ø„ ô9ð U×%6Ñ%6ð ¸5×;LÑ;L÷ r<   rn  c                   ót   ‡ — e Zd Zˆ fd„Zdeej                     dej                  dej                  fd„Zˆ xZS )ÚCanineOutputc                 ó(  •— t         ‰| �  «        t        j                  |j                  |j
                  «      | _        t        j                  |j
                  |j                  ¬«      | _        t        j                  |j                  «      | _        y r;  )rž   rŸ   r   r  rp  r    r<  rU   r§   r¨   r©   rª   rä   s     €r=   rŸ   zCanineOutput.__init__Ÿ  s`   ø€ Ü‰ÑÔÜ—Y‘Y˜v×7Ñ7¸×9KÑ9KÓLˆŒ
ÜŸ™ f×&8Ñ&8¸f×>SÑ>SÔTˆŒÜ—z‘z &×"<Ñ"<Ó=ˆ�r<   r2   r=  rÉ   c                 ór   — | j                  |«      }| j                  |«      }| j                  ||z   «      }|S rû   r?  r@  s      r=   rÔ   zCanineOutput.forward¥  s7   € ØŸ
™
 =Ó1ˆØŸ™ ]Ó3ˆØŸ™ }°|Ñ'CÓDˆØÐr<   rA  r×   s   @r=   rw  rw  ž  s:   ø„ ô>ð U¨5×+<Ñ+<Ñ%=ð ÈU×M^ÑM^ð Ðch×ctÑct÷ r<   rw  c                   óÔ   ‡ — e Zd Zˆ fd„Z	 	 	 d	deej                     deej                     deej                     dee   deej                  eej                     f   f
d„Z	d„ Z
ˆ xZS )
ÚCanineLayerc	           
      óÀ   •— t         ‰	| �  «        |j                  | _        d| _        t	        ||||||||«      | _        t        |«      | _        t        |«      | _	        y ©Nr   )
rž   rŸ   Úchunk_size_feed_forwardÚseq_len_dimrC  Ú	attentionrn  Úintermediaterw  rK  rO  s
            €r=   rŸ   zCanineLayer.__init__­  se   ø€ ô 	‰ÑÔØ'-×'EÑ'EˆÔ$ØˆÔÜ(ØØØ+Ø)Ø#Ø$Ø!Ø"ó	
ˆŒô /¨vÓ6ˆÔÜ" 6Ó*ˆ�r<   r2   r  r  r  rÉ   c                 ó¨   — | j                  ||||¬«      }|d   }|dd  }t        | j                  | j                  | j                  |«      }|f|z   }|S )N)r  r   r   )r€  r   Úfeed_forward_chunkr~  r  )	r¯   r2   r  r  r  Úself_attention_outputsrX  r5  Úlayer_outputs	            r=   rÔ   zCanineLayer.forwardÈ  sv   € ð "&§¡ØØØØ/ð	 "0ó "
Ðð 2°!Ñ4Ðà(¨¨Ð,ˆä0Ø×#Ñ# T×%AÑ%AÀ4×CSÑCSÐUeó
ˆð  �/ GÑ+ˆàˆr<   c                 óL   — | j                  |«      }| j                  ||«      }|S rû   )r�  rK  )r¯   rX  Úintermediate_outputr…  s       r=   rƒ  zCanineLayer.feed_forward_chunkà  s,   € Ø"×/Ñ/Ð0@ÓAÐØ—{‘{Ð#6Ð8HÓIˆØÐr<   r6  )r4   r5   r6   rŸ   r   r8   r9   r   r7  rÔ   rƒ  rÖ   r×   s   @r=   r{  r{  ¬  s�   ø„ ô+ð< 7;Ø15Ø,1ñà˜U×.Ñ.Ñ/ðð ! ×!2Ñ!2Ñ3ðð ˜E×-Ñ-Ñ.ð	ð
 $ D™>ðð 
ˆu× Ñ  (¨5×+<Ñ+<Ñ"=Ð=Ñ	>óö0r<   r{  c                   óÈ   ‡ — e Zd Z	 	 	 	 	 	 	 d
ˆ fd„	Z	 	 	 	 	 ddeej                     deej                     deej                     dee   dee   dee   de	ee
f   fd	„Zˆ xZS )ÚCanineEncoderc	                 óÞ   •— t         ‰
| �  «        || _        t        j                  t        |j                  «      D �	cg c]  }	t        ||||||||«      ‘Œ c}	«      | _        d| _	        y c c}	w )NF)
rž   rŸ   rƒ   r   Ú
ModuleListr|   Únum_hidden_layersr{  ÚlayerÚgradient_checkpointing)r¯   rƒ   rN  rD  rE  rF  rG  rH  rI  Ú_r±   s             €r=   rŸ   zCanineEncoder.__init__ç  sx   ø€ ô 	‰ÑÔØˆŒÜ—]‘]ô ˜v×7Ñ7Ó8öð ô ØØØ3Ø1Ø+Ø,Ø)Ø*õ	òó
ˆŒ
ð ',ˆÕ#ùòs   ½A*r2   r  r  r  Úoutput_hidden_statesÚreturn_dictrÉ   c                 óx  — |rdnd }|rdnd }t        | j                  «      D ]j  \  }	}
|r||fz   }|�||	   nd }| j                  r,| j                  r | j	                  |
j
                  ||||«      }n |
||||«      }|d   }|sŒb||d   fz   }Œl |r||fz   }|st        d„ |||fD «       «      S t        |||¬«      S )Nr;   r   r   c              3   ó&   K  — | ]	  }|€Œ|–— Œ y ­wrû   r;   ©rI   Úvs     r=   rK   z(CanineEncoder.forward.<locals>.<genexpr>*  s   è ø€ Òm˜qÐ_`Ñ_lœÑmùó   ‚Š)r0   r2   r3   )rÀ   r�  rŽ  ÚtrainingÚ_gradient_checkpointing_funcÚ__call__rV  r   )r¯   r2   r  r  r  r�  r‘  Úall_hidden_statesÚall_self_attentionsr‘   Úlayer_moduleÚlayer_head_maskÚlayer_outputss                r=   rÔ   zCanineEncoder.forward  s  € ñ #7™B¸DÐÙ$5™b¸4Ðä(¨¯©Ó4ò 	P‰OˆAˆ|Ù#Ø$5¸Ð8HÑ$HÐ!à.7Ð.C˜i¨šlÈˆOà×*Ò*¨t¯}ª}Ø $× AÑ AØ ×)Ñ)Ø!Ø"Ø#Ø%ó!‘ñ !-¨]¸NÈOÐ]nÓ o�à)¨!Ñ,ˆMÚ Ø&9¸]È1Ñ=MÐ<OÑ&OÑ#ð'	Pñ*  Ø 1°]Ð4DÑ DÐáÜÑm ]Ð4EÐGZÐ$[ÔmÓmÐmÜØ+Ø+Ø*ô
ð 	
r<   rk  )NNFFT)r4   r5   r6   rŸ   r   r8   r9   r   r7  r   r   rÔ   rÖ   r×   s   @r=   r‰  r‰  æ  s±   ø„ ð Ø(-Ø&+Ø #Ø!$Ø!Ø"õ,ðB 7;Ø15Ø,1Ø/4Ø&*ñ*
à˜U×.Ñ.Ñ/ð*
ð ! ×!2Ñ!2Ñ3ð*
ð ˜E×-Ñ-Ñ.ð	*
ð
 $ D™>ð*
ð ' t™nð*
ð ˜d‘^ð*
ð 
ˆu�oÐ%Ñ	&÷*
r<   r‰  c                   ó\   ‡ — e Zd Zˆ fd„Zdeej                     dej                  fd„Zˆ xZS )ÚCaninePoolerc                 ó²   •— t         ‰| �  «        t        j                  |j                  |j                  «      | _        t        j                  «       | _        y rû   )rž   rŸ   r   r  r    r<  ÚTanhrã   rä   s     €r=   rŸ   zCaninePooler.__init__3  s9   ø€ Ü‰ÑÔÜ—Y‘Y˜v×1Ñ1°6×3EÑ3EÓFˆŒ
ÜŸ'™'›)ˆ�r<   r2   rÉ   c                 ó\   — |d d …df   }| j                  |«      }| j                  |«      }|S )Nr   )r<  rã   )r¯   r2   Úfirst_token_tensorÚpooled_outputs       r=   rÔ   zCaninePooler.forward8  s6   € ð +ª1¨a¨4Ñ0ÐØŸ
™
Ð#5Ó6ˆØŸ™¨Ó6ˆØÐr<   rA  r×   s   @r=   r   r   2  s,   ø„ ô$ð
 U¨5×+<Ñ+<Ñ%=ð À%×BSÑBS÷ r<   r   c                   ó\   ‡ — e Zd Zˆ fd„Zdeej                     dej                  fd„Zˆ xZS )ÚCaninePredictionHeadTransformc                 óh  •— t         ‰| �  «        t        j                  |j                  |j                  «      | _        t        |j                  t        «      rt        |j                     | _
        n|j                  | _
        t        j                  |j                  |j                  ¬«      | _        y r;  )rž   rŸ   r   r  r    r<  rq  râ   rr  r   Útransform_act_fnrU   r§   rä   s     €r=   rŸ   z&CaninePredictionHeadTransform.__init__B  s{   ø€ Ü‰ÑÔÜ—Y‘Y˜v×1Ñ1°6×3EÑ3EÓFˆŒ
Ü�f×'Ñ'¬Ô-Ü$*¨6×+<Ñ+<Ñ$=ˆDÕ!à$*×$5Ñ$5ˆDÔ!ÜŸ™ f×&8Ñ&8¸f×>SÑ>SÔTˆ�r<   r2   rÉ   c                 ól   — | j                  |«      }| j                  |«      }| j                  |«      }|S rû   )r<  r©  rU   ru  s     r=   rÔ   z%CaninePredictionHeadTransform.forwardK  s4   € ØŸ
™
 =Ó1ˆØ×-Ñ-¨mÓ<ˆØŸ™ }Ó5ˆØÐr<   rA  r×   s   @r=   r§  r§  A  s-   ø„ ôUð U¨5×+<Ñ+<Ñ%=ð À%×BSÑBS÷ r<   r§  c                   ó\   ‡ — e Zd Zˆ fd„Zdeej                     dej                  fd„Zˆ xZS )ÚCanineLMPredictionHeadc                 óH  •— t         ‰| �  «        t        |«      | _        t	        j
                  |j                  |j                  d¬«      | _        t	        j                  t        j                  |j                  «      «      | _        | j                  | j                  _        y )NF)r^   )rž   rŸ   r§  Ú	transformr   r  r    Ú
vocab_sizeÚdecoderÚ	Parameterr8   rÏ   r^   rä   s     €r=   rŸ   zCanineLMPredictionHead.__init__S  sm   ø€ Ü‰ÑÔÜ6°vÓ>ˆŒô —y‘y ×!3Ñ!3°V×5FÑ5FÈUÔSˆŒä—L‘L¤§¡¨V×->Ñ->Ó!?Ó@ˆŒ	ð !ŸI™Iˆ�‰Õr<   r2   rÉ   c                 óJ   — | j                  |«      }| j                  |«      }|S rû   )r®  r°  ru  s     r=   rÔ   zCanineLMPredictionHead.forward`  s$   € ØŸ™ }Ó5ˆØŸ™ ]Ó3ˆØÐr<   rA  r×   s   @r=   r¬  r¬  R  s,   ø„ ô&ð U¨5×+<Ñ+<Ñ%=ð À%×BSÑBS÷ r<   r¬  c                   ób   ‡ — e Zd Zˆ fd„Zdeej                     deej                     fd„Zˆ xZS )ÚCanineOnlyMLMHeadc                 óB   •— t         ‰| �  «        t        |«      | _        y rû   )rž   rŸ   r¬  Úpredictionsrä   s     €r=   rŸ   zCanineOnlyMLMHead.__init__g  s   ø€ Ü‰ÑÔÜ1°&Ó9ˆÕr<   Úsequence_outputrÉ   c                 ó(   — | j                  |«      }|S rû   )r¶  )r¯   r·  Úprediction_scoress      r=   rÔ   zCanineOnlyMLMHead.forwardk  s   € ð !×,Ñ,¨_Ó=ÐØ Ð r<   )	r4   r5   r6   rŸ   r   r8   rë   rÔ   rÖ   r×   s   @r=   r´  r´  f  s1   ø„ ô:ð!à˜uŸ|™|Ñ,ð!ð 
ˆu�|‰|Ñ	÷!r<   r´  c                   ó&   — e Zd ZdZeZeZdZdZ	d„ Z
y)ÚCaninePreTrainedModelz†
    An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained
    models.
    ÚcanineTc                 ó"  — t        |t        j                  t        j                  f«      rm|j                  j
                  j                  d| j                  j                  ¬«       |j                  �%|j                  j
                  j                  «        yyt        |t        j                  «      rz|j                  j
                  j                  d| j                  j                  ¬«       |j                  �2|j                  j
                  |j                     j                  «        yyt        |t        j                  «      rJ|j                  j
                  j                  «        |j                  j
                  j                  d«       yy)zInitialize the weightsg        )ÚmeanÚstdNr  )rq  r   r  rà   r[   r�   Únormal_rƒ   Úinitializer_ranger^   Úzero_r£   Úpadding_idxrU   Úfill_)r¯   Úmodules     r=   Ú_init_weightsz#CaninePreTrainedModel._init_weights~  s  € ä�fœrŸy™y¬"¯)©)Ð4Ô5ð �M‰M×Ñ×&Ñ&¨C°T·[±[×5RÑ5RÐ&ÔSØ�{‰{Ð&Ø—‘× Ñ ×&Ñ&Õ(ð 'ä˜¤§¡Ô-Ø�M‰M×Ñ×&Ñ&¨C°T·[±[×5RÑ5RÐ&ÔSØ×!Ñ!Ð-Ø—‘×"Ñ" 6×#5Ñ#5Ñ6×<Ñ<Õ>ð .ä˜¤§¡Ô-Ø�K‰K×Ñ×"Ñ"Ô$Ø�M‰M×Ñ×$Ñ$ SÕ)ð .r<   N)r4   r5   r6   r7   r   Úconfig_classr’   Úload_tf_weightsÚbase_model_prefixÚsupports_gradient_checkpointingrÆ  r;   r<   r=   r»  r»  s  s$   „ ñð
  €LØ/€OØ ÐØ&*Ð#ó*r<   r»  aI  
    This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class. Use
    it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and
    behavior.

    Parameters:
        config ([`CanineConfig`]): Model configuration class with all the parameters of the model.
            Initializing with a config file does not load the weights associated with the model, only the
            configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
a5
  
    Args:
        input_ids (`torch.LongTensor` of shape `({0})`):
            Indices of input sequence tokens in the vocabulary.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are input IDs?](../glossary#input-ids)
        attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*):
            Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:

            - 1 for tokens that are **not masked**,
            - 0 for tokens that are **masked**.

            [What are attention masks?](../glossary#attention-mask)
        token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*):
            Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
            1]`:

            - 0 corresponds to a *sentence A* token,
            - 1 corresponds to a *sentence B* token.

            [What are token type IDs?](../glossary#token-type-ids)
        position_ids (`torch.LongTensor` of shape `({0})`, *optional*):
            Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
            config.max_position_embeddings - 1]`.

            [What are position IDs?](../glossary#position-ids)
        head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*):
            Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`:

            - 1 indicates the head is **not masked**,
            - 0 indicates the head is **masked**.

        inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*):
            Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
            is useful if you want more control over how to convert *input_ids* indices into associated vectors than the
            model's internal embedding lookup matrix.
        output_attentions (`bool`, *optional*):
            Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
            tensors for more detail.
        output_hidden_states (`bool`, *optional*):
            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
            more detail.
        return_dict (`bool`, *optional*):
            Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
z`The bare CANINE Model transformer outputting raw hidden-states without any specific head on top.c                   óØ  ‡ — e Zd Zdˆ fd„	Zd„ Zd„ Zdej                  defd„Z	dej                  ded	ej                  fd
„Z
 eej                  d«      «       eeee¬«      	 	 	 	 	 	 	 	 	 ddeej&                     deej(                     deej&                     deej&                     deej(                     deej(                     dee   dee   dee   d	eeef   fd„«       «       Zˆ xZS )ÚCanineModelc           
      óÒ  •— t         ‰| �  |«       || _        t        j                  |«      }d|_        t        |«      | _        t        |ddd|j                  |j                  |j                  |j                  ¬«      | _
        t        |«      | _        t        |«      | _        t        |«      | _        t        |«      | _        |rt#        |«      nd | _        | j'                  «        y )Nr   TF)rN  rD  rE  rF  rG  rH  rI  )rž   rŸ   rƒ   ÚcopyÚdeepcopyrŒ  r”   Úchar_embeddingsr‰  Úlocal_transformer_striderQ   rÙ   rR   rM   rí   rW   rT   r   ÚpoolerÚ	post_init)r¯   rƒ   Úadd_pooling_layerÚshallow_configr±   s       €r=   rŸ   zCanineModel.__init__Ñ  sÇ   ø€ Ü‰Ñ˜Ô ØˆŒÜŸ™ vÓ.ˆØ+,ˆÔ(ä/°Ó7ˆÔä$1ØØØ,1Ø*/Ø$*×$CÑ$CØ%+×%DÑ%DØ"(×"AÑ"AØ#)×#BÑ#Bô	%
ˆÔ!ô #8¸Ó"?ˆÔä$ VÓ,ˆŒÜ(¨Ó0ˆŒä"/°Ó"?ˆÔá.?”l 6Ô*ÀTˆŒð 	�‰Õr<   c                 ó˜   — |j                  «       D ]7  \  }}| j                  j                  |   j                  j	                  |«       Œ9 y)z�
        Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base
        class PreTrainedModel
        N)ÚitemsrM   r�  r€  rT  )r¯   Úheads_to_pruner�  rR  s       r=   Ú_prune_headszCanineModel._prune_headsï  sE   € ð
 +×0Ñ0Ó2ò 	C‰LˆE�5Ø�L‰L×Ñ˜uÑ%×/Ñ/×;Ñ;¸EÕBñ	Cr<   c                 ó  — |j                   d   |j                   d   }}|j                   d   }t        j                  ||d|f«      j                  «       }t        j                  ||dft        j
                  |j                  ¬«      }||z  }|S )aP  
        Create 3D attention mask from a 2D tensor mask.

        Args:
            from_tensor: 2D or 3D Tensor of shape [batch_size, from_seq_length, ...].
            to_mask: int32 Tensor of shape [batch_size, to_seq_length].

        Returns:
            float Tensor of shape [batch_size, from_seq_length, to_seq_length].
        r   r   )rÎ   rÌ   rÍ   )r~   r8   Úreshaper   ÚonesÚfloat32rÍ   )r¯   r  Úto_maskÚ
batch_sizerY  rZ  Úbroadcast_onesÚmasks           r=   Ú)_create_3d_attention_mask_from_input_maskz5CanineModel._create_3d_attention_mask_from_input_mask÷  s‹   € ð '2×&7Ñ&7¸Ñ&:¸K×<MÑ<MÈaÑ<P�Oˆ
àŸ™ aÑ(ˆä—-‘- ¨*°a¸Ð)GÓH×NÑNÓPˆô
 Ÿ™¨*°oÀqÐ)IÔQV×Q^ÑQ^Ðgn×guÑguÔvˆð  Ñ'ˆàˆr<   Úchar_attention_maskrá   c                 óì   — |j                   \  }}t        j                  ||d|f«      }t        j                  j	                  ||¬«      |j                  «       «      }t        j                  |d¬«      }|S )z[Downsample 2D character attention mask to 2D molecule attention mask using MaxPool1d layer.r   )rÞ   rß   rš   r¾   )r~   r8   rÛ  r   Ú	MaxPool1dr   Úsqueeze)r¯   rã  rá   rß  Úchar_seq_lenÚpoolable_char_maskÚpooled_molecule_maskÚmolecule_attention_masks           r=   Ú_downsample_attention_maskz&CanineModel._downsample_attention_mask  sw   € ð $7×#<Ñ#<Ñ ˆ
�LÜ"Ÿ]™]Ð+>ÀÈQÐP\Ð@]Ó^Ðô  %Ÿx™x×1Ñ1Ð>OÐXiÐ1ÓjØ×$Ñ$Ó&ó 
Ðô
 #(§-¡-Ð0DÈ"Ô"MÐà&Ð&r<   Ú	moleculesÚchar_seq_lengthrÉ   c                 ó  — | j                   j                  }|dd…dd…dd…f   }t        j                  ||d¬«      }|dd…dd…dd…f   }||z  }t        j                  |||z   d¬«      }t        j                  ||gd¬«      S )zDRepeats molecules to make them the same length as the char sequence.Nr   rS   )Úrepeatsr¿   rš   r¾   )rƒ   rá   r8   Úrepeat_interleaverÁ   )	r¯   rì  rí  ÚrateÚmolecules_without_extra_clsÚrepeatedÚlast_moleculeÚremainder_lengthÚremainder_repeateds	            r=   Ú_repeat_moleculeszCanineModel._repeat_molecules#  s”   € ð �{‰{×,Ñ,ˆà&/²°1±2²q°Ñ&9Ð#ä×*Ñ*Ð+FÐPTÐZ\Ô]ˆð "¢! R¡Sª! )Ñ,ˆØ*¨TÑ1ÐÜ"×4Ñ4Øà$ tÑ+Øô	
Ðô �y‰y˜(Ð$6Ð7¸RÔ@Ð@r<   úbatch_size, sequence_length©Ú
checkpointÚoutput_typerÇ  r¶   r  rÇ   r™   r  rÈ   r  r�  r‘  c
                 óB  — |�|n| j                   j                  }|�|n| j                   j                  }|rdnd }
|rdnd }|	�|	n| j                   j                  }	|�|�t	        d«      ‚|�#| j                  ||«       |j                  «       }n!|�|j                  «       d d }nt	        d«      ‚|\  }}|�|j                  n|j                  }|€t        j                  ||f|¬«      }|€&t        j                  |t        j                  |¬«      }| j                  ||«      }| j                  || j                   j                  ¬«      }| j                  |||j                  d   f«      }| j!                  || j                   j"                  «      }| j%                  ||||¬«      }| j'                  |�|n||«      }| j)                  ||||¬	«      }|j*                  }| j-                  |«      }| j/                  ||||||	¬
«      }|d   }| j0                  �| j1                  |«      nd }| j3                  ||d   ¬«      }t        j4                  ||gd¬«      }| j7                  |«      }| j9                  ||||¬	«      }|j*                  }|r2|	r|j:                  n|d   }|
|j:                  z   |z   |j:                  z   }
|r2|	r|j<                  n|d   } ||j<                  z   | z   |j<                  z   }|	s||f}!|!t?        d„ |
|fD «       «      z  }!|!S tA        |||
|¬«      S )Nr;   zDYou cannot specify both input_ids and inputs_embeds at the same timerš   z5You have to specify either input_ids or inputs_embeds)rÍ   rË   )rá   )r¶   r™   rÇ   rÈ   )r  r  r�  )r  r  r  r�  r‘  r   )rí  r¾   r   c              3   ó&   K  — | ]	  }|€Œ|–— Œ y ­wrû   r;   r”  s     r=   rK   z&CanineModel.forward.<locals>.<genexpr>Ö  s   è ø€ Òa !ÐSTÑS`œAÑaùr–  )r0   r1   r2   r3   )!rƒ   r  r�  Úuse_return_dictr   Ú%warn_if_padding_and_no_attention_maskrÎ   rÍ   r8   rÜ  rÏ   rÐ   Úget_extended_attention_maskrë  rá   r~   Úget_head_maskrŒ  rÐ  râ  rQ   r0   rR   rM   rÒ  r÷  rÁ   rW   rT   r2   r3   rV  r/   )"r¯   r¶   r  rÇ   r™   r  rÈ   r  r�  r‘  rš  r›  rÑ   rß  rÒ   rÍ   Úextended_attention_maskrê  Ú extended_molecule_attention_maskÚinput_char_embeddingsrã  Úinit_chars_encoder_outputsÚinput_char_encodingÚinit_molecule_encodingÚencoder_outputsÚmolecule_sequence_outputr¥  Úrepeated_moleculesÚconcatr·  Úfinal_chars_encoder_outputsÚdeep_encoder_hidden_statesÚdeep_encoder_self_attentionsrK  s"                                     r=   rÔ   zCanineModel.forward<  sÑ  € ð$ 2CÐ1NÑ-ÐTX×T_ÑT_×TqÑTqÐà$8Ð$DÑ È$Ï+É+×JjÑJjð 	ñ #7™B¸DÐÙ$5™b¸4ÐØ%0Ð%<‘kÀ$Ç+Á+×B]ÑB]ˆàÐ  ]Ð%>ÜÐcÓdÐdØÐ"Ø×6Ñ6°yÀ.ÔQØ#Ÿ.™.Ó*‰KØÐ&Ø'×,Ñ,Ó.¨s°Ð3‰KäÐTÓUÐUà!,Ñˆ
�JØ%.Ð%:�×!Ò!À×@TÑ@TˆàÐ!Ü"ŸZ™Z¨*°jÐ)AÈ6ÔRˆNØÐ!Ü"Ÿ[™[¨¼E¿J¹JÈvÔVˆNð 15×0PÑ0PÐQ_ÐalÓ0mÐØ"&×"AÑ"AØ¨d¯k©k×.KÑ.Kð #Bó #
Ðð :>×9YÑ9YØ# jÐ2I×2OÑ2OÐPRÑ2SÐ%Tó:
Ð(ð ×&Ñ& y°$·+±+×2OÑ2OÓPˆ	ð !%× 4Ñ 4ØØ%Ø)Ø'ð	 !5ó !
Ðð #×LÑLØ"Ð.‰I°MÀ>ó
Ðð &*×%>Ñ%>Ø!Ø.Ø/Ø!5ð	 &?ó &
Ð"ð 9×JÑJÐð  "&×!8Ñ!8Ð9LÓ!MÐð Ÿ,™,Ø"Ø;ØØ/Ø!5Ø#ð 'ó 
ˆð $3°1Ñ#5Ð ØAEÇÁÐAX˜Ÿ™Ð$<Ô=Ð^bˆð "×3Ñ3Ð4LÐ^iÐjlÑ^mÐ3ÓnÐô —‘Ð/Ð1CÐDÈ"ÔMˆð Ÿ/™/¨&Ó1ˆð '+×&=Ñ&=ØØ2Ø/Ø!5ð	 '>ó '
Ð#ð 6×GÑGˆáÙJU¨×)FÒ)FÐ[jÐklÑ[mÐ&à!Ø,×:Ñ:ñ;à,ñ-ð .×;Ñ;ñ<ð ñ ÙIT¨?×+EÒ+EÐZiÐjlÑZmÐ(à#Ø,×7Ñ7ñ8à.ñ/ð .×8Ñ8ñ9ð  ñ Ø% }Ð5ˆFØ”eÑaÐ(9Ð;NÐ'OÔaÓaÑaˆFØˆMä+Ø-Ø'Ø+Ø*ô	
ð 	
r<   )T)	NNNNNNNNN)r4   r5   r6   rŸ   rÙ  râ  r8   rë   r{   rë  r÷  r   ÚCANINE_INPUTS_DOCSTRINGÚformatr   Ú_CHECKPOINT_FOR_DOCr/   Ú_CONFIG_FOR_DOCr   rÕ   r9   r7  r   r   rÔ   rÖ   r×   s   @r=   rÌ  rÌ  Ì  su  ø„ õ
ò<Còð6'¸e¿l¹lð 'Ð_bó 'ð"A¨5¯<©<ð AÈ#ð AÐRW×R^ÑR^ó Añ2 +Ð+B×+IÑ+IÐJgÓ+hÓiÙØ&Ø0Ø$ôð 15Ø6:Ø59Ø37Ø15Ø59Ø,0Ø/3Ø&*ñ\
à˜E×,Ñ,Ñ-ð\
ð ! ×!2Ñ!2Ñ3ð\
ð ! ×!1Ñ!1Ñ2ð	\
ð
 ˜u×/Ñ/Ñ0ð\
ð ˜E×-Ñ-Ñ.ð\
ð   × 1Ñ 1Ñ2ð\
ð $ D™>ð\
ð ' t™nð\
ð ˜d‘^ð\
ð 
ˆuÐ2Ð2Ñ	3ò\
óó jô\
r<   rÌ  zž
    CANINE Model transformer with a sequence classification/regression head on top (a linear layer on top of the pooled
    output) e.g. for GLUE tasks.
    c                   óŠ  ‡ — e Zd Zˆ fd„Z eej                  d«      «       eee	e
¬«      	 	 	 	 	 	 	 	 	 	 ddeej                     deej                     deej                     deej                     deej                     d	eej                     d
eej                     dee   dee   dee   deee	f   fd„«       «       Zˆ xZS )ÚCanineForSequenceClassificationc                 ó,  •— t         ‰| �  |«       |j                  | _        t        |«      | _        t        j                  |j                  «      | _        t        j                  |j                  |j                  «      | _        | j                  «        y rû   ©rž   rŸ   Ú
num_labelsrÌ  r¼  r   r¨   r©   rª   r  r    Ú
classifierrÓ  rä   s     €r=   rŸ   z(CanineForSequenceClassification.__init__é  ói   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒä! &Ó)ˆŒÜ—z‘z &×"<Ñ"<Ó=ˆŒÜŸ)™) F×$6Ñ$6¸×8IÑ8IÓJˆŒð 	�‰Õr<   rø  rù  r¶   r  rÇ   r™   r  rÈ   Úlabelsr  r�  r‘  rÉ   c                 ó@  — |
�|
n| j                   j                  }
| j                  ||||||||	|
¬«	      }|d   }| j                  |«      }| j	                  |«      }d}|��‡| j                   j
                  €�| j                  dk(  rd| j                   _        nl| j                  dkD  rL|j                  t        j                  k(  s|j                  t        j                  k(  rd| j                   _        nd| j                   _        | j                   j
                  dk(  rIt        «       }| j                  dk(  r& ||j                  «       |j                  «       «      }nŒ |||«      }n‚| j                   j
                  dk(  r=t        «       } ||j                  d| j                  «      |j                  d«      «      }n,| j                   j
                  dk(  rt        «       } |||«      }|
s|f|dd z   }|�|f|z   S |S t!        |||j"                  |j$                  ¬	«      S )
a�  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        N©r  rÇ   r™   r  rÈ   r  r�  r‘  r   Ú
regressionÚsingle_label_classificationÚmulti_label_classificationrš   r`   ©ÚlossÚlogitsr2   r3   )rƒ   rþ  r¼  rª   r  Úproblem_typer  rÌ   r8   rÐ   r{   r
   ræ  r	   r  r   r   r2   r3   )r¯   r¶   r  rÇ   r™   r  rÈ   r  r  r�  r‘  r5  r¥  r"  r!  Úloss_fctrK  s                    r=   rÔ   z'CanineForSequenceClassification.forwardô  sè  € ð2 &1Ð%<‘kÀ$Ç+Á+×B]ÑB]ˆà—+‘+ØØ)Ø)Ø%ØØ'Ø/Ø!5Ø#ð ó 

ˆð   ™
ˆàŸ™ ]Ó3ˆØ—‘ Ó/ˆàˆØÑØ�{‰{×'Ñ'Ð/Ø—?‘? aÒ'Ø/;�D—K‘KÕ,Ø—_‘_ qÒ(¨f¯l©l¼e¿j¹jÒ.HÈFÏLÉLÔ\a×\eÑ\eÒLeØ/L�D—K‘KÕ,à/K�D—K‘KÔ,à�{‰{×'Ñ'¨<Ò7Ü"›9�Ø—?‘? aÒ'Ù# F§N¡NÓ$4°f·n±nÓ6FÓG‘Dá# F¨FÓ3‘DØ—‘×)Ñ)Ð-JÒJÜ+Ó-�Ù §¡¨B°·±Ó @À&Ç+Á+ÈbÃ/ÓR‘Ø—‘×)Ñ)Ð-IÒIÜ,Ó.�Ù ¨Ó/�ÙØ�Y ¨¨ Ñ,ˆFØ)-Ð)9�T�G˜fÑ$ÐE¸vÐEä'ØØØ!×/Ñ/Ø×)Ñ)ô	
ð 	
r<   ©
NNNNNNNNNN)r4   r5   r6   rŸ   r   r  r  r   r  r   r  r   r8   rÕ   r9   r7  r   r   rÔ   rÖ   r×   s   @r=   r  r  á  sB  ø„ ô	ñ +Ð+B×+IÑ+IÐJgÓ+hÓiÙØ&Ø,Ø$ôð 15Ø6:Ø59Ø37Ø15Ø59Ø-1Ø,0Ø/3Ø&*ñE
à˜E×,Ñ,Ñ-ðE
ð ! ×!2Ñ!2Ñ3ðE
ð ! ×!1Ñ!1Ñ2ð	E
ð
 ˜u×/Ñ/Ñ0ðE
ð ˜E×-Ñ-Ñ.ðE
ð   × 1Ñ 1Ñ2ðE
ð ˜×)Ñ)Ñ*ðE
ð $ D™>ðE
ð ' t™nðE
ð ˜d‘^ðE
ð 
ˆuÐ.Ð.Ñ	/òE
óó jôE
r<   r  z§
    CANINE Model with a multiple choice classification head on top (a linear layer on top of the pooled output and a
    softmax) e.g. for RocStories/SWAG tasks.
    c                   óŠ  ‡ — e Zd Zˆ fd„Z eej                  d«      «       eee	e
¬«      	 	 	 	 	 	 	 	 	 	 ddeej                     deej                     deej                     deej                     deej                     d	eej                     d
eej                     dee   dee   dee   deee	f   fd„«       «       Zˆ xZS )ÚCanineForMultipleChoicec                 óö   •— t         ‰| �  |«       t        |«      | _        t	        j
                  |j                  «      | _        t	        j                  |j                  d«      | _
        | j                  «        y r}  )rž   rŸ   rÌ  r¼  r   r¨   r©   rª   r  r    r  rÓ  rä   s     €r=   rŸ   z CanineForMultipleChoice.__init__J  sV   ø€ Ü‰Ñ˜Ô ä! &Ó)ˆŒÜ—z‘z &×"<Ñ"<Ó=ˆŒÜŸ)™) F×$6Ñ$6¸Ó:ˆŒð 	�‰Õr<   z(batch_size, num_choices, sequence_lengthrù  r¶   r  rÇ   r™   r  rÈ   r  r  r�  r‘  rÉ   c                 óL  — |
�|
n| j                   j                  }
|�|j                  d   n|j                  d   }|�!|j                  d|j	                  d«      «      nd}|�!|j                  d|j	                  d«      «      nd}|�!|j                  d|j	                  d«      «      nd}|�!|j                  d|j	                  d«      «      nd}|�1|j                  d|j	                  d«      |j	                  d«      «      nd}| j                  ||||||||	|
¬«	      }|d   }| j                  |«      }| j                  |«      }|j                  d|«      }d}|�t        «       } |||«      }|
s|f|dd z   }|�|f|z   S |S t        |||j                  |j                  ¬«      S )aJ  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the multiple choice classification loss. Indices should be in `[0, ...,
            num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See
            `input_ids` above)
        Nr   rš   rS   r  r`   r   )rƒ   rþ  r~   r  rÎ   r¼  rª   r  r	   r   r2   r3   )r¯   r¶   r  rÇ   r™   r  rÈ   r  r  r�  r‘  Únum_choicesr5  r¥  r"  Úreshaped_logitsr!  r$  rK  s                      r=   rÔ   zCanineForMultipleChoice.forwardT  sÝ  € ð2 &1Ð%<‘kÀ$Ç+Á+×B]ÑB]ˆØ,5Ð,A�i—o‘o aÒ(À}×GZÑGZÐ[\ÑG]ˆà>GÐ>S�I—N‘N 2 y§~¡~°bÓ'9Ô:ÐY]ˆ	ØM[ÐMg˜×,Ñ,¨R°×1DÑ1DÀRÓ1HÔIÐmqˆØM[ÐMg˜×,Ñ,¨R°×1DÑ1DÀRÓ1HÔIÐmqˆØGSÐG_�|×(Ñ(¨¨\×->Ñ->¸rÓ-BÔCÐeiˆð Ð(ð ×Ñ˜r =×#5Ñ#5°bÓ#9¸=×;MÑ;MÈbÓ;QÔRàð 	ð —+‘+ØØ)Ø)Ø%ØØ'Ø/Ø!5Ø#ð ó 

ˆð   ™
ˆàŸ™ ]Ó3ˆØ—‘ Ó/ˆØ Ÿ+™+ b¨+Ó6ˆàˆØÐÜ'Ó)ˆHÙ˜O¨VÓ4ˆDáØ%Ð'¨'°!°"¨+Ñ5ˆFØ)-Ð)9�T�G˜fÑ$ÐE¸vÐEä(ØØ"Ø!×/Ñ/Ø×)Ñ)ô	
ð 	
r<   r%  )r4   r5   r6   rŸ   r   r  r  r   r  r   r  r   r8   rÕ   r9   r7  r   r   rÔ   rÖ   r×   s   @r=   r'  r'  B  sB  ø„ ôñ +Ð+B×+IÑ+IÐJtÓ+uÓvÙØ&Ø-Ø$ôð 15Ø6:Ø59Ø37Ø15Ø59Ø-1Ø,0Ø/3Ø&*ñ@
à˜E×,Ñ,Ñ-ð@
ð ! ×!2Ñ!2Ñ3ð@
ð ! ×!1Ñ!1Ñ2ð	@
ð
 ˜u×/Ñ/Ñ0ð@
ð ˜E×-Ñ-Ñ.ð@
ð   × 1Ñ 1Ñ2ð@
ð ˜×)Ñ)Ñ*ð@
ð $ D™>ð@
ð ' t™nð@
ð ˜d‘^ð@
ð 
ˆuÐ/Ð/Ñ	0ò@
óó wô@
r<   r'  z¥
    CANINE Model with a token classification head on top (a linear layer on top of the hidden-states output) e.g. for
    Named-Entity-Recognition (NER) tasks.
    c                   óˆ  ‡ — e Zd Zˆ fd„Z eej                  d«      «       eee	¬«      	 	 	 	 	 	 	 	 	 	 dde
ej                     de
ej                     de
ej                     de
ej                     de
ej                     d	e
ej                     d
e
ej                     de
e   de
e   de
e   deeef   fd„«       «       Zˆ xZS )ÚCanineForTokenClassificationc                 ó,  •— t         ‰| �  |«       |j                  | _        t        |«      | _        t        j                  |j                  «      | _        t        j                  |j                  |j                  «      | _        | j                  «        y rû   r  rä   s     €r=   rŸ   z%CanineForTokenClassification.__init__¥  r  r<   rø  )rû  rÇ  r¶   r  rÇ   r™   r  rÈ   r  r  r�  r‘  rÉ   c                 ó¨  — |
�|
n| j                   j                  }
| j                  ||||||||	|
¬«	      }|d   }| j                  |«      }| j	                  |«      }d}|�<t        «       } ||j                  d| j                  «      |j                  d«      «      }|
s|f|dd z   }|�|f|z   S |S t        |||j                  |j                  ¬«      S )a’  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`.

        Returns:

        Example:

        ```python
        >>> from transformers import AutoTokenizer, CanineForTokenClassification
        >>> import torch

        >>> tokenizer = AutoTokenizer.from_pretrained("google/canine-s")
        >>> model = CanineForTokenClassification.from_pretrained("google/canine-s")

        >>> inputs = tokenizer(
        ...     "HuggingFace is a company based in Paris and New York", add_special_tokens=False, return_tensors="pt"
        ... )

        >>> with torch.no_grad():
        ...     logits = model(**inputs).logits

        >>> predicted_token_class_ids = logits.argmax(-1)

        >>> # Note that tokens are classified rather then input words which means that
        >>> # there might be more predicted token classes than words.
        >>> # Multiple token classes might account for the same word
        >>> predicted_tokens_classes = [model.config.id2label[t.item()] for t in predicted_token_class_ids[0]]
        >>> predicted_tokens_classes  # doctest: +SKIP
        ```

        ```python
        >>> labels = predicted_token_class_ids
        >>> loss = model(**inputs, labels=labels).loss
        >>> round(loss.item(), 2)  # doctest: +SKIP
        ```Nr  r   rš   r`   r   )rƒ   rþ  r¼  rª   r  r	   r  r  r   r2   r3   )r¯   r¶   r  rÇ   r™   r  rÈ   r  r  r�  r‘  r5  r·  r"  r!  r$  rK  s                    r=   rÔ   z$CanineForTokenClassification.forward°  s÷   € ðf &1Ð%<‘kÀ$Ç+Á+×B]ÑB]ˆà—+‘+ØØ)Ø)Ø%ØØ'Ø/Ø!5Ø#ð ó 

ˆð " !™*ˆàŸ,™, Ó7ˆØ—‘ Ó1ˆàˆØÐÜ'Ó)ˆHÙ˜FŸK™K¨¨D¯O©OÓ<¸f¿k¹kÈ"»oÓNˆDáØ�Y ¨¨ Ñ,ˆFØ)-Ð)9�T�G˜fÑ$ÐE¸vÐEä$ØØØ!×/Ñ/Ø×)Ñ)ô	
ð 	
r<   r%  )r4   r5   r6   rŸ   r   r  r  r   r   r  r   r8   rÕ   r9   r7  r   r   rÔ   rÖ   r×   s   @r=   r-  r-  �  s=  ø„ ô	ñ +Ð+B×+IÑ+IÐJgÓ+hÓiÙÐ+@ÈÔ_ð 15Ø6:Ø59Ø37Ø15Ø59Ø-1Ø,0Ø/3Ø&*ñR
à˜E×,Ñ,Ñ-ðR
ð ! ×!2Ñ!2Ñ3ðR
ð ! ×!1Ñ!1Ñ2ð	R
ð
 ˜u×/Ñ/Ñ0ðR
ð ˜E×-Ñ-Ñ.ðR
ð   × 1Ñ 1Ñ2ðR
ð ˜×)Ñ)Ñ*ðR
ð $ D™>ðR
ð ' t™nðR
ð ˜d‘^ðR
ð 
ˆuÐ+Ð+Ñ	,òR
ó `ó jôR
r<   r-  zß
    CANINE Model with a span classification head on top for extractive question-answering tasks like SQuAD (a linear
    layers on top of the hidden-states output to compute `span start logits` and `span end logits`).
    c                   ó®  ‡ — e Zd Zˆ fd„Z eej                  d«      «       edee	dd¬«      	 	 	 	 	 	 	 	 	 	 	 dde
ej                     de
ej                     d	e
ej                     d
e
ej                     de
ej                     de
ej                     de
ej                     de
ej                     de
e   de
e   de
e   deeef   fd„«       «       Zˆ xZS )ÚCanineForQuestionAnsweringc                 óä   •— t         ‰| �  |«       |j                  | _        t        |«      | _        t        j                  |j                  |j                  «      | _        | j                  «        y rû   )
rž   rŸ   r  rÌ  r¼  r   r  r    Ú
qa_outputsrÓ  rä   s     €r=   rŸ   z#CanineForQuestionAnswering.__init__  sS   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒä! &Ó)ˆŒÜŸ)™) F×$6Ñ$6¸×8IÑ8IÓJˆŒð 	�‰Õr<   rø  zSplend1dchan/canine-c-squadz'nice puppet'g…ëQ¸ž!@)rú  rû  rÇ  Úexpected_outputÚexpected_lossr¶   r  rÇ   r™   r  rÈ   Ústart_positionsÚend_positionsr  r�  r‘  rÉ   c                 óð  — |�|n| j                   j                  }| j                  |||||||	|
|¬«	      }|d   }| j                  |«      }|j	                  dd¬«      \  }}|j                  d«      }|j                  d«      }d}|�·|�µt        |j                  «       «      dkD  r|j                  d«      }t        |j                  «       «      dkD  r|j                  d«      }|j                  d«      }|j                  d|«       |j                  d|«       t        |¬«      } |||«      } |||«      }||z   dz  }|s||f|dd z   }|�|f|z   S |S t        ||||j                  |j                  ¬	«      S )
a  
        start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for position (index) of the start of the labelled span for computing the token classification loss.
            Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
            are not taken into account for computing the loss.
        end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for position (index) of the end of the labelled span for computing the token classification loss.
            Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
            are not taken into account for computing the loss.
        Nr  r   r   rš   r¾   )Úignore_indexr`   )r!  Ústart_logitsÚ
end_logitsr2   r3   )rƒ   rþ  r¼  r3  rs   ræ  rz   rÎ   Úclamp_r	   r   r2   r3   )r¯   r¶   r  rÇ   r™   r  rÈ   r6  r7  r  r�  r‘  r5  r·  r"  r:  r;  Ú
total_lossÚignored_indexr$  Ú
start_lossÚend_lossrK  s                          r=   rÔ   z"CanineForQuestionAnswering.forward  s­  € ð@ &1Ð%<‘kÀ$Ç+Á+×B]ÑB]ˆà—+‘+ØØ)Ø)Ø%ØØ'Ø/Ø!5Ø#ð ó 

ˆð " !™*ˆà—‘ Ó1ˆØ#)§<¡<°°r <Ó#:Ñ ˆ�jØ#×+Ñ+¨BÓ/ˆØ×'Ñ'¨Ó+ˆ
àˆ
ØÐ&¨=Ð+Dä�?×'Ñ'Ó)Ó*¨QÒ.Ø"1×"9Ñ"9¸"Ó"=�Ü�=×%Ñ%Ó'Ó(¨1Ò,Ø -× 5Ñ 5°bÓ 9�à(×-Ñ-¨aÓ0ˆMØ×"Ñ" 1 mÔ4Ø× Ñ   MÔ2ä'°]ÔCˆHÙ! ,°Ó@ˆJÙ 
¨MÓ:ˆHØ$ xÑ/°1Ñ4ˆJáØ" JÐ/°'¸!¸"°+Ñ=ˆFØ/9Ð/E�Z�M FÑ*ÐQÈ6ÐQä+ØØ%Ø!Ø!×/Ñ/Ø×)Ñ)ô
ð 	
r<   )NNNNNNNNNNN)r4   r5   r6   rŸ   r   r  r  r   r   r  r   r8   rÕ   r9   r7  r   r   rÔ   rÖ   r×   s   @r=   r1  r1    sa  ø„ ôñ +Ð+B×+IÑ+IÐJgÓ+hÓiÙØ0Ø0Ø$Ø'Øôð 15Ø6:Ø59Ø37Ø15Ø59Ø6:Ø48Ø,0Ø/3Ø&*ñH
à˜E×,Ñ,Ñ-ðH
ð ! ×!2Ñ!2Ñ3ðH
ð ! ×!1Ñ!1Ñ2ð	H
ð
 ˜u×/Ñ/Ñ0ðH
ð ˜E×-Ñ-Ñ.ðH
ð   × 1Ñ 1Ñ2ðH
ð " %×"2Ñ"2Ñ3ðH
ð   × 0Ñ 0Ñ1ðH
ð $ D™>ðH
ð ' t™nðH
ð ˜d‘^ðH
ð 
ˆuÐ2Ð2Ñ	3òH
óó jôH
r<   r1  )r'  r1  r  r-  r{  rÌ  r»  r’   )Hr7   rÎ  r  rj   Údataclassesr   Útypingr   r   r   r8   Útorch.utils.checkpointr   Útorch.nnr   r	   r
   Úactivationsr   Úmodeling_outputsr   r   r   r   r   r   Úmodeling_utilsr   Úpytorch_utilsr   r   r   Úutilsr   r   r   r   r   Úconfiguration_caniner   Ú
get_loggerr4   rh   r  r  rµ   r/   r’   ÚModuler”   rÙ   rí   rý   r9  rC  rn  rw  r{  r‰  r   r§  r¬  r´  r»  ÚCANINE_START_DOCSTRINGr  rÌ  r  r'  r-  r1  Ú__all__r;   r<   r=   ú<module>rO     s–  ðñ ã Û Û 	Ý !ß )Ñ )ã Û Ý ß AÑ Aå !÷÷ õ .ß lÑ l÷õ õ /ð 
ˆ×	Ñ	˜HÓ	%€à'Ð Ø €ò U€ð ô: ;ó :ó ð:òD^ôBb�r—y‘yô bôJ+˜BŸI™Iô +ô\7�R—Y‘Yô 7ôta˜"Ÿ)™)ô aôH�r—y‘yô ô L�b—i‘iô Lô^˜Ÿ™ô ô�2—9‘9ô ô7�"—)‘)ô 7ôtI
�B—I‘Iô I
ôX�2—9‘9ô ô B§I¡Iô ô"˜RŸY™Yô ô(
!˜Ÿ	™	ô 
!ô*˜Oô *ð8	Ð ð/Ð ñd ØfØóôN
Ð'ó N
ó	ðN
ñb ðð óôW
Ð&;ó W
óðW
ñt ðð óôQ
Ð3ó Q
óðQ
ñh ðð óô`
Ð#8ó `
óð`
ñF ðð óô[
Ð!6ó [
óð[
ò|	�r<   