Ë
    S^(h®â  ã            
       ó¾  — d dl Z d dlZd dlmZmZmZmZmZ d dlZ	d dl
Z
ddlmZ  e«       rd dlmZ ddlmZ ddlmZ dd	lmZmZmZ erdd
lmZ ddlmZ ddlmZ  G d„ d«      Z G d„ de«      Z G d„ de«      Z G d„ d«      Z  G d„ d«      Z! G d„ de«      Z" G d„ de«      Z# G d„ de«      Z$d„ Z%dee&ef   de'd e(d!ee&ef   fd"„Z)dee&ef   de'd!ee&ef   fd#„Z*y)$é    N)ÚTYPE_CHECKINGÚAnyÚDictÚOptionalÚTupleé   )Úis_sklearn_available)Ú	roc_curve)ÚDynamicCache)Úisin_mps_friendlyé   )ÚLogitsProcessorListÚMinLengthLogitsProcessorÚSuppressTokensLogitsProcessor)ÚPreTrainedModel)ÚPreTrainedTokenizerBase)ÚGenerationConfigc                   ó¨   — e Zd ZdZdej
                  deej
                  eej                     f   fd„Z	dej
                  dej                  de
fd„Zy)	ÚCandidateGeneratorz`Abstract base class for all candidate generators that can be applied during assisted generation.Ú	input_idsÚreturnc                 ó2   — t        | j                  › d�«      ‚)ag  
        Fetches the candidates to be tried for the current input.

        Args:
            input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
                Indices of input sequence tokens in the vocabulary. [What are input IDs?](../glossary#input-ids)

        Return:
            `torch.LongTensor` of shape `(batch_size, candidate_length)` containing the candidate sequences to be
            assessed by the model and, optionally, a `torch.FloatTensor` of shape `(batch_size, candidate_length,
            vocabulary_size)` containing the logits associated to each candidate.
        zT is an abstract class. Only classes inheriting this class can call `get_candidates`.©ÚNotImplementedErrorÚ	__class__)Úselfr   s     úi/var/www/skyplay_api_hub/venv/lib/python3.12/site-packages/transformers/generation/candidate_generator.pyÚget_candidatesz!CandidateGenerator.get_candidates+   s!   € ô "Ø�~‰~ÐÐrÐsó
ð 	
ó    ÚscoresÚnum_matchesc                 ó2   — t        | j                  › d�«      ‚)áâ  
        Updates the candidate generation strategy based on the outcomes.

        Args:
            input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
                Indices of input sequence tokens in the vocabulary. [What are input IDs?](../glossary#input-ids)
            scores (`torch.FloatTensor` of shape `(batch_size, candidate_length, config.vocab_size)`):
                Prediction scores of a language modeling head. These can be logits for each vocabulary when not using
                beam search or log softmax for each vocabulary token when using beam search
            num_matches (`int`):
                The number of matches between the candidate sequences and the model predictions.
        z_ is an abstract class. Only classes inheriting this class can call `update_candidate_strategy`.r   ©r   r   r    r!   s       r   Úupdate_candidate_strategyz,CandidateGenerator.update_candidate_strategy<   s%   € ô "Ø�~‰~Ðð +ð +ó
ð 	
r   N)Ú__name__Ú
__module__Ú__qualname__Ú__doc__ÚtorchÚ
LongTensorr   r   ÚFloatTensorr   Úintr%   © r   r   r   r   (   sb   „ Ùjð
¨×(8Ñ(8ð 
¸UÀ5×CSÑCSÐU]Ð^c×^oÑ^oÑUpÐCpÑ=qó 
ð"
°5×3CÑ3Cð 
ÈU×M^ÑM^ð 
Ðmpô 
r   r   c                   óÎ  — e Zd ZdZ	 	 ddej
                  ddddded	eej                     d
dfd„Z	dej
                  de
ej
                  eej                     f   fd„Zdej
                  dej                  defd„Zdej
                  de
eef   fd„Z	 ddej
                  dededefd„Zdej
                  dededefd„Zdede
ej
                  eej                     f   fd„Zy)ÚAssistedCandidateGeneratorar  
    `CandidateGenerator` class to be used for assisted generation and speculative decoding. This class generates
    candidates through the use of a smaller model. Read the following blog post for more information:
    https://huggingface.co/blog/assisted-generation

    Args:
        input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
            Indices of input sequence tokens in the vocabulary. [What are input IDs?](../glossary#input-ids)
        assistant_model (`PreTrainedModel`):
            The model to be used for generating candidates. This model should be smaller than the main model.
        generation_config (`~generation.GenerationConfig`, *optional*):
            The generation configuration to be used as base parametrization for the generation call.
        logits_processor (`LogitsProcessorList`):
            An instance of [`LogitsProcessorList`]. List of instances of class derived from [`LogitsProcessor`]
            used to modify the prediction scores of the language modeling head applied at each generation step.
        model_kwargs (`Dict`):
            The keyword arguments that will be passed to the main model, and are used as base inputs for the assistant
            model as well.
        inputs_tensor (`torch.Tensor`, *optional*):
            The model input tensor. In encoder-decoder models, this is the encoder input.
    Nr   Úassistant_modelr   Úgeneration_configr   Úmodel_kwargsÚinputs_tensorÚlogits_processorr   c                 óè  — |j                   }|j                  |«      }|�|j                  |«      }|| _        |j                  j                  | _        |j                  j
                  | _        |j                  | j                  j                  _        i }|j                  «       D ][  \  }	}
|	dvsŒt        |
t        j                  «      r|
j                  «       j                  |«      nt        j                  |
«      ||	<   Œ] d|j                  «       v r|j                  «       s|d= |j                   j"                  rJ|j%                  ||j                  j&                  |«      \  }}}|j)                  ||||j                  «      }nd|v r|d   |d<   || _        |j                   j"                  rd| _        nyd|v rnd| _        | j*                  j/                  dt        j0                  |j2                  d   df|j                   t        j4                  ¬	«      «      | j*                  d
<   nd| _        |�|n	t7        «       | _        t        j                  |«      | _        d| j                  _        d| j                  _        | j
                  | j                  _        d| j                  _        | j                  j@                  | _!        d| j                  _         d | j                  _"        | j8                  D ]  }t        |tF        «      sŒtI        d«      ‚ d | j                  _%        tM        «       rB| j                  j                  j
                  r!tO        | «      tP        u rg | _)        g | _*        y y y y )N)Úencoder_outputsÚpast_key_valuesÚlogits_to_keepr7   Údecoder_input_idsr   Údecoder_attention_maskr   r   )ÚdeviceÚdtypeÚattention_maskTz†Passing `MinLengthLogitsProcessor` when using `assisted_generation is disabled. Please pass in `min_length` into `.generate()` instead)+r<   Útor1   r2   Únum_assistant_tokensÚassistant_confidence_thresholdÚeos_token_idÚitemsÚ
isinstancer*   ÚTensorÚdetachÚcopyÚdeepcopyÚkeysÚ_supports_logits_to_keepÚconfigÚis_encoder_decoderÚ_prepare_model_inputsÚbos_token_idÚ._prepare_encoder_decoder_kwargs_for_generationÚassistant_kwargsÚinput_ids_keyÚgetÚonesÚshapeÚlongr   r5   Úreturn_dict_in_generateÚoutput_scoresÚis_assistantÚ
min_lengthÚmain_model_min_lengthÚmin_new_tokensr   Ú
ValueErrorÚcache_implementationr	   Útyper0   ÚprobsÚmatches)r   r   r1   r2   r3   r4   r5   r<   rP   ÚkeyÚvalueÚmodel_input_nameÚ	processors                r   Ú__init__z#AssistedCandidateGenerator.__init__f   s@  € ð !×'Ñ'ˆØ—L‘L Ó(ˆ	ØÐ$Ø)×,Ñ,¨VÓ4ˆMð  /ˆÔØ$3×$EÑ$E×$ZÑ$ZˆÔ!Ø.=×.OÑ.O×.nÑ.nˆÔ+ð ?P×>\Ñ>\ˆ×Ñ×.Ñ.Ô;ð ÐØ&×,Ñ,Ó.ò 	‰JˆC�ØÐ@Ò@ä1;¸EÄ5Ç<Á<Ô1P�E—L‘L“N×%Ñ% fÔ-ÔVZ×VcÑVcÐdiÓVjð ! Ò%ð	ð Ð/×4Ñ4Ó6Ñ6¸×?gÑ?gÔ?iØ Ð!1Ð2ð ×!Ñ!×4Ò4Ø@O×@eÑ@eØ˜×@Ñ@×MÑMÐO_óAÑ=ˆMÐ+Ð-=ð  /×]Ñ]ØÐ/Ð1AÀ?×CdÑCdó Ñð  ,Ñ.Ø2>Ð?PÑ2QÐÐ.Ñ/Ø 0ˆÔð ×!Ñ!×4Ò4à!4ˆDÕØÐ"2Ñ2à!,ˆDÔØ6:×6KÑ6K×6OÑ6OØ(Ü—
‘
˜IŸO™O¨AÑ.°Ð2¸9×;KÑ;KÔSX×S]ÑS]Ô^ó7ˆD×!Ñ!Ð"2Ò3ð "-ˆDÔð 5EÐ4PÑ 0ÔViÓVkˆÔÜ!%§¡Ð/@Ó!AˆÔà9=ˆ×ÑÔ6Ø/3ˆ×ÑÔ,Ø@D×@cÑ@cˆ×ÑÔ=à.2ˆ×ÑÔ+ð &*×%;Ñ%;×%FÑ%FˆÔ"Ø,-ˆ×ÑÔ)Ø04ˆ×ÑÔ-Ø×.Ñ.ò 	ˆIÜ˜)Ô%=Õ>Ü ðMóð ð	ð 7;ˆ×ÑÔ3ô !Ô"Ø×$Ñ$×6Ñ6×UÒUÜ�T“
Ô8Ñ8àˆDŒJØˆD�Lð 9ð Vð #r   r   c                 óþ   — |j                  | j                  j                  «      }| j                  |«      \  }}|dk(  r|dfS | j	                  |«       | j                  |||«      }| j                  |«      \  }}||fS )áZ  
        Fetches the candidates to be tried for the current input.

        Args:
            input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
                Indices of input sequence tokens in the vocabulary. [What are input IDs?](../glossary#input-ids)

        Return:
            `torch.LongTensor` of shape `(batch_size, candidate_length)` containing the candidate sequences to be
            assessed by the model and a `torch.FloatTensor` of shape `(batch_size, candidate_length,
            vocabulary_size)` containing the logits associated to each candidate.
        r   N)r?   r1   r<   Ú_calculate_new_tokensÚ_update_past_and_masksÚ_prepare_generation_argsÚ_generate_candidates)r   r   r[   Úmax_new_tokensÚgeneration_argsÚcandidate_idsÚcandidate_logitss          r   r   z)AssistedCandidateGenerator.get_candidatesÅ   sŠ   € ð —L‘L ×!5Ñ!5×!<Ñ!<Ó=ˆ	à)-×)CÑ)CÀIÓ)NÑ&ˆ˜Ø˜QÒØ˜d�?Ð"à×#Ñ# IÔ.à×7Ñ7¸	À>ÐSaÓbˆØ*.×*CÑ*CÀOÓ*TÑ'ˆÐ'ØÐ.Ð.Ð.r   r    r!   c                 óÞ  — | j                   j                  j                  dv rH|t        |d   «      dz
  k(  r| xj                  dz  c_        nt        d| j                  dz
  «      | _        t        «       �rx| j                   j                  j                  �rVt        | «      t        u �rC| j                  j                  dg|z  «       t        | j                  «      t        | j                  «      kD  r| j                  j                  d«       t        | j                  «      t        | j                  «      z
  }|dkD  r| j                  | d…= t        | j                  «      dkD  r…ddhj                  | j                  «      rgt        | j                  | j                  «      \  }}}d|z
  }|d|z  z   }	t!        j"                  |	«      }
||
   }|| j                   j                  _        yyyyyy)	r#   >   Ú	heuristicÚheuristic_transientr   r   g       @g      ð?Né   é   )r1   r2   Únum_assistant_tokens_scheduleÚlenr@   Úmaxr	   rA   r^   r0   r`   Úextendr_   ÚappendÚissubsetr
   ÚnpÚargmin)r   r   r    r!   Úexcess_lengthÚfprÚtprÚ
thresholdsÚfnrÚcostsÚoptimal_threshold_indexÚbest_thresholds               r   r%   z4AssistedCandidateGenerator.update_candidate_strategyÞ   s­  € ð  ×Ñ×1Ñ1×OÑOð T
ñ 
ð
 œc &¨¡)›n¨qÑ0Ò0Ø×)Ò)¨SÑ0Ö)ä,/°°T×5NÑ5NÐQTÑ5TÓ,U�Ô)ô
 !Õ"Ø×$Ñ$×6Ñ6×UÓUÜ�T“
Ô8Ò8ð �L‰L×Ñ   kÑ 1Ô2Ü�4—:‘:‹¤ T§\¡\Ó!2Ò2Ø—‘×#Ñ# AÔ&ô   §
¡
›O¬c°$·,±,Ó.?Ñ?ˆMØ˜qÒ Ø—J‘J ˜~™Ð/ô �D—J‘J“ !Ò#¨¨A¨¯©¸¿¹Ô(Eä'0°·±¸t¿z¹zÓ'JÑ$��S˜*Ø˜#‘g�ð ˜a #™g™�ô +-¯)©)°EÓ*:Ð'Ø!+Ð,CÑ!D�àXf�×$Ñ$×6Ñ6ÕUð )FÐ#ð 9ð Vð #r   c                 óà   — |j                   d   }t        t        | j                  «      | j                  j
                  |z
  dz
  «      }t        t        || j                  |z
  «      d«      }||fS )zCCalculate the minimum and maximum number of new tokens to generate.éÿÿÿÿr   r   )rT   Úminr-   r@   r2   Ú
max_lengthrw   rZ   )r   r   Únew_cur_lenrl   r[   s        r   rh   z0AssistedCandidateGenerator._calculate_new_tokens  si   € à—o‘o bÑ)ˆÜœS ×!:Ñ!:Ó;¸T×=SÑ=S×=^Ñ=^ÐalÑ=lÐopÑ=pÓqˆÜœS °×1KÑ1KÈkÑ1YÓZÐ\]Ó^ˆØ˜~Ð-Ð-r   Úremove_from_pkvÚnum_added_tokensc                 ó¶  — | j                   j                  dd«      du}|r¸|j                  d   dz
  |z
  }t        | j                  | j                   d   ||z
  «      | j                   d<   t        | j                   |j                  d   | j                  j                  j                  «      | _         t        | j                   |j                  d   «      | _         |S )zLUpdate past key values and attention masks for subsequent generation rounds.r8   Nr†   r   )	rP   rR   rT   Ú_crop_past_key_valuesr1   Ú_prepare_attention_maskrK   rL   Ú_prepare_token_type_ids)r   r   rŠ   r‹   Úhas_past_key_valuesÚnew_cache_sizes         r   ri   z1AssistedCandidateGenerator._update_past_and_masks  sÖ   € ð #×3Ñ3×7Ñ7Ð8IÈ4ÓPÐX\Ð\ÐÙØ&Ÿ_™_¨RÑ0°1Ñ4°ÑFˆNÜ7LØ×$Ñ$ d×&;Ñ&;Ð<MÑ&NÐP^ÐaqÑPqó8ˆD×!Ñ!Ð"3Ñ4ô %<Ø×%Ñ% y§¡°rÑ':¸D×<PÑ<P×<WÑ<W×<jÑ<jó%ˆDÔ!ô %<¸D×<QÑ<QÐS\×SbÑSbÐceÑSfÓ$gˆDÔ!à"Ð"r   r[   rl   c           
      óV   — | j                   |d|d|d| j                  d| j                  iS )z*Prepare arguments for the generation call.r[   rl   r2   r5   )rQ   r2   r5   )r   r   r[   rl   s       r   rj   z3AssistedCandidateGenerator._prepare_generation_args0  s9   € ð ×Ñ 	Ø˜nØ˜nØ ×!7Ñ!7Ø × 5Ñ 5ð
ð 	
r   rm   c                 óž  —  | j                   j                  di |¤| j                  ¤Ž}|j                  | j                  d<   t	        «       rÒ| j                   j
                  j                  r²t        | «      t        u r¡t        j                  |j                  d¬«      }t        j                  |d¬«      }|j                  dt        |j                  «       d…f   }|t        t        |«      «      |f   }| j                   j#                  |j%                  «       «       t        j&                  |j                  d¬«      }|j                  }||fS )z7Generate candidate sequences using the assistant model.r8   r   ©Údimr†   Nr   r.   )r1   ÚgeneraterP   r8   r	   r2   rA   r^   r0   r*   Úcatr    ÚsoftmaxÚ	sequencesrv   Úranger_   rx   ÚtolistÚstack)	r   rm   Úassistant_outputÚscores_tensorÚscores_softmaxÚidsÚpro   rn   s	            r   rk   z/AssistedCandidateGenerator._generate_candidates:  s  € à8˜4×/Ñ/×8Ñ8Ñd¸?ÐdÈd×NcÑNcÑdÐØ3C×3SÑ3Sˆ×ÑÐ/Ñ0ä Ô"Ø×$Ñ$×6Ñ6×UÒUÜ�T“
Ô8Ñ8ä!ŸI™IÐ&6×&=Ñ&=À1ÔEˆMÜ"Ÿ]™]¨=¸bÔAˆNØ"×,Ñ,¨R´#Ð6F×6MÑ6MÓ2NÐ1NÑ1PÐ-PÑQˆCØœu¤S¨£X›°Ð3Ñ4ˆAØ�J‰J×Ñ˜aŸh™h›jÔ)Ü Ÿ;™;Ð'7×'>Ñ'>ÀAÔFÐØ(×2Ñ2ˆØÐ.Ð.Ð.r   ©NN)r   r   )r&   r'   r(   r)   r*   r+   r   r   rE   re   r   r,   r   r-   r%   rh   Úboolri   rj   rk   r.   r   r   r0   r0   O   st  „ ñð8 15Ø26ñ]à×#Ñ#ð]ð +ð]ð .ð	]ð
 ð]ð   §¡Ñ-ð]ð 0ó]ð~/¨×(8Ñ(8ð /¸UÀ5×CSÑCSÐU]Ð^c×^oÑ^oÑUpÐCpÑ=qó /ð28g°5×3CÑ3Cð 8gÈU×M^ÑM^ð 8gÐmpó 8gðt.¨u×/?Ñ/?ð .ÀEÈ#ÈsÈ(ÁOó .ð ^_ñ#Ø×)Ñ)ð#Ø<?ð#ØWZð#à	ó#ð"
°%×2BÑ2Bð 
ÐTWð 
Ðilð 
Ðquó 
ð/°Dð /¸UÀ5×CSÑCSÐU]Ð^c×^oÑ^oÑUpÐCpÑ=qô /r   r0   c                   ó´  ‡ — e Zd ZdZ	 	 ddej
                  dddddddd	d
edeej                     ddfˆ fd„Z	e
d„ «       Ze
d„ «       Ze
d„ «       Zd„ Zdej
                  deej
                  eej                      f   fd„Zdej
                  deej
                  ef   fd„Zdej
                  dej
                  dej
                  dej
                  fd„Zˆ xZS )Ú-AssistedCandidateGeneratorDifferentTokenizersaƒ  
    `CandidateGenerator` class to be used for Universal Assisted Generation (UAD): assisted generation with different tokenizers
    for the assistant and main models. This class generates candidates through the use of a smaller
    model.

    The main model input tokens are re-encoded into assistant model tokens, then candidate tokens are generated in the assistant encoding, which are
    in turn re-encoded into main model candidate tokens. Validation then proceeds as explained above.
    The re-encoding steps involve decoding token ids into text and then encoding the text using a different tokenizer.
    Since re-encoding the tokens may result in tokenization discrepancies, UAD finds the longest common subsequence between the source and target encodings,
    to ensure the new tokens include the correct prompt suffix.

    Args:
        input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
            Indices of input sequence tokens in the vocabulary. [What are input IDs?](../glossary#input-ids)
        assistant_model (`PreTrainedModel`):
            The model to be used for generating candidates. This model should be smaller than the main model.
        target_tokenizer (`PreTrainedTokenizerBase`):
            The tokenizer used for the target model.
        assistant_tokenizer (`PreTrainedTokenizerBase`):
            The tokenizer used for the assistant model.
        generation_config (`~generation.GenerationConfig`, *optional*):
            The generation configuration to be used as base parametrization for the generation call.
        logits_processor (`LogitsProcessorList`):
            An instance of [`LogitsProcessorList`]. List of instances of class derived from [`LogitsProcessor`]
            used to modify the prediction scores of the language modeling head applied at each generation step.
        model_kwargs (`Dict`):
            The keyword arguments that will be passed to the main model, and are used as base inputs for the assistant
            model as well.
        inputs_tensor (`torch.Tensor`, *optional*):
            The model input tensor. In encoder-decoder models, this is the encoder input.
    r   r1   r   Útarget_tokenizerr   Úassistant_tokenizerr2   r   r3   r4   r5   r   c	                 óÒ   •— t         ‰	| �  ||||||«       || _        || _        d | _        d | _        |j                  j                  | _        |j                  j                  | _        y ©N)	Úsuperre   r¦   r§   Úprev_target_ids_lenÚprev_assistant_idsr2   Útarget_lookbehindÚassistant_lookbehind)
r   r   r1   r¦   r§   r2   r3   r4   r5   r   s
            €r   re   z6AssistedCandidateGeneratorDifferentTokenizers.__init__n  sg   ø€ ô 	‰Ñ˜ OÐ5FÈÐVcÐeuÔvà 0ˆÔØ#6ˆÔ Ø26ˆÔ Ø"&ˆÔØ!0×!BÑ!B×!TÑ!TˆÔØ$3×$EÑ$E×$ZÑ$ZˆÕ!r   c                 ó  — t        «       }i }|D ]í  }t        j                  |«      }t        |j	                  «       «      }||v rŒ6|j                  |«       d}|dz  }|d   | j                  d   k  r†|d   | j                  d   k  rqt        |j	                  «       «      }|j                  |«       | |d   |d   f   dk(  r|dz  }|dz  }nn+|d   | j                  d   k  r|d   | j                  d   k  rŒq|||<   Œï |S )aÆ  
        Calculates the length of the longest diagonal sequence in a given matrix.
        Args:
            input_matrix (torch.Tensor): The input matrix.
            nonzero_idx (torch.Tensor): The indices of the non-zero elements in the matrix.
        Returns:
            dict: A dictionary where the keys are the indices of the non-zero elements and the values are the lengths of the longest diagonal sequences starting from those indices.
        r   r   )Úsetr*   ÚcloneÚtupler›   ÚaddrT   )Úinput_matrixÚnonzero_idxÚvisitedÚdiagsÚidxÚ	start_idxÚtuple_start_idxÚcur_diag_lens           r   Ú_get_longest_diag_dictzDAssistedCandidateGeneratorDifferentTokenizers._get_longest_diag_dict‚  s$  € ô “%ˆØˆØò 	&ˆCÜŸ™ CÓ(ˆIÜ# I×$4Ñ$4Ó$6Ó7ˆOà 'Ñ)Øà�K‰K˜Ô(ØˆLØ˜‰NˆIØ˜A‘, ×!3Ñ!3°AÑ!6Ò6¸9ÀQ¹<È,×J\ÑJ\Ð]^ÑJ_Ò;_Ü"'¨	×(8Ñ(8Ó(:Ó";�Ø—‘˜OÔ,à 	¨!¡¨i¸©lÐ :Ñ;¸qÒ@Ø  AÑ%�LØ ‘N‘Iàð ˜A‘, ×!3Ñ!3°AÑ!6Ò6¸9ÀQ¹<È,×J\ÑJ\Ð]^ÑJ_Ó;_ð &ˆE�#ŠJð)	&ð* ˆr   c                 óô   — t         j                  | | j                  «       «      }t        |j	                  «       «      }t        |j                  «       «      }t        j                  |«      }||   }||   }||fS )a  
        Returns the start index and length of the longest diagonal in the given input.
        Args:
            input_matrix (numpy.ndarray): The input matrix.
        Returns:
            tuple: A tuple containing the start index and length of the longest diagonal.
        )r¥   r¼   ÚnonzeroÚlistÚvaluesrI   r{   Úargmax)r´   r·   Údiags_valuesÚ
diags_keysÚ	best_diagÚdiag_start_indexÚdiag_start_lengths          r   Ú_get_longest_diag_indexzEAssistedCandidateGeneratorDifferentTokenizers._get_longest_diag_index¦  st   € ô >×TÑTØ˜,×.Ñ.Ó0ó
ˆô ˜EŸL™L›NÓ+ˆÜ˜%Ÿ*™*›,Ó'ˆ
Ü—I‘I˜lÓ+ˆ	Ø% iÑ0ÐØ(¨Ñ3ÐØÐ!2Ð2Ð2r   c                 ó®  — |j                   | k(  }t        j                  |«      st        j                  |«      }|j	                  t
        «      }|j                  «       j                  «       syt        j                  |«      \  }}|d   |z   }|d   |z   }| j                  d   |z
  j                  «       }|dd…||z   d…f   }	|dd…|||z   …f   }
||	|
fS )a¹  
        Input:
            prompt: 2D array of shape (batch_size, prompt_length), represents the original prompt tokens
            prompt_plus_new_tokens: 2D array of shape (batch_size, prompt_length), represents the suffix of the original prompt, with additional new tokens.
        Output:
            discrepancy_length: int, represents the number of tokens that need to be replaced from prompt
            new_tokens_only: 2D array of shape (batch_size, new_token_length), represents the new tokens that are not in prompt
            discrepancy_only: 2D array of shape (batch_size, discrepancy_length), represents the new tokens that are in prompt but not in prompt_plus_new_tokens
        )NNNr   r   N)ÚTr*   Ú	is_tensorÚtensorr?   r-   ÚanyÚitemr¥   rÇ   rT   )ÚpromptÚprompt_plus_new_tokensÚcompare_matÚcompare_mat_intÚlongest_locationÚlongest_diag_lengthÚnew_token_start_indexÚdiscrepancy_with_oldÚdiscrepancy_lengthÚnew_tokens_onlyÚdiscrepancy_onlys              r   Ú_get_tokens_diagz>AssistedCandidateGeneratorDifferentTokenizers._get_tokens_diagº  sø   € ð -×.Ñ.°&Ñ8ˆÜ�‰˜{Ô+ÜŸ,™, {Ó3ˆKà%Ÿ.™.¬Ó-ˆà×"Ñ"Ó$×)Ñ)Ô+à#ä0]×0uÑ0uØó1
Ñ-ÐÐ-ð !1°Ñ 3Ð6IÑ IÐØ/°Ñ2Ð5HÑHÐØ$Ÿl™l¨1™oÐ0DÑD×JÑJÓLÐØ0²Ð4IÐL^Ñ4^Ñ4`Ð1`ÑaˆØ1ÚÐ$Ð'<Ð?QÑ'QÐQÐQñ
Ðð " ?Ð4DÐDÐDr   c                 ó|   — |j                  |dd¬«      } ||dd¬«      d   }|j                  |j                  «      S )a$  
        Convert token IDs from one tokenizer to another.
        Args:
            input_ids: The input token IDs.
            source_tokenizer: The source tokenizer.
            destination_tokenizer: The destination tokenizer.
        Returns:
            The converted token IDs.
        T©Úskip_special_tokensÚclean_up_tokenization_spacesÚpt©Úadd_special_tokensÚreturn_tensorsr   )Úbatch_decoder?   r<   )r   r   Úsource_tokenizerÚdestination_tokenizerÚtextÚdest_idss         r   Ú&convert_source_tokens_to_target_tokenszTAssistedCandidateGeneratorDifferentTokenizers.convert_source_tokens_to_target_tokensÛ  sF   € ð  ×,Ñ,¨YÈDÐosÐ,ÓtˆÙ(¨À$ÐW[Ô\Ð]hÑiˆØ�{‰{˜9×+Ñ+Ó,Ð,r   r   c                 óò  — t        | j                  «      }|dk(  r|dfS |j                  | j                  j                  «      }d}| j                  |«      \  }}|| _        t        t        || j                  |j                  d   z
  «      d«      }| j                  ||«       | j                  |||«      }| j                  j                  dd«        | j                  j                  di |¤| j                  ¤Ž}| j!                  ||j"                  |«      }|j                  d   | _        |j&                  | j                  d<   |j"                  | _        | j$                  |j                  d   k\  r|dfS |dfS )rg   r   Nr†   r>   r   r8   r.   )r-   r@   r?   r1   r<   Ú_prepare_assistant_input_idsr¬   rw   r‡   rZ   rT   ri   rj   rP   Úpopr–   Ú_process_assistant_outputsr™   r«   r8   )	r   r   rl   rŠ   Úassistant_input_idsr[   rm   r�   Únew_target_idss	            r   r   z<AssistedCandidateGeneratorDifferentTokenizers.get_candidatesî  sw  € ô ˜T×6Ñ6Ó7ˆØ˜QÒØ˜d�?Ð"à—L‘L ×!5Ñ!5×!<Ñ!<Ó=ˆ	Øˆà/3×/PÑ/PÐQZÓ/[Ñ,Ð˜_Ø"5ˆÔäœS °×1KÑ1KÐNa×NgÑNgÐhjÑNkÑ1kÓlÐnoÓpˆà×#Ñ#Ð$7¸ÔIØ×7Ñ7Ð8KÈ^Ð]kÓlˆØ×Ñ×!Ñ!Ð"2°DÔ9à8˜4×/Ñ/×8Ñ8Ñd¸?ÐdÈd×NcÑNcÑdÐØ×8Ñ8¸ÐDT×D^ÑD^Ð`sÓtˆð $-§?¡?°1Ñ#5ˆÔ Ø3C×3SÑ3Sˆ×ÑÐ/Ñ0Ø"2×"<Ñ"<ˆÔà×#Ñ# ~×';Ñ';¸AÑ'>Ò>Ø˜d�?Ð"à˜tÐ#Ð#r   c                 ór  — | j                   | j                  dœ}d}| j                  ��e| j                  | j                  kD  �rK| j                  | j                  z
  } | j
                  |dd…|d…f   fi |¤Ž}|j                  d   }| j                  dd…| d…f   }| j                  ||«      \  }}	}
| j                  }|	�°|dkD  r||
j                  d   dkD  rj||
j                  d   k(  r|
|dd…| d…f<   nI||
j                  d   kD  r7||
j                  d   z
  }|dd…d| …f   }|
|dd…|
j                  d    d…f<   |}|	j                  d   dkD  r6t        j                  ||	gd¬«      }||fS t        j                  ||gd¬«      }||fS  | j
                  |fi |¤Ž}|j                  d   | _        ||fS )zIConverts target input IDs to assistant input IDs, handling discrepancies.©rã   rä   r   Nr   r†   r”   )
r¦   r§   r¬   r«   r­   rç   rT   rÙ   r*   r—   )r   r   Úconvert_kwargsrŠ   Ústart_index_in_target_windowÚnew_assistant_idsÚprompt_use_lengthÚ
prompt_userÖ   r×   rØ   rì   Údiscrepancy_length_diffs                r   ré   zJAssistedCandidateGeneratorDifferentTokenizers._prepare_assistant_input_ids  s4  € ð !%× 5Ñ 5Ø%)×%=Ñ%=ñ
ˆð ˆà×"Ñ"Ñ.°4×3KÑ3KÈd×NdÑNdÓ3dà+/×+CÑ+CÀd×F\ÑF\Ñ+\Ð(à K × KÑ KØš!Ð9Ñ:Ð:Ñ;ñ!Ø?Mñ!Ðð !2× 7Ñ 7¸Ñ :ÐØ×0Ñ0²Ð5FÐ4FÑ4GÐ1GÑHˆJàDH×DYÑDYØÐ-óEÑAÐ Ð1Að #'×"9Ñ"9ÐàÐ*Ø%¨Ò)Ð.>×.DÑ.DÀQÑ.GÈ!Ò.KØ)Ð-=×-CÑ-CÀAÑ-FÒFØGWÐ+ªAÐ0BÐ/BÑ/CÐ,CÒDà+Ð.>×.DÑ.DÀQÑ.GÒGØ2DÐGW×G]ÑG]Ð^_ÑG`Ñ2`Ð/Ø.AÂ!ÐE^ÐG^ÐF^ÐE^ÐB^Ñ._Ð+ØO_Ð+ªAÐ0@×0FÑ0FÀqÑ0IÐ/IÑ/KÐ,KÑLà&8�Oà"×(Ñ(¨Ñ+¨aÒ/Ü*/¯)©)Ð5HÈ/Ð4ZÐ`bÔ*cÐ'ð # OÐ3Ð3ô ',§i¡iÐ1DÐFWÐ0XÐ^`Ô&aÐ#ð
 # OÐ3Ð3ð #N $×"MÑ"MÈiÑ"jÐ[iÑ"jÐØ'0§¡°qÑ'9ˆDÔ$à" OÐ3Ð3r   Úassistant_sequencesrì   c                 ó  — | j                   j                  d   }|| j                  z
  }| j                  |dd…|d…f   | j                  | j
                  ¬«      }|j                  d   }|dd…| d…f   }| j                  ||«      \  }	}
}	|}|
�,|
j                  d   dkD  r3t        j                  ||
gd¬«      }nt        j                  ||gd¬«      }t        | j                  d«      r|dd…d| j                  j                  …f   }|S )z7Processes assistant outputs to obtain target input IDs.r   Nrï   r   r†   r”   rˆ   )r¬   rT   r®   rç   r§   r¦   rÙ   r*   r—   Úhasattrr2   rˆ   )r   r   rö   rì   Únum_prev_assistantÚstart_assistant_look_indexÚnew_target_ids_from_windowÚtarget_prompt_use_lengthÚtarget_prompt_useÚ_Útarget_new_tokens_onlyrí   s               r   rë   zHAssistedCandidateGeneratorDifferentTokenizers._process_assistant_outputsF  s1  € ð "×4Ñ4×:Ñ:¸1Ñ=ÐØ%7¸$×:SÑ:SÑ%SÐ"à%)×%PÑ%PØ¢Ð#=Ñ#>Ð >Ñ?Ø!×5Ñ5Ø"&×"7Ñ"7ð &Qó &
Ð"ð
 $>×#CÑ#CÀAÑ#FÐ à%¢aÐ*BÐ)BÑ)CÐ&CÑDÐà'+×'<Ñ'<Ð=NÐPjÓ'kÑ$ˆÐ! 1à"ˆà!Ð-Ø%×+Ñ+¨AÑ.°Ò2Ü!&§¡¨NÐ<RÐ+SÐY[Ô!\‘ô #ŸY™Y¨Ð8RÐ'SÐY[Ô\ˆNä�4×)Ñ)¨<Ô8Ø+ªAÐ/R°×1GÑ1G×1RÑ1RÐ/RÐ,RÑSˆNàÐr   r¢   )r&   r'   r(   r)   r*   r+   r   r   rE   re   Ústaticmethodr¼   rÇ   rÙ   rç   r   r,   r   r-   ré   rë   Ú__classcell__©r   s   @r   r¥   r¥   M  sa  ø„ ñðP 15Ø26ñ[à×#Ñ#ð[ð +ð[ð 4ð	[ð
 7ð[ð .ð[ð ð[ð   §¡Ñ-ð[ð 0õ[ð( ñ!ó ð!ðF ñ3ó ð3ð& ñEó ðEò@-ð&($¨×(8Ñ(8ð ($¸UÀ5×CSÑCSÐU]Ð^c×^oÑ^oÑUpÐCpÑ=qó ($ðT,4°e×6FÑ6Fð ,4È5ÐQV×QaÑQaÐcfÐQfÑKgó ,4ð\Ø×)Ñ)ðØ@E×@PÑ@PðØgl×gwÑgwðà	×	Ñ	÷r   r¥   c            	       óà   — e Zd ZU dZ ed«       Zeed<   dZeed<   	 dddddd	ed
e	fd„Z
d„ Zdee   fd„Zdej                  dej                  fd„Zdej"                  dej"                  fd„Zy)ÚAssistantToTargetTranslatoraÿ  
    Translates token ids and logits between assistant and target model vocabularies. This class is used to handle
    vocabulary mismatches when using different tokenizers for the assistant and target models in speculative decoding,
    as introduced in the paper "Lossless Speculative Decoding Algorithms for Heterogeneous Vocabularies"
    (https://www.arxiv.org/abs/2502.05202).
    It maintains mappings between the two vocabularies and handles token/logit conversion.

    Args:
        target_tokenizer (`PreTrainedTokenizerBase`):
            The tokenizer used by the target (main) model.
        assistant_tokenizer (`PreTrainedTokenizerBase`):
            The tokenizer used by the assistant model.
        assistant_model_device (`str`, defaults to "cpu"):
            The device where the assistant model is located. Used for placing tensors.
        target_vocab_size (`int`, *optional*):
            The size of the target model's vocabulary. If not provided, will be inferred from the target tokenizer.
    ÚInfÚFILTER_VALUEr†   ÚSUPPRESS_TOKEN_IDr¦   r   r§   Útarget_vocab_sizeÚassistant_model_devicec                 óF  — || _         || _        || _        || _        | j	                  «       \  | _        | _        | j                  «       | _        d | _	        t        | j                  «      dkD  r4t        t        | j                  «       | j                  «      g«      | _	        y y ©Nr   )Ú_target_tokenizerÚ_assistant_tokenizerÚ_assistant_model_devicer  Ú"_get_assistant_to_target_input_idsÚ_assistant_to_target_input_idsÚtarget_to_assistant_input_idsÚ_get_suppress_input_idsÚ_suppress_input_idsÚlogits_processorsrv   r   r   )r   r¦   r§   r  r	  s        r   re   z$AssistantToTargetTranslator.__init__}  s�   € ð =MˆÔØ?RˆÔ!Ø,BˆÔ$Ø&7ˆÔà×3Ñ3Ó5ñ 	PˆÔ+¨TÔ-Oð /3×.JÑ.JÓ.LˆÔ Ø@DˆÔÜˆt×'Ñ'Ó(¨1Ò,ä%8Ü.¨t×/KÑ/KÓ/MÈt×OkÑOkÓlÐmó&ˆDÕ"ð -r   c           	      óV  — | j                   j                  «       }| j                  j                  «       }d}| j                  |d¬«      d   }t        |«      dkD  r²| j                   j	                  |«      d   d   }| j                  |d¬«      d   }t        |«      dkD  rm| j                  j	                  |«      d   d   }||k7  rG|j                  «       D ��	ci c],  \  }}	|j                  |«      r|j                  ||d«      n||	“Œ. }}}	t        |j                  «       «      }
t        j                  |
dz   f| j                  t        ¬«      }i }|j                  «       D ]#  \  }}|j                  |«      }|€Œ|||<   |||<   Œ% |j                  | j                   «      |fS c c}	}w )Nú F)rà   r   r   r   ©r=   )r  Ú	get_vocabr  rv   Úconvert_ids_to_tokensrC   Ú
startswithÚreplacerw   rÀ   r*   Úfullr  r-   rR   r?   r  )r   Útarget_vocabÚassistant_vocabÚ	space_strÚtarget_space_idsÚtarget_space_signÚassistant_space_idsÚassistant_space_signÚtokr¸   Úmax_assistant_indexÚassistant_to_target_input_idsr  Úassistant_idÚ	target_ids                  r   r  z>AssistantToTargetTranslator._get_assistant_to_target_input_ids“  sÜ  € Ø×-Ñ-×7Ñ7Ó9ˆØ×3Ñ3×=Ñ=Ó?ˆàˆ	Ø×1Ñ1°)ÐPUÐ1ÓVÐWbÑcÐÜÐÓ  1Ò$Ø $× 6Ñ 6× LÑ LÐM]Ó ^Ð_`Ñ aÐbcÑ dÐà"&×";Ñ";¸IÐZ_Ð";Ó"`ÐalÑ"mÐÜÐ&Ó'¨!Ò+Ø'+×'@Ñ'@×'VÑ'VÐWjÓ'kÐlmÑ'nÐopÑ'qÐ$à$Ð(<Ò<ð )8×(=Ñ(=Ó(?÷'ñ %˜C ð  #Ÿ~™~Ð.BÔCð  ŸK™KÐ(<Ð>OÐQRÔSà!$Øñð'�Oñ 'ô " /×"8Ñ"8Ó":Ó;ÐÜ(-¯
©
Ð4GÈ!Ñ4KÐ3MÈt×OeÑOeÔmpÔ(qÐ%Ø8:Ð%Ø!0×!6Ñ!6Ó!8ò 	HÑˆC�Ø$×(Ñ(¨Ó-ˆIØÑ$Ø>GÐ-¨lÑ;Ø;GÐ-¨iÒ8ð		Hð
 -×/Ñ/°×0LÑ0LÓMÐOlÐlÐlùó#'s   Ã1F%r   c                 ó`   — t        j                  | j                  | j                  k(  «      d   S )z`
        Get the input ids that are in the assistant vocab but not in the target vocab.
        r   )r*   Úwherer  r  )r   s    r   r  z3AssistantToTargetTranslator._get_suppress_input_ids¶  s*   € ô �{‰{˜4×>Ñ>À$×BXÑBXÑXÓYÐZ[Ñ\Ð\r   Úassistant_candidate_idsc                 óÊ   — t        |d   «      |j                  d   z
  }|dk(  r|S | j                  |d| d…f      }t        j                  ||j                  d«      fd¬«      S )aS  
        Return the target candidate ids that correspond to the assistant candidate ids.
        Note that we have already the target ids for the prompt and we only need to find the target ids for the new tokens.
        Moreover, assistant ids of the original prompt does not necessarily appear in _assistant_to_target_input_ids.
        r   r   Nr”   )rv   rT   r  r*   r—   Ú	unsqueeze)r   rì   Útarget_input_idsr+  Únum_new_tokensÚtransformed_slices         r   Úget_target_idsz*AssistantToTargetTranslator.get_target_ids¼  s|   € ô Ð4°QÑ7Ó8Ð;N×;TÑ;TÐUVÑ;WÑWˆØ˜QÒØ#Ð#à $× CÑ CÐD[Ð\]Ð`nÐ_nÑ_oÐ\oÑDpÑ qÐÜ—9‘9Ð.Ð0A×0KÑ0KÈAÓ0NÐOÐUVÔWÐWr   Úassistant_logitsc                 óB  — g |j                   dd ¢| j                  ‘­}t        j                  || j                  | j
                  ¬«      }| j                  | j                  k7  }| j                  |   }|dd| j                  j                   d   …f   }|d|f   |d|f<   |S )zS
        Return the target logits that correspond to the assistant logits.
        Nr†   ©r<   .r   )rT   r  r*   r  r  r  r  r  )r   r2  Útarget_shapeÚtarget_logitsÚassistant_indices_maskÚtarget_logits_supported_indicesÚvalid_assistant_logitss          r   Úget_target_logitsz-AssistantToTargetTranslator.get_target_logitsÌ  s¾   € ð
 )_Ð*:×*@Ñ*@ÀÀ"Ð*EÐ(^Àt×G]ÑG]Ñ(^ˆÜ+0¯:©:Ø˜$×+Ñ+°D×4PÑ4Pô,
ˆð "&×!DÑ!DÈ×H^ÑH^Ñ!^Ðà*.×*MÑ*MÐNdÑ*eÐ'Ø!1°#Ð7e¸×9\Ñ9\×9bÑ9bÐcdÑ9eÐ7eÐ2eÑ!fÐà>TÐUXÐZpÐUpÑ>qˆ�cÐ:Ð:Ñ;àÐr   N©Úcpu)r&   r'   r(   r)   Úfloatr  Ú__annotations__r  r-   Ústrre   r  r¿   r  r*   r+   r1  r,   r:  r.   r   r   r  r  g  s°   … ññ$ ! ›<˜-€L�%Ó'ØÐ�sÓð ',ñà3ðð 7ðð ð	ð
 !$óò,!mðF]¨¨c©ó ]ðXØNS×N^ÑN^ðXà	×	Ñ	óXð °%×2CÑ2Cð È×HYÑHYô r   r  c                   ól   — e Zd ZdZ ej
                  «       Ze	 ddddddede	de
f
d„«       Zed	„ «       Zy
)ÚAssistantVocabTranslatorCachez§
    Cache for `AssistantToTargetTranslator` instances. The instances are computed at
    pre-processing time, and this cache allows us to avoid recomputing them.
    r¦   r   r§   r  r	  r   c                 óÒ   — | j                   j                  |«      }|€#t        j                  «       }|| j                   |<   |j                  |«      }|€t	        ||||«      }|||<   |S r©   )Ú_cacherR   ÚweakrefÚWeakKeyDictionaryr  )Úclsr¦   r§   r  r	  Úassistant_dictÚmappings          r   Úget_translatorz,AssistantVocabTranslatorCache.get_translatorè  sx   € ð Ÿ™Ÿ™Ð(8Ó9ˆØÐ!Ü$×6Ñ6Ó8ˆNØ+9ˆC�J‰JÐ'Ñ(à ×$Ñ$Ð%8Ó9ˆØˆ?Ü1Ø Ð"5Ð7HÐJ`óˆGð 3:ˆNÐ.Ñ/àˆr   c                 óî   — | j                   D �cg c]  }|�Œ|‘Œ	 }}|D ]  }| j                   |= Œ | j                   j                  «       D ]  }|D �cg c]  }|�Œ|‘Œ	 }}|D ]  }||= Œ Œ  yc c}w c c}w )z²
        Clean up dead references in the cache.
        This removes entries where either the target_tokenizer or assistant_tokenizer
        has been garbage collected.
        N)rC  rÀ   )rF  ra   Ú	dead_keysrG  s       r   Úcleanupz%AssistantVocabTranslatorCache.cleanupþ  s�   € ð %(§J¡JÖ>˜S°#±+’SÐ>ˆ	Ð>Øò 	 ˆCØ—
‘
˜3‘ð	 ð "Ÿj™j×/Ñ/Ó1ò 	(ˆNØ(6ÖF ¸#¹+šÐFˆIÐFØ ò (�Ø" 3Ñ'ñ(ñ	(ùò ?ùò Gs   �A-—A-ÁA2ÁA2Nr;  )r&   r'   r(   r)   rD  rE  rC  Úclassmethodr-   r?  r  rI  rL  r.   r   r   rA  rA  à  st   „ ñð
 'ˆW×&Ñ&Ó(€Fàð ',ñà3ðð 7ðð ð	ð
 !$ðð 
%òó ðð* ñ(ó ñ(r   rA  c                   ó>  ‡ — e Zd ZdZ	 	 ddej
                  dddddddd	d
ededeej                     ddfˆ fd„Z
dej
                  deej
                  eej                     f   fd„Zddej
                  dedefˆ fd„Zdej
                  dej
                  fd„Zˆ xZS )Ú%UniversalSpeculativeDecodingGeneratorzó
    `CandidateGenerator` class to be used for Universal Speculative Decoding (USD): speculative decoding with different tokenizers
    for the assistant and main models. This class generates candidates through the use of a smaller model.
    r   r1   r   r¦   r   r§   r2   r   r3   Úatm_translatorr4   r5   r   c
           
      ó\   •— || _         t        ‰
| �	  ||||||||	«       d| _        d | _        y r  )Ú_atm_translatorrª   re   Ú_target_seq_len_with_candidatesÚ_prev_assistant_ids)r   r   r1   r¦   r§   r2   r3   rP  r4   r5   r   s             €r   re   z.UniversalSpeculativeDecodingGenerator.__init__  sD   ø€ ð  .ˆÔÜ‰ÑØØØØØØØØô		
ð 56ˆÔ,Ø?CˆÕ r   r   c                 óf  — |j                  | j                  j                  «      }| j                  |«      \  }}| j	                  |«      \  }}|dk(  r|dfS | j                  ||¬«       | j                  |||«      }d|d   _        d|d   _        | j                  j                  �| j                  j                  |d<   | j                  |«      \  | _        }| j                  j                  ||| j                  «      }	|	j                  d   | _        | j                  j!                  |«      }
|	|
fS )zk
        Simplified version of get_candidates that uses the translator cache for token conversion.
        r   N©r‹   Tr2   r5   r†   )r?   r1   r<   ré   rh   ri   rj   rW   rV   rR  r  rk   rT  r1  rT   rS  r:  )r   r   r.  rì   r‹   r[   rl   rm   Úassistant_candidate_logitsÚtarget_candidate_idsÚtarget_candidate_logitss              r   r   z4UniversalSpeculativeDecodingGenerator.get_candidates3  sO  € ð %Ÿ<™<¨×(<Ñ(<×(CÑ(CÓDÐØ04×0QÑ0QÐRbÓ0cÑ-ÐÐ-Ø)-×)CÑ)CÐDTÓ)UÑ&ˆ˜à˜QÒØ˜d�?Ð"à×#Ñ#Ð$7ÐJZÐ#Ô[Ø×7Ñ7Ð8KÈ^Ð]kÓlˆð >BˆÐ+Ñ,Ô:ØGKˆÐ+Ñ,ÔDð ×Ñ×1Ñ1Ð=Ø26×2FÑ2F×2XÑ2XˆOÐ.Ñ/Ø?C×?XÑ?XÐYhÓ?iÑ<ˆÔ Ð"<ð  $×3Ñ3×BÑBØÐ!1°4×3KÑ3Kó 
Ðð 0D×/IÑ/IÈ"Ñ/MˆÔ,Ø"&×"6Ñ"6×"HÑ"HÐIcÓ"dÐà#Ð%<Ð<Ð<r   rì   r‹   c                 óÌ   •— | j                   €Gt        | j                  |j                  d   | j                  j
                  j                  «      | _        t        ‰| �!  ||¬«      S )Nr†   rV  )	rT  rŽ   rP   rT   r1   rK   rL   rª   ri   )r   rì   r‹   r   s      €r   ri   z<UniversalSpeculativeDecodingGenerator._update_past_and_masksS  sb   ø€ Ø×#Ñ#Ð+ô %<Ø×%Ñ%Ð':×'@Ñ'@ÀÑ'DÀd×FZÑFZ×FaÑFa×FtÑFtó%ˆDÔ!ô ‰wÑ-Ð.AÐTdÐ-ÓeÐer   r.  c                 ó&  — |j                   d   }| j                  dk(  r|}nd}|dd…| d…f   }d}| j                  dkD  r6| j                  j                  j	                  |d   j                  «       «      }|€Y| j                  j                  |dd¬«      }| j                  |dd¬	«      d
   j                  | j                  j                  «      }n-t        j                  |gg| j                  j                  ¬«      }| j                  €|}nU| j                  dz   |z
  }|dkD  r| j                  dd…d| …f   | _        t        j                  | j                  |gd¬«      }|j                  t        j                   ¬«      }|t#        |d   «      fS )zM
        Simplified token conversion that only processes new tokens.
        r†   r   r   NTrÛ   FrÞ   rß   r   r4  r”   r  )rT   rS  rR  r  rR   rÍ   r¦   râ   r§   r?   r1   r<   r*   rË   rT  r—   rU   rv   )	r   r.  Útarget_seq_lenÚnew_token_countÚtarget_new_idsÚassistant_new_idsÚtarget_new_textrì   Útokens_to_removes	            r   ré   zBUniversalSpeculativeDecodingGenerator._prepare_assistant_input_ids\  s³  € ð
 *×/Ñ/°Ñ3ˆØ×/Ñ/°1Ò4Ø,‰OàˆOØ)ª!¨oÐ-=Ñ->Ð*>Ñ?ˆð !ÐØ×/Ñ/°!Ò3à $× 4Ñ 4× RÑ R× VÑ VÐWeÐfgÑWh×WmÑWmÓWoÓ pÐØÐ$Ø"×3Ñ3×@Ñ@Ø°DÐW[ð Aó ˆOð !%× 8Ñ 8Ø°EÈ$ð !9ó !àñ!ç™2˜d×2Ñ2×9Ñ9Ó:ñ ô !&§¡Ð/@Ð.AÐ-BÈ4×K_ÑK_×KfÑKfÔ gÐð ×#Ñ#Ð+Ø"3Ñà#×CÑCÀaÑGÈ.ÑXÐà !Ò#Ø+/×+CÑ+CÂAÐGYÐIYÐHYÐGYÐDYÑ+Z�Ô(Ü"'§)¡)¨T×-EÑ-EÐGXÐ,YÐ_aÔ"bÐØ1×4Ñ4¼5¿:¹:Ð4ÓFÐà"¤CÐ(9¸!Ñ(<Ó$=Ð=Ð=r   r¢   )r   )r&   r'   r(   r)   r*   r+   r   r  r   rE   re   r   r,   r   r-   r£   ri   ré   r  r  s   @r   rO  rO    s  ø„ ñð 15Ø26ñDà×#Ñ#ðDð +ðDð 4ð	Dð
 7ðDð .ðDð ðDð 4ðDð   §¡Ñ-ðDð 0õDð8=¨×(8Ñ(8ð =¸UÀ5×CSÑCSÐU]Ð^c×^oÑ^oÑUpÐCpÑ=qó =ñ@f¸%×:JÑ:Jð fÐ^að fÐjnõ fð&>¸U×=MÑ=Mð &>ÐRW×RbÑRb÷ &>r   rO  c            	       óê   — e Zd ZdZ	 	 	 	 ddeej                     dedee   defd„Zdej                  d	e
ej                  eej                     f   fd
„Zdej                  dej                  defd„Zy)ÚPromptLookupCandidateGeneratora
  
    `CandidateGenerator` class to be used for prompt lookup generation. This class generates candidates by looking up
    likely continuations in the provided prompt (input_ids) itself.
    Read the following blog post for more information: https://github.com/apoorvumang/prompt-lookup-decoding

    Args:
        max_matching_ngram_size (`int`):
            The maximum ngram size to be considered for matching in the prompt
        num_output_tokens (`int`):
            The number of tokens to be output as candidate tokens.
        max_length (`int`):
            The number of total maximum tokens that can be generated. For decoder-only models that includes the prompt length.
            Defaults to 20, which is the max length used as default in generation config.
    NrB   Únum_output_tokensÚmax_matching_ngram_sizerˆ   c                 ó–   — || _         |r|nd| _        || _        || _        | j                  dk  s| j                   dk  rt	        d«      ‚y )Nr   r   z4Invalid max_matching_ngram_size or num_output_tokens)rd  re  rˆ   rB   r\   )r   rB   rd  re  rˆ   s        r   re   z'PromptLookupCandidateGenerator.__init__•  sV   € ð "3ˆÔÙBYÑ'>Ð_`ˆÔ$Ø$ˆŒØ(ˆÔà×'Ñ'¨1Ò,°×0FÑ0FÈ!Ò0KÜÐSÓTÐTð 1Lr   r   r   c                 óú  — |j                  d«      }| j                  |dz   k(  r|dfS d}d}t        t        | j                  |dz
  «      dd«      D ]è  }|j                  d|d¬«      }|d| d…f   }||k(  j                  d¬«      }|j                  d	¬
«      d   }	|	D ]•  }
|
|z   }|| j                  z   }t        ||| j                  «      }||k  sŒ4|d||…f   }d	}t        || j                  «      }t        j                  |«      }|j                  «       dkD  r|d   j                  «       }|d| } n |sŒè n |�t        |«      dk(  r|dfS |j                  d«      }t        j                   ||fd¬«      }|dfS )a¤  
        Fetches the candidates to be tried for the current input.

        Args:
            input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
                Indices of input sequence tokens in the vocabulary. [What are input IDs?](../glossary#input-ids)

        Return:
            `torch.LongTensor` of shape `(num_candidates, candidate_length)`: The candidate sequences to be tried.
        r   NFr   r†   )Ú	dimensionÚsizeÚstepr   r”   T)Úas_tuple)ri  rˆ   rš   r‡   re  ÚunfoldÚallr¾   rd  r   rB   r*   ÚnumelrÍ   rv   r-  r—   )r   r   Úinput_lengthÚ
chosen_idsÚmatch_foundÚ
ngram_sizeÚwindowsÚngram_tensorr`   Úmatch_indicesr¸   r¹   Úend_idxÚmaskÚmatch_indices_eosÚfirst_eos_indexÚcandidate_input_idss                    r   r   z-PromptLookupCandidateGenerator.get_candidates¤  sÑ  € ð !—~‘~ aÓ(ˆð �?‰?˜l¨QÑ.Ò.Ø˜d�?Ð"àˆ
ØˆÜ¤ D×$@Ñ$@À,ÐQRÑBRÓ SÐUVÐXZÓ[ò !	ˆJà×&Ñ&°¸È!Ð&ÓLˆGð % Q¨¨© _Ñ5ˆLð  ,Ñ.×3Ñ3¸Ð3Ó:ˆGð $ŸO™O°T˜OÓ:¸1Ñ=ˆMð %ò �Ø *Ñ,�	Ø# d×&<Ñ&<Ñ<�Ü˜g |°T·_±_ÓE�à˜wÓ&Ø!*¨1¨i¸Ð.?Ð+?Ñ!@�JØ"&�Kô
 -¨Z¸×9JÑ9JÓK�DÜ(-¯©°dÓ(;Ð%Ø(×.Ñ.Ó0°1Ò4Ø*;¸AÑ*>×*CÑ*CÓ*E˜Ø%/Ð0@°Ð%A˜
Ùð#ò$ ÙðC!	ðF Ð¤ Z£°AÒ!5à˜d�?Ð"ð  ×)Ñ)¨!Ó,ˆ
Ü#Ÿi™i¨°JÐ(?ÀQÔGÐà" DÐ(Ð(r   r    r!   c                  ó   — y)r#   Nr.   r$   s       r   r%   z8PromptLookupCandidateGenerator.update_candidate_strategyä  s   € ð 	r   )Né
   Né   )r&   r'   r(   r)   r   r*   rE   r-   re   r+   r   r,   r   r%   r.   r   r   rc  rc  …  s°   „ ñð" 04Ø!#Ø15ØñUà˜uŸ|™|Ñ,ðUð ðUð "*¨#¡ð	Uð
 óUð>)¨×(8Ñ(8ð >)¸UÀ5×CSÑCSÐU]Ð^c×^oÑ^oÑUpÐCpÑ=qó >)ð@°5×3CÑ3Cð ÈU×M^ÑM^ð Ðmpô r   rc  c                   óÐ   ‡ — e Zd ZdZ	 	 ddej
                  dddddedeej                     d	d
fˆ fd„Z	dej
                  de
ej
                  eej                     f   fˆ fd„Zˆ xZS )ÚEarlyExitCandidateGeneratora™  
    `CandidateGenerator` class to be used for assisted generation and speculative decoding. This class generates
    candidates through the use of **the model itself**, exiting early. Can only be used with models that support early
    exit, e.g., `facebook/layerskip-llama3.2-1B`.

    Args:
        input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
            Indices of input sequence tokens in the vocabulary. [What are input IDs?](../glossary#input-ids)
        assistant_model (`PreTrainedModel`):
            The original model. This model must support early exit (i.e. is trained to compute logits in earlier
            layers).
        generation_config (`~generation.GenerationConfig`, *optional*):
            The generation configuration to be used as base parametrization for the generation call.
        logits_processor (`LogitsProcessorList`):
            An instance of [`LogitsProcessorList`]. List of instances of class derived from [`LogitsProcessor`]
            used to modify the prediction scores of the language modeling head applied at each generation step.
        model_kwargs (`Dict`):
            The keyword arguments that will be passed to the main model, and are used as base inputs for the assistant
            model as well.
        inputs_tensor (`torch.Tensor`, *optional*):
            The model input tensor. In encoder-decoder models, this is the encoder input.
    r   r1   r   r2   r   r3   r4   r5   r   c                 óˆ   •— t         ‰| �  ||||||¬«       | j                  j                  | _        d | j                  _        y )N)r   r1   r2   r3   r4   r5   )rª   re   r2   Úassistant_early_exit)r   r   r1   r2   r3   r4   r5   r   s          €r   re   z$EarlyExitCandidateGenerator.__init__  sM   ø€ ô 	‰ÑØØ+Ø/Ø%Ø'Ø-ð 	ô 	
ð %)×$:Ñ$:×$OÑ$OˆÔ!Ø6:ˆ×ÑÕ3r   r   c                 ó  •— t        | j                  | j                  j                  «      }|j                  j                  }| j
                  |j                  _        t        ‰| �  |«      \  }}||j                  _        ||fS r©   )Úgetattrr1   Úbase_model_prefixrK   Únum_hidden_layersr�  rª   r   )r   r   Ú
base_modelÚoriginal_num_hidden_layersrn   ro   r   s         €r   r   z*EarlyExitCandidateGenerator.get_candidates#  sv   ø€ ä˜T×1Ñ1°4×3GÑ3G×3YÑ3YÓZˆ
Ø%/×%6Ñ%6×%HÑ%HÐ"Ø.2×.GÑ.Gˆ
×ÑÔ+Ü*/©'Ñ*@ÀÓ*KÑ'ˆÐ'Ø.Hˆ
×ÑÔ+ØÐ.Ð.Ð.r   r¢   )r&   r'   r(   r)   r*   r+   r   r   rE   re   r   r,   r   r  r  s   @r   r  r  õ  s™   ø„ ñð: 15Ø26ñ;à×#Ñ#ð;ð +ð;ð .ð	;ð
 ð;ð   §¡Ñ-ð;ð 0õ;ð,/¨×(8Ñ(8ð /¸UÀ5×CSÑCSÐU]Ð^c×^oÑ^oÑUpÐCpÑ=q÷ /ñ /r   r  c           
      ó0  — g }| j                   j                  rqt        t        |«      «      D ]M  }|j	                  ||   d   dd…dd…d|…dd…f   ||   d   dd…dd…d|…dd…f   ||   d   ||   d   f«       ŒO t        |«      }|S d| j                  j                  j                  «       v s?| j                   j                  � d| j                   j                  d   j                  «       v rw| j                   j                  r/t        t        |«      «      D ]  }||   dd…d|…dd…f   ||<   Œ |S t        t        |«      «      D ]  }||   dd…dd…d|…dd…f   ||<   Œ |S t        |t        «      r|j                  |«       |S |�‹t        t        |«      «      D ]i  }||   g g fk7  r>|j	                  ||   d   dd…dd…d|…dd…f   ||   d   dd…dd…d|…dd…f   f«       ŒK|j	                  ||   d   ||   d   f«       Œk t        |«      }|S )z9Crops the past key values up to a certain maximum length.r   Nr   r   rt   Ú
gptbigcode)rK   rL   rš   rv   ry   r²   r   r&   ÚlowerÚarchitecturesÚmulti_queryrD   r   Úcrop)Úmodelr8   rˆ   Únew_pastr¸   s        r   r�   r�   -  sh  € à€HØ‡|�|×&Ò&Üœ˜_Ó-Ó.ò 	ˆCØ�O‰Oà# CÑ(¨Ñ+ªAªq°+°:°+ºqÐ,@ÑAØ# CÑ(¨Ñ+ªAªq°+°:°+ºqÐ,@ÑAØ# CÑ(¨Ñ+Ø# CÑ(¨Ñ+ð	õð	ô   ›/ˆð2 Ðð/ 
˜Ÿ™×1Ñ1×7Ñ7Ó9Ñ	9Ø�‰×"Ñ"Ð.°<À5Ç<Á<×C]ÑC]Ð^_ÑC`×CfÑCfÓChÑ3hà�<‰<×#Ò#ÜœS Ó1Ó2ò O�Ø'6°sÑ';ºA¸{À
¸{ÊAÐ<MÑ'N� Ò$ðOð& Ðô! œS Ó1Ó2ò R�Ø'6°sÑ';ºAºqÀ+À:À+ÊqÐ<PÑ'Q� Ò$ðRð  Ðô 
�O¤\Ô	2Ø×Ñ˜ZÔ(ð Ðð 
Ð	$Üœ˜_Ó-Ó.ò 		TˆCØ˜sÑ#¨¨B xÒ/Ø—‘à'¨Ñ,¨QÑ/²²1°k°z°kÂ1Ð0DÑEØ'¨Ñ,¨QÑ/²²1°k°z°kÂ1Ð0DÑEðõð —‘ °Ñ!5°aÑ!8¸/È#Ñ:NÈqÑ:QÐ RÕSð		Tô   ›/ˆØÐr   r3   Ú
new_lengthrL   r   c                 óv  — |rdnd}|| vr| S | |   }||j                   d   z
  }|dk  r|dd…d|…f   | |<   n?|dkD  r:t        j                  ||j                  |j                   d   |f«      gd¬«      | |<   d| v r`| d   }|dk  r|dd…d|…f   | d<   | S |dkD  r?|dd…dd…dd…dd…f   j	                  d|dd«      }t        j                  ||gd¬«      | d<   | S d	| v rZ| d	   }|dk  r|dd…d|…f   | d	<   | S |dkD  r;|dd…dd…dd…f   j	                  d|d«      }t        j                  ||gd¬«      | d	<   | S )
zNExpands or crops the model's mask for decoding purposes, to the defined lengthr;   r>   r   r   Nr†   r”   Úcross_attention_maskÚimage_attention_mask)rT   r*   r—   Únew_onesÚrepeat)r3   r�  rL   Úmask_keyrw  Úmask_length_diffÚ
cross_maskÚnew_masks           r   rŽ   rŽ   V  s»  € ñ ,>Ñ'ÐCS€HØ�|Ñ#ØÐà˜Ñ!€DØ! D§J¡J¨q¡MÑ1Ðà˜!ÒØ!%¢aÐ):Ð*:Ð):Ð&:Ñ!;ˆ�XÒØ	˜AÒ	Ü!&§¡¨D°$·-±-ÀÇÁÈAÁÐP`Ð@aÓ2bÐ+cÐikÔ!lˆ�XÑð  Ñ-à!Ð"8Ñ9ˆ
Ø˜aÒØ3=ºaÐARÐBRÐARÐ>RÑ3SˆLÐ/Ñ0ð Ðð  Ò!Ø!¢! R¡Sª!ªQ ,Ñ/×6Ñ6°qÐ:JÈAÈqÓQˆHÜ38·9±9¸jÈ(Ð=SÐYZÔ3[ˆLÐ/Ñ0ð Ðð 
  <Ñ	/à!Ð"8Ñ9ˆ
Ø˜aÒØ3=ºaÐARÐBRÐARÐ>RÑ3SˆLÐ/Ñ0ð
 Ðð	  Ò!Ø!¢! R¡Sª! )Ñ,×3Ñ3°AÐ7GÈÓKˆHÜ38·9±9¸jÈ(Ð=SÐYZÔ3[ˆLÐ/Ñ0àÐr   c                 ó
  — d| vs| d   €| S | d   }|dd…df   j                  d«      }||j                  d   z
  }|dk  r|dd…d|…f   }| S |dkD  r1|j                  d|«      }t        j                  | d   |gd¬«      | d<   | S )zXExpands or crops the model's token_type_ids for decoding purposes, to the defined lengthÚtoken_type_idsNr†   r   r   r”   )r-  rT   r•  r*   r—   )r3   r�  r›  Úfinal_token_typeÚtype_length_diffÚtoken_type_copiess         r   r�   r�   z  sÃ   € à˜|Ñ+¨|Ð<LÑ/MÐ/UØÐà!Ð"2Ñ3€NØ%¢a¨ eÑ,×6Ñ6°rÓ:ÐØ! N×$8Ñ$8¸Ñ$;Ñ;Ðà˜!ÒØ'ªÐ+<Ð,<Ð+<Ð(<Ñ=ˆð Ðð 
˜AÒ	Ø,×3Ñ3°AÐ7GÓHÐÜ).¯©°LÐAQÑ4RÐTeÐ3fÐlnÔ)oˆÐ%Ñ&ØÐr   )+rG   rD  Útypingr   r   r   r   r   Únumpyr{   r*   Úutilsr	   Úsklearn.metricsr
   Úcache_utilsr   Úpytorch_utilsr   Úlogits_processr   r   r   Úmodeling_utilsr   Útokenization_utils_baser   Úconfiguration_utilsr   r   r0   r¥   r  rA  rO  rc  r  r�   r?  r-   r£   rŽ   r�   r.   r   r   ú<module>r©     s  ðó  Û ß <Õ <ã Û å (ñ ÔÝ)å &Ý -ß hÑ hñ Ý0ÝAÝ5÷$
ñ $
ôN{/Ð!3ô {/ô|WÐ4Nô W÷tvñ v÷r.(ñ .(ôbq>Ð,Yô q>ôhmÐ%7ô mô`5/Ð"<ô 5/òp&ðR!¨$¨s°C¨x©.ð !Àcð !Ð_cð !ÐhlÐmpÐruÐmuÑhvó !ðH¨$¨s°C¨x©.ð Àcð ÈdÐSVÐX[ÐS[Énô r   