Ë
    S^(hÃµ ã                   ó¬  — d dl Z d dlZd dlZd dlZd dlmZ d dlmZm	Z	m
Z
mZmZmZmZ d dlZd dlmZ d dlmZ ddlmZ ddlmZmZmZmZ  e«       rd d	lmZ  ej<                  e«      Z  G d
„ d«      Z!e G d„ d«      «       Z"e G d„ de"«      «       Z#e G d„ de"«      «       Z$ G d„ de!«      Z%de%fd„Z&de%fd„Z'dej,                  jP                  jR                  fd„Z*d„ Z+ ed«      riej,                  jP                  jY                  e%e&e*e%jZ                  › de%j>                  › �e'¬«       ej\                  jP                  j_                  e%e+«        G d„ de%«      Z0 G d„ d e%«      Z1 G d!„ d"e1«      Z2 G d#„ d$e1«      Z3 G d%„ d&e!«      Z4 G d'„ d(e!«      Z5 G d)„ d*e5«      Z6 G d+„ d,e!«      Z7 G d-„ d.e!«      Z8 G d/„ d0e!«      Z9 G d1„ d2«      Z: G d3„ d4e5«      Z;y)5é    N)Ú	dataclass)ÚAnyÚDictÚIterableÚListÚOptionalÚTupleÚUnion)Úversion)Ú"is_torch_greater_or_equal_than_2_6é   )ÚPretrainedConfig)Úis_hqq_availableÚis_optimum_quanto_availableÚis_torch_greater_or_equalÚlogging)Ú	Quantizerc                   ó&  ‡ — e Zd ZdZdZˆ fd„Z	 ddej                  dej                  dede	e
eef      deej                  ej                  f   f
d	„Zdde	e   defd
„Zde	e   fd„Zddede	e   defd„Zdej$                  fd„Zed„ «       Zˆ xZS )ÚCachezf
    Base, abstract class for all caches. The actual data structure is specific to each subclass.
    Fc                 ó"   •— t         ‰| �  «        y ©N)ÚsuperÚ__init__©ÚselfÚ	__class__s    €úV/var/www/skyplay_api_hub/venv/lib/python3.12/site-packages/transformers/cache_utils.pyr   zCache.__init__   s   ø€ Ü‰ÑÕó    Ú
key_statesÚvalue_statesÚ	layer_idxÚcache_kwargsÚreturnc                 ó   — t        d«      ‚)a·  
        Updates the cache with the new `key_states` and `value_states` for the layer `layer_idx`.

        Parameters:
            key_states (`torch.Tensor`):
                The new key states to cache.
            value_states (`torch.Tensor`):
                The new value states to cache.
            layer_idx (`int`):
                The index of the layer to cache the states for.
            cache_kwargs (`Dict[str, Any]`, `optional`):
                Additional arguments for the cache subclass. These are specific to each subclass and allow new types of
                cache to be created.

        Return:
            A tuple containing the updated key and value states.
        z.Make sure to implement `update` in a subclass.©ÚNotImplementedError)r   r   r    r!   r"   s        r   ÚupdatezCache.update!   s   € ô0 "Ð"RÓSÐSr   c                 ó   — t        d«      ‚)úYReturns the sequence length of the cached states. A layer index can be optionally passed.z6Make sure to implement `get_seq_length` in a subclass.r%   ©r   r!   s     r   Úget_seq_lengthzCache.get_seq_length;   s   € ô "Ð"ZÓ[Ð[r   c                 ó   — t        d«      ‚)zKReturns the maximum sequence length (i.e. max capacity) of the cache objectz;Make sure to implement `get_max_cache_shape` in a subclass.r%   ©r   s    r   Úget_max_cache_shapezCache.get_max_cache_shape@   s   € ä!Ð"_Ó`Ð`r   Únew_seq_lengthc                 óf   — | j                  «       }| j                  |«      }|�||z   |kD  r||z
  S |S )zTGiven the sequence length of the new inputs, returns the usable length of the cache.)r.   r+   )r   r/   r!   Ú
max_lengthÚprevious_seq_lengths        r   Úget_usable_lengthzCache.get_usable_lengthD   sG   € ð
 ×-Ñ-Ó/ˆ
Ø"×1Ñ1°)Ó<ÐØÐ!Ð&9¸NÑ&JÈZÒ&WØ Ñ.Ð.Ø"Ð"r   Úbeam_idxc                 ó  — t        t        | j                  «      «      D ]å  }| j                  |   j                  «       rT| j                  |   j                  }| j                  |   j                  d|j                  |«      «      | j                  |<   | j                  |   j                  «       sŒ’| j                  |   j                  }| j                  |   j                  d|j                  |«      «      | j                  |<   Œç y)úDReorders the cache for beam search, given the selected beam indices.r   N)ÚrangeÚlenÚ	key_cacheÚnumelÚdeviceÚindex_selectÚtoÚvalue_cache)r   r4   r!   r;   s       r   Úreorder_cachezCache.reorder_cacheO   sß   € äœs 4§>¡>Ó2Ó3ò 	oˆIØ�~‰~˜iÑ(×.Ñ.Ô0ØŸ™¨	Ñ2×9Ñ9�Ø,0¯N©N¸9Ñ,E×,RÑ,RÐSTÐV^×VaÑVaÐbhÓViÓ,j�—‘˜yÑ)Ø×Ñ 	Ñ*×0Ñ0Õ2Ø×)Ñ)¨)Ñ4×;Ñ;�Ø.2×.>Ñ.>¸yÑ.I×.VÑ.VÐWXÐZb×ZeÑZeÐflÓZmÓ.n�× Ñ  Ò+ñ	or   c                 ó^   — t         j                  d«       t        | d«      r| j                  S y )NzuThe `seen_tokens` attribute is deprecated and will be removed in v4.41. Use the `cache_position` model input instead.Ú_seen_tokens)ÚloggerÚwarning_onceÚhasattrrA   r-   s    r   Úseen_tokenszCache.seen_tokensY   s0   € ä×Ñð#ô	
ô �4˜Ô(Ø×$Ñ$Ð$àr   r   ©r   )Ú__name__Ú
__module__Ú__qualname__Ú__doc__Úis_compileabler   ÚtorchÚTensorÚintr   r   Ústrr   r	   r'   r+   r.   r3   Ú
LongTensorr?   ÚpropertyrE   Ú__classcell__©r   s   @r   r   r      sî   ø„ ñð €Nôð 26ñTà—L‘LðTð —l‘lðTð ð	Tð
 ˜t C¨ H™~Ñ.ðTð 
ˆu�|‰|˜UŸ\™\Ð)Ñ	*óTñ4\¨°©ð \¸có \ð
a X¨c¡]ó añ	#°ð 	#ÀÈÁð 	#ÐVYó 	#ðo e×&6Ñ&6ó oð ñó ôr   r   c                   ó„   — e Zd ZU dZded<   ed„ «       Zdeee	j                  f   fd„Zdeeef   fd„Zd	„ Zd
„ Zd„ Zd„ Zy)ÚCacheConfigz&
    Base class for cache configs
    NÚcache_implementationc                 óÒ   —  | di |¤Ž}g }|j                  «       D ]0  \  }}t        ||«      sŒt        |||«       |j                  |«       Œ2 |D ]  }|j	                  |d«       Œ |S )ar  
        Constructs a CacheConfig instance from a dictionary of parameters.
        Args:
            config_dict (Dict[str, Any]): Dictionary containing configuration parameters.
            **kwargs: Additional keyword arguments to override dictionary values.

        Returns:
            CacheConfig: Instance of CacheConfig constructed from the dictionary.
        N© )ÚitemsrD   ÚsetattrÚappendÚpop)ÚclsÚconfig_dictÚkwargsÚconfigÚ	to_removeÚkeyÚvalues          r   Ú	from_dictzCacheConfig.from_dictm   sw   € ñ Ñ#�{Ñ#ˆØˆ	Ø Ÿ,™,›.ò 	&‰JˆC�Ü�v˜sÕ#Ü˜  UÔ+Ø× Ñ  Õ%ð	&ð ò 	"ˆCØ�J‰J�s˜DÕ!ð	"àˆr   Újson_file_pathc                 óÂ   — t        |dd¬«      5 }| j                  «       }t        j                  |dd¬«      dz   }|j	                  |«       ddd«       y# 1 sw Y   yxY w)	aÃ  
        Save this instance to a JSON file.

        Args:
            json_file_path (`str` or `os.PathLike`):
                Path to the JSON file in which this configuration instance's parameters will be saved.
            use_diff (`bool`, *optional*, defaults to `True`):
                If set to `True`, only the difference between the config instance and the default
                `QuantizationConfig()` is serialized to JSON file.
        Úwzutf-8)Úencodingé   T)ÚindentÚ	sort_keysú
N)ÚopenÚto_dictÚjsonÚdumpsÚwrite)r   re   Úwriterr^   Újson_strings        r   Úto_json_filezCacheConfig.to_json_fileƒ   sU   € ô �. #°Ô8ð 	&¸FØŸ,™,›.ˆKÜŸ*™* [¸ÀdÔKÈdÑRˆKà�L‰L˜Ô%÷		&÷ 	&ñ 	&ús   �=AÁAr#   c                 ó@   — t        j                  | j                  «      S )z´
        Serializes this instance to a Python dictionary. Returns:
            `Dict[str, Any]`: Dictionary of all the attributes that make up this configuration instance.
        )ÚcopyÚdeepcopyÚ__dict__r-   s    r   rn   zCacheConfig.to_dict•   s   € ô
 �}‰}˜TŸ]™]Ó+Ð+r   c              #   ó‚   K  — t        j                  | j                  «      j                  «       D ]  \  }}||f–— Œ y­w)zTallows `dict(obj)` for situations where obj may be a dict or QuantizationConfigMixinN)rv   rw   rx   rY   )r   Úattrrc   s      r   Ú__iter__zCacheConfig.__iter__�   s9   è ø€ äŸ=™=¨¯©Ó7×=Ñ=Ó?ò 	‰KˆD�%Ø˜�+Óñ	ùs   ‚=?c                 óT   — | j                   j                  › d| j                  «       › �S )Nú )r   rG   Úto_json_stringr-   s    r   Ú__repr__zCacheConfig.__repr__£   s(   € Ø—.‘.×)Ñ)Ð*¨!¨D×,?Ñ,?Ó,AÐ+BÐCÐCr   c                 óJ   — t        j                  | j                  d¬«      dz   S )z§
        Serializes this instance to a JSON formatted string.
        Returns:
            str: JSON formatted string representing the configuration instance.
        ri   )rj   rl   )ro   rp   rx   r-   s    r   r~   zCacheConfig.to_json_string¦   s   € ô �z‰z˜$Ÿ-™-°Ô2°TÑ9Ð9r   c                 óì   — g }|j                  «       D ]0  \  }}t        | |«      sŒt        | ||«       |j                  |«       Œ2 |j                  «       D ��ci c]  \  }}||vsŒ||“Œ }}}|S c c}}w )a¯  
        Updates attributes of this class instance with attributes from `kwargs` if they match existing attributes,
        returning all the unused kwargs.

        Args:
            kwargs (`Dict[str, Any]`):
                Dictionary of attributes to tentatively update this class.

        Returns:
            `Dict[str, Any]`: Dictionary containing all the key-value pairs that were not used to update the instance.
        )rY   rD   rZ   r[   )r   r_   ra   rb   rc   Úunused_kwargss         r   r'   zCacheConfig.update¯   s}   € ð ˆ	Ø Ÿ,™,›.ò 	&‰JˆC�Ü�t˜SÕ!Ü˜˜c 5Ô)Ø× Ñ  Õ%ð	&ð 7=·l±l³n×]©
¨¨UÈÐS\ÒH\˜˜e™Ð]ˆÑ]ØÐùó ^s   ÁA0Á&A0)rG   rH   rI   rJ   Ú__annotations__Úclassmethodrd   r
   rO   ÚosÚPathLikert   r   r   rn   r{   r   r~   r'   rX   r   r   rU   rU   e   sd   … ñð Óàñó ðð*&¨5°°b·k±kÐ1AÑ+Bó &ð$,˜˜c 3˜h™ó ,òòDò:ór   rU   c                   ó¢   — e Zd ZdZddddddej
                  dfded	ee   d
ee   dee   dee   dee   deej                     dee   fd„Z
d„ Zy)ÚQuantizedCacheConfiga  
    Configuration class for quantized cache settings.

    Attributes:
        backend (`str`, *optional*, defaults to `"quanto"`):
            Backend to use when performing quantization, Can be one of [`quanto`, `HQQ`]
        nbits (`Optional[int]`, *optional*, defaults to 4):
            Number of bits, can be 2 or 4 for the `quanto` backend and one of [1, 2, 3, 4, 8] for the `HQQ` backend. Defaults to 2.
        axis_key (`int`, *optional*, defaults to 0):
            Axis over which to perform grouping for the key tensors. Can be [0, -1] for `quanto` backend and [0, 1] for `HQQ` backend.
        axis_value (`int`, *optional*, defaults to 0):
            Axis over which to perform grouping for the value tensors. Can be [0, -1] for `quanto` backend and [0, 1] for `HQQ` backend.
        q_group_size (`Optional[int]`, *optional*, defaults to 64):
            Size of the quantization group, should be a divisor of the model's hidden dimension.
            Defaults to 64.
        residual_length (`Optional[int]`, *optional*, defaults to 128):
            Length of the residual cache which will always be stored in original precision.
            Defaults to 128.
        compute_dtype (`torch.dtype`, *optional*, defaults to `torch.float16`):
            The default dtype used for computations in the model. Keys and Values will be cast to this dtype after dequantization.
        device (`str`, *optional*, defaults to `"cpu"`):
            Device on which to perform computations, should be same as the model's device.
    Úquantoé   r   é@   é€   ÚcpuÚbackendÚnbitsÚaxis_keyÚ
axis_valueÚq_group_sizeÚresidual_lengthÚcompute_dtyper;   c	                 ót   — || _         || _        || _        || _        || _        || _        || _        || _        y r   )rŽ   r�   r�   r‘   r’   r“   r”   r;   )	r   rŽ   r�   r�   r‘   r’   r“   r”   r;   s	            r   r   zQuantizedCacheConfig.__init__à   s?   € ð ˆŒØˆŒ
Ø ˆŒØ$ˆŒØ(ˆÔØ.ˆÔØ*ˆÔØˆ�r   c                 ó  — d}| j                   dvr't        |j                  dd| j                   ¬«      «      ‚| j                  dk  r't        |j                  dd| j                  ¬«      «      ‚| j                  dk  r't        |j                  d	d| j                  ¬«      «      ‚| j
                  d
vr't        |j                  dd| j
                  ¬«      «      ‚| j                  d
vr't        |j                  dd| j                  ¬«      «      ‚y)ú-Validates if the arguments passed are correctúvSome of the keys in `cache_config` are defined incorrectly. `{key}` should be {correct_value}` but found {found_value}©r   ri   é   rŠ   é   r�   z2 or 4 or 8©rb   Úcorrect_valueÚfound_valuer   r’   za positive integerr“   )r   r   éÿÿÿÿr�   z`1` or `0`, `-1`r‘   z`1` or `0` or `-1`N)r�   Ú
ValueErrorÚformatr’   r“   r�   r‘   ©r   Úincorrect_arg_msgs     r   ÚvalidatezQuantizedCacheConfig.validateô   s@  € ð&ð 	ð �:‰:˜_Ñ,ÜØ!×(Ñ(ØØ"/Ø $§
¡
ð )ó óð ð ×Ñ Ò!ÜØ!×(Ñ(Ø&Ø"6Ø $× 1Ñ 1ð )ó óð ð ×Ñ !Ò#ÜØ!×(Ñ(Ø)Ø"6Ø $× 4Ñ 4ð )ó óð ð �=‰= 
Ñ*ÜØ!×(Ñ(Ø"Ø"4Ø $§¡ð )ó óð ð �?‰? *Ñ,ÜØ!×(Ñ(Ø$Ø"6Ø $§¡ð )ó óð ð -r   N)rG   rH   rI   rJ   rL   Úfloat16rO   r   rN   Údtyper   r¤   rX   r   r   rˆ   rˆ   Æ   sž   „ ñð4  Ø Ø"#Ø$%Ø&(Ø),Ø/4¯}©}Ø %ñàðð ˜‰}ðð ˜3‘-ð	ð
 ˜S‘Mðð ˜s‘mðð " #™ðð   §¡Ñ,ðð ˜‘óó(2r   rˆ   c                   ó,   — e Zd ZdZdZddedefd„Zd„ Zy)	ÚStaticCacheConfigz8
    Configuration class for static cache settings.
    ÚstaticÚ
batch_sizeÚmax_cache_lenc                 ó.   — || _         || _        || _        y r   )rª   r«   r;   )r   rª   r«   r;   s       r   r   zStaticCacheConfig.__init__1  s   € Ø$ˆŒØ*ˆÔØˆ�r   c                 óà   — d}| j                   dk  r't        |j                  dd| j                   ¬«      «      ‚| j                  dk  r't        |j                  dd| j                  ¬«      «      ‚y)r—   r˜   r   rª   z> 0rœ   r«   N)rª   r    r¡   r«   r¢   s     r   r¤   zStaticCacheConfig.validate6  s‹   € ð&ð 	ð
 �?‰?˜aÒÜØ!×(Ñ(Ø$Ø"'Ø $§¡ð )ó óð ð ×Ñ Ò"ÜØ!×(Ñ(Ø'Ø"'Ø $× 2Ñ 2ð )ó óð ð #r   N)r�   )rG   rH   rI   rJ   rV   rN   r   r¤   rX   r   r   r¨   r¨   )  s'   „ ñð $Ðñ 3ð °só ó
r   r¨   c                   ó  ‡ — e Zd ZdZddeddfˆ fd„Zdedeee	j                        fd„Zd„ Zd	„ Z	 dd
e	j                  de	j                  dedeeeef      dee	j                  e	j                  f   f
d„Zddee   defd„Zdee   fd„Zdeee	j                     ee	j                     f   fd„Zeddeeee	j.                           dd fd„«       Zdefd„Zdededed    fd„Zeded    dd fd„«       Zdefd„Zde	j                  fd„Zˆ xZS ) ÚDynamicCacheaÎ  
    A cache that grows dynamically as more tokens are generated. This is the default for generative models.

    It stores the Key and Value states as a list of tensors, one for each layer. The expected shape for each tensor is
    `[batch_size, num_heads, seq_len, head_dim]`.

    Example:

        ```python
        >>> from transformers import AutoTokenizer, AutoModelForCausalLM, DynamicCache

        >>> model = AutoModelForCausalLM.from_pretrained("Qwen/Qwen2-0.5B-Instruct")
        >>> tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen2-0.5B-Instruct")

        >>> inputs = tokenizer(text="My name is Qwen2", return_tensors="pt")

        >>> # Prepare a cache class and pass it to model's forward
        >>> past_key_values = DynamicCache()
        >>> outputs = model(**inputs, past_key_values=past_key_values, use_cache=True)
        >>> outputs.past_key_values # access cache filled with key/values from generation
        DynamicCache()
        ```
    NÚ_distributed_cache_datar#   c                 óÒ   •— t         ‰| �  «        d| _        g | _        g | _        |�A|D ];  \  }}| j                  j                  |«       | j                  j                  |«       Œ= y y ©Nr   )r   r   rA   r9   r>   r[   )r   r°   r   r    r   s       €r   r   zDynamicCache.__init__j  sl   ø€ Ü‰ÑÔØˆÔØ-/ˆŒØ/1ˆÔð #Ð.Ø,Cò 6Ñ(�
˜LØ—‘×%Ñ% jÔ1Ø× Ñ ×'Ñ'¨Õ5ñ6ð /r   r!   c                 óŽ   — |t        | «      k  r| j                  |   | j                  |   fS t        dt        | «      › d|› �«      ‚©z–
        Support for backwards-compatible `past_key_value` indexing, e.g. `past_key_value[0][0].shape[2]` to get the
        sequence length.
        úCache only has ú. layers, attempted to access layer with index )r8   r9   r>   ÚKeyErrorr*   s     r   Ú__getitem__zDynamicCache.__getitem__{  sN   € ð
 ”s˜4“yÒ Ø—N‘N 9Ñ-¨t×/?Ñ/?À	Ñ/JÐKÐKä˜_¬S°«Y¨KÐ7eÐfoÐepÐqÓrÐrr   c              #   ó~   K  — t        t        | «      «      D ]"  }| j                  |   | j                  |   f–— Œ$ y­w)z–
        Support for backwards-compatible `past_key_value` iteration, e.g. `for x in past_key_value:` to iterate over
        keys and values
        N©r7   r8   r9   r>   r*   s     r   r{   zDynamicCache.__iter__…  sA   è ø€ ô
 œs 4›yÓ)ò 	KˆIØ—>‘> )Ñ,¨d×.>Ñ.>¸yÑ.IÐJÓJñ	Kùs   ‚;=c                 ó,   — t        | j                  «      S ©z¬
        Support for backwards-compatible `past_key_value` length, e.g. `len(past_key_value)`. This value corresponds
        to the number of layers in the model.
        )r8   r9   r-   s    r   Ú__len__zDynamicCache.__len__�  s   € ô
 �4—>‘>Ó"Ð"r   r   r    r"   c                 ót  — |dk(  r"| xj                   |j                  d   z  c_         |��qt        | j                  «      |k  r·t	        t        | j                  «      |«      D ]^  }| j                  j                  t        j                  g «      «       | j                  j                  t        j                  g «      «       Œ` | j                  j                  |«       | j                  j                  |«       n¢| j                  |   j                  «       s|| j                  |<   || j                  |<   nft        j                  | j                  |   |gd¬«      | j                  |<   t        j                  | j                  |   |gd¬«      | j                  |<   | j                  |   | j                  |   fS )a‹  
        Updates the cache with the new `key_states` and `value_states` for the layer `layer_idx`.

        Parameters:
            key_states (`torch.Tensor`):
                The new key states to cache.
            value_states (`torch.Tensor`):
                The new value states to cache.
            layer_idx (`int`):
                The index of the layer to cache the states for.
            cache_kwargs (`Dict[str, Any]`, `optional`):
                Additional arguments for the cache subclass. No additional arguments are used in `DynamicCache`.

        Return:
            A tuple containing the updated key and value states.
        r   éþÿÿÿ©Údim)rA   Úshaper8   r9   r7   r[   rL   Útensorr>   r:   Úcat)r   r   r    r!   r"   Ú_s         r   r'   zDynamicCache.update”  sg  € ð0 ˜Š>Ø×Ò ×!1Ñ!1°"Ñ!5Ñ5Õð Ñ!Ü�4—>‘>Ó" iÒ/äœs 4§>¡>Ó2°IÓ>ò >�AØ—N‘N×)Ñ)¬%¯,©,°rÓ*:Ô;Ø×$Ñ$×+Ñ+¬E¯L©L¸Ó,<Õ=ð>ð —‘×%Ñ% jÔ1Ø× Ñ ×'Ñ'¨Õ5à—N‘N 9Ñ-×3Ñ3Ô5à,6�—‘˜yÑ)Ø.:�× Ñ  Ò+ä,1¯I©I°t·~±~ÀiÑ7PÐR\Ð6]ÐceÔ,f�—‘˜yÑ)Ü.3¯i©i¸×9IÑ9IÈ)Ñ9TÐVbÐ8cÐikÔ.l�× Ñ  Ñ+à�~‰~˜iÑ(¨$×*:Ñ*:¸9Ñ*EÐEÐEr   c                 óî   — t        | j                  «      dk(  xs8 t        | j                  «      |k  xs | j                  |   j                  «        }|s| j                  |   j                  d   }|S d}|S ©r)   r   r¿   )r8   r9   r:   rÂ   )r   r!   Úis_empty_layerÚlayer_seq_lengths       r   r+   zDynamicCache.get_seq_lengthÃ  sƒ   € ô �—‘Ó 1Ñ$ò 5Ü�4—>‘>Ó" iÑ/ò5à—>‘> )Ñ,×2Ñ2Ó4Ð4ð 	ñ
 GU˜4Ÿ>™>¨)Ñ4×:Ñ:¸2Ñ>ÐØÐð [\ÐØÐr   c                  ó   — y)zeReturns the maximum sequence length of the cache object. DynamicCache does not have a maximum length.NrX   r-   s    r   r.   z DynamicCache.get_max_cache_shapeÎ  s   € àr   c                 ó€   — d}t        t        | «      «      D ]$  }|| j                  |   | j                  |   ffz  }Œ& |S )z�Converts the `DynamicCache` instance into the its equivalent in the legacy cache format. Used for
        backward compatibility.rX   rº   )r   Úlegacy_cacher!   s      r   Úto_legacy_cachezDynamicCache.to_legacy_cacheÒ  sO   € ð ˆÜœs 4›yÓ)ò 	XˆIØ˜dŸn™n¨YÑ7¸×9IÑ9IÈ)Ñ9TÐUÐWÑW‰Lð	XàÐr   Úpast_key_valuesc                 ó€   —  | «       }|�4t        t        |«      «      D ]  }||   \  }}|j                  |||«       Œ |S )zwConverts a cache in the legacy cache format into an equivalent `DynamicCache`. Used for
        backward compatibility.)r7   r8   r'   ©r]   rÎ   Úcacher!   r   r    s         r   Úfrom_legacy_cachezDynamicCache.from_legacy_cacheÚ  sR   € ñ “ˆØÐ&Ü"¤3 Ó#7Ó8ò B�	Ø+:¸9Ñ+EÑ(�
˜LØ—‘˜Z¨°yÕAðBð ˆr   r1   c                 ó–  — |dk  r| j                  «       t        |«      z
  }| j                  «       |k  ry|| _        t        t	        | j
                  «      «      D ]l  }| j
                  |   j                  «       sŒ!| j
                  |   dd|…dd…f   | j
                  |<   | j                  |   dd|…dd…f   | j                  |<   Œn y)zÈCrop the past key values up to a new `max_length` in terms of tokens. `max_length` can also be
        negative to remove `max_length` tokens. This is used in assisted decoding and contrastive search.r   N.)r+   ÚabsrA   r7   r8   r9   r:   r>   )r   r1   Úidxs      r   ÚcropzDynamicCache.cropå  sÅ   € ð ˜Š>Ø×,Ñ,Ó.´°Z³Ñ@ˆJà×ÑÓ  JÒ.Øà&ˆÔÜœ˜TŸ^™^Ó,Ó-ò 	SˆCØ�~‰~˜cÑ"×(Ñ(Õ*Ø&*§n¡n°SÑ&9¸#¸{À
¸{ÊAÐ:MÑ&N�—‘˜sÑ#Ø(,×(8Ñ(8¸Ñ(=¸cÀ;ÀJÀ;ÒPQÐ>QÑ(R�× Ñ  Ò%ñ	Sr   Úfull_batch_sizeÚ
split_sizec                 ó.  — g }t        d||«      D ]x  }t        «       }| j                  |_        | j                  D �cg c]
  }||||z    ‘Œ c}|_        | j                  D �cg c]
  }||||z    ‘Œ c}|_        |j                  |«       Œz |S c c}w c c}w )ú–Split the current instance into a list of `DynamicCache` by the batch size. This will be used by
        `_split_model_inputs()` in `generation.utils`r   )r7   r¯   rA   r9   r>   r[   )r   r×   rØ   ÚoutÚiÚcurrent_splitrÃ   s          r   Úbatch_splitzDynamicCache.batch_splitõ  sš   € ð ˆÜ�q˜/¨:Ó6ò 	&ˆAÜ(›NˆMØ)-×):Ñ):ˆMÔ&ØPT×P^ÑP^Ö&_Àf v¨a°!°j±.Ò'AÒ&_ˆMÔ#ØRV×RbÑRbÖ(cÈ¨°°A¸
±NÒ)CÒ(cˆMÔ%Ø�J‰J�}Õ%ð	&ð ˆ
ùò '`ùÚ(cs   ¼BÁ!BÚsplitsc                 óÖ  —  | «       }t        t        |d   «      «      D ]½  }|D �cg c]/  }|j                  |   j                  «       sŒ!|j                  |   ‘Œ1 }}|D �cg c]/  }|j                  |   j                  «       sŒ!|j                  |   ‘Œ1 }}|g k7  sŒ}t        j                  |d¬«      }t        j                  |d¬«      }|j                  |||«       Œ¿ |S c c}w c c}w ©z‚This is the opposite of the above `batch_split()` method. This will be used by `stack_model_outputs` in
        `generation.utils`r   rÀ   )r7   r8   r9   r:   r>   rL   rÄ   r'   )	r]   rß   rÑ   rÕ   Úcurrentr9   r>   Ú
layer_keysÚlayer_valuess	            r   Úfrom_batch_splitszDynamicCache.from_batch_splits  sØ   € ñ “ˆÜœ˜V A™Y›Ó(ò 	<ˆCØ?EÖh°GÈ×IZÑIZÐ[^ÑI_×IeÑIeÕIg˜×*Ñ*¨3Ó/ÐhˆIÐhØCIÖn¸ÈW×M`ÑM`ÐadÑMe×MkÑMkÕMm˜7×.Ñ.¨sÓ3ÐnˆKÐnØ˜B‹Ü"ŸY™Y y°aÔ8�
Ü$Ÿy™y¨¸!Ô<�Ø—‘˜Z¨°sÕ;ð	<ð ˆùò iùÚns   ¦"C!Á	C!Á "C&ÂC&Úrepeatsc                 óê   — t        t        | «      «      D ]\  }| j                  |   j                  |d¬«      | j                  |<   | j                  |   j                  |d¬«      | j                  |<   Œ^ y)úTRepeat the cache `repeats` times in the batch dimension. Used in contrastive search.r   rÀ   N)r7   r8   r9   Úrepeat_interleaver>   )r   ræ   r!   s      r   Úbatch_repeat_interleavez$DynamicCache.batch_repeat_interleave  sr   € äœs 4›yÓ)ò 	hˆIØ(,¯©°yÑ(A×(SÑ(SÐT[ÐabÐ(SÓ(cˆD�N‰N˜9Ñ%Ø*.×*:Ñ*:¸9Ñ*E×*WÑ*WÐX_ÐefÐ*WÓ*gˆD×Ñ˜YÒ'ñ	hr   Úindicesc                 óº   — t        t        | «      «      D ]D  }| j                  |   |df   | j                  |<   | j                  |   |df   | j                  |<   ŒF y)úXOnly keep the `indices` in the batch dimension of the cache. Used in contrastive search..Nrº   )r   rë   r!   s      r   Úbatch_select_indicesz!DynamicCache.batch_select_indices  s`   € äœs 4›yÓ)ò 	TˆIØ(,¯©°yÑ(AÀ'È3À,Ñ(OˆD�N‰N˜9Ñ%Ø*.×*:Ñ*:¸9Ñ*EÀgÈsÀlÑ*SˆD×Ñ˜YÒ'ñ	Tr   r   rF   )rG   rH   rI   rJ   r   r   rN   r   r	   rL   rM   r¸   r{   r½   r   r   rO   r   r'   r+   r.   rÍ   r„   ÚFloatTensorrÒ   rÖ   rÞ   rå   rê   rî   rR   rS   s   @r   r¯   r¯   Q  s­  ø„ ññ06°ð 6ÀDõ 6ð"s Sð s¨T°%¸¿¹Ñ2EÑ-Fó sòKò#ð 26ñ-Fà—L‘Lð-Fð —l‘lð-Fð ð	-Fð
 ˜t C¨ H™~Ñ.ð-Fð 
ˆu�|‰|˜UŸ\™\Ð)Ñ	*ó-Fñ^	 ¨°©ð 	 ¸có 	 ð X¨c¡]ó ð  u¨U¯\©\Ñ':¸EÀ%Ç,Á,Ñ<OÐ'OÑ!Pó ð ñ°¸¸uÀU×EVÑEVÑ?WÑ9XÑ0Yð Ðesò ó ððS˜só Sð 
¨3ð 
¸Cð 
ÀDÈÑDXó 
ð ð t¨NÑ';ð Àò ó ððh¨só hðT¨E¯L©L÷ Tr   r¯   Údynamic_cachec                 óò   — t        | t        «      st        d«      ‚t        st        j                  d«       t        | d«      t        | d«      dœ}t        j                  j                  j                  |«      S )zTFlattens DynamicCache into flat list of tensors for `torch.export.export` to consumezFThis pytree flattening function should only be applied to DynamicCachez[DynamicCache + torch.export is tested on torch 2.6.0+ and may not work on earlier versions.r9   r>   ©r9   r>   )Ú
isinstancer¯   ÚRuntimeErrorr   rB   rC   ÚgetattrrL   ÚutilsÚ_pytreeÚ_dict_flatten©rð   Ú
dictionarys     r   Ú_flatten_dynamic_cacherû     sh   € ô �m¤\Ô2ÜÐcÓdÐdå-Ü×ÑØiô	
ô ˜]¨KÓ8Ü˜}¨mÓ<ñ€Jô �;‰;×Ñ×,Ñ,¨ZÓ8Ð8r   c                 ó†   — t        | d«      t        | d«      dœ}t        j                  j                  j	                  |«      S )Nr9   r>   rò   )rõ   rL   rö   r÷   Ú_dict_flatten_with_keysrù   s     r   Ú _flatten_with_keys_dynamic_cacherþ   1  s:   € ä˜]¨KÓ8Ü˜}¨mÓ<ñ€Jô �;‰;×Ñ×6Ñ6°zÓBÐBr   Úcontextc                 ó¸   — t         j                  j                  j                  | |«      }t	        «       }|j                  «       D ]  \  }}t        |||«       Œ |S r   )rL   rö   r÷   Ú_dict_unflattenr¯   rY   rZ   )Úvaluesrÿ   rú   rÑ   ÚkÚvs         r   Ú_unflatten_dynamic_cacher  9  sT   € ô —‘×$Ñ$×4Ñ4°V¸WÓE€JÜ‹N€EØ× Ñ Ó"ò ‰ˆˆ1Ü��q˜!Õðà€Lr   c                 óŒ   — t        | d«      t        | d«      dœ}t        j                  j                  j	                  |«      d   S )Nr9   r>   rò   r   )rõ   rL   rö   r÷   Útree_flatten)rÑ   Úspecrú   s      r   Ú_flatten_dynamic_cache_for_fxr	  D  s?   € ä˜U KÓ0Ü˜u mÓ4ñ€Jô �;‰;×Ñ×+Ñ+¨JÓ7¸Ñ:Ð:r   z2.3ú.)Úserialized_type_nameÚflatten_with_keys_fnc                   ó  ‡ — e Zd ZdZdˆ fd„Zdefd„Zdefd„Zdedee	e
j                        fd„Zd	e
j                  fd
„Z	 dde
j                  de
j                  dedeeeef      de	e
j                  e
j                  f   f
d„ZdZdZˆ xZS )ÚOffloadedCacheañ  
    A drop-in replacement for DynamicCache that conserves accelerator(GPU, XPU) memory at the expense of more CPU memory.
    Useful for generating from models with very long context.

    In addition to the default accelerator stream, where all forward() computations happen,
    this class uses another stream, the prefetch stream, which it creates itself.
    Since scheduling of operations on separate streams happens independently, this class uses
    the prefetch stream to asynchronously prefetch the KV cache of layer k+1 when layer k is executing.
    The movement of the layer k-1 cache to the CPU is handled by the default stream as a simple way to
    ensure the eviction is scheduled after all computations on that cache are finished.
    r#   Nc                 ó¬  •— t         j                  j                  «       sNt        dd¬«      rt         j                  j                  «       s#t        dt        dd¬«      rdz   «      ‚dz   «      ‚t        ‰| �  «        g | _        d | _	        t        dd¬«      rt        j                  «       nt         j                  j                  «       | _	        d | _        y )Nú2.7T©Ú
accept_devz*OffloadedCache can only be used with a GPUz or XPUÚ )rL   ÚcudaÚis_availabler   Úxpurô   r   r   Úoriginal_deviceÚprefetch_streamÚStreamr4   r   s    €r   r   zOffloadedCache.__init__e  s³   ø€ ä�J‰J×#Ñ#Ô%Ü)¨%¸DÕAÄeÇiÁi×F\ÑF\ÔF^äØ<Ü 9¸%ÈDÕ Q�9ñ[óð àWYñ[óð ô
 	‰ÑÔØ!ˆÔØ#ˆÔä7¸È$ÕOŒE�L‰LŒNÔUZ×U_ÑU_×UfÑUfÓUhð 	Ôð ˆ�r   r!   c                 ó¢  — |t        | «      k  rµt        dd¬«      r| j                  n(t        j                  j                  | j                  «      5  | j                  |   }| j                  |   j                  |d¬«      | j                  |<   | j                  |   j                  |d¬«      | j                  |<   ddd«       yy# 1 sw Y   yxY w)z'Starts prefetching the next layer cacher  Tr  ©Únon_blockingN)
r8   r   r  rL   r  Ústreamr  r9   r=   r>   )r   r!   r;   s      r   Úprefetch_layerzOffloadedCache.prefetch_layerw  sÇ   € à”s˜4“yÒ ô -¨U¸tÕDð ×$Ò$ä—Z‘Z×&Ñ& t×';Ñ';Ó<ñhð ×-Ñ-¨iÑ8�Ø,0¯N©N¸9Ñ,E×,HÑ,HÈÐ^bÐ,HÓ,c�—‘˜yÑ)Ø.2×.>Ñ.>¸yÑ.I×.LÑ.LÈVÐbfÐ.LÓ.g�× Ñ  Ñ+÷hð hð !÷hð hús   ÁA*CÃCc                 óø   — t        | «      dkD  rl|dz
  t        | «      z  }| j                  |   j                  dd¬«      | j                  |<   | j                  |   j                  dd¬«      | j                  |<   yy)z)Moves the previous layer cache to the CPUri   r   r�   Tr  N)r8   r9   r=   r>   )r   r!   Úprev_layer_idxs      r   Úevict_previous_layerz#OffloadedCache.evict_previous_layer„  sy   € äˆt‹9�qŠ=à'¨!™m¬s°4«yÑ8ˆNØ-1¯^©^¸NÑ-K×-NÑ-NÈuÐcgÐ-NÓ-hˆD�N‰N˜>Ñ*Ø/3×/?Ñ/?ÀÑ/O×/RÑ/RÐSXÐgkÐ/RÓ/lˆD×Ñ˜^Ò,ð	 r   c                 óà  — |t        | «      k  �rFt        dd¬«      r-t        j                  j	                  «       j                  «        n,t        j                  j	                  «       j                  «        | j                  |«       | j                  |   }| j                  j                  «        | j                  |   }| j                  |   }| j                  �X| j                  j                  |«      | _        |j                  d| j                  «      }|j                  d| j                  «      }| j                  |dz   t        | «      z  «       ||fS t!        dt        | «      › d|› �«      ‚)z_Gets the cache for this layer to the device. Prefetches the next and evicts the previous layer.r  Tr  r   r   rµ   r¶   )r8   r   rL   ÚacceleratorÚcurrent_streamÚsynchronizer  r!  r  r  r9   r>   r4   r=   r<   r  r·   )r   r!   r  Ú
key_tensorÚvalue_tensors        r   r¸   zOffloadedCache.__getitem__Œ  s+  € à”s˜4“yÓ ä(¨¸4Õ@Ü×!Ñ!×0Ñ0Ó2×>Ñ>Õ@ä—
‘
×)Ñ)Ó+×7Ñ7Ô9Ø×%Ñ% iÔ0à"×2Ñ2°9Ñ=ˆOØ× Ñ ×,Ñ,Ô.ØŸ™¨	Ñ2ˆJØ×+Ñ+¨IÑ6ˆLà�}‰}Ð(Ø $§¡× 0Ñ 0°Ó A�”Ø'×4Ñ4°Q¸¿¹ÓF�
Ø+×8Ñ8¸¸D¿M¹MÓJ�à×Ñ ¨Q¡´#°d³)Ñ ;Ô<Ø Ð-Ð-ä˜_¬S°«Y¨KÐ7eÐfoÐepÐqÓrÐrr   r4   c                 ó2   — | ` |j                  «       | _         y)zTSaves the beam indices and reorders the cache when the tensor is back to its device.N)r4   Úclone©r   r4   s     r   r?   zOffloadedCache.reorder_cache¥  s   € ð ˆMØ Ÿ™Ó(ˆ�r   r   r    r"   c                 ó„  — |dk(  r"| xj                   |j                  d   z  c_         t        | j                  «      |k  rt	        d«      ‚t        | j                  «      |k(  rm| j                  j                  |«       | j                  j                  |«       | j                  j                  |j                  «       | j                  |«       nT| |   \  }}t        j                  ||gd¬«      | j                  |<   t        j                  ||gd¬«      | j                  |<   | j                  |   | j                  |   fS )a‹  
        Updates the cache with the new `key_states` and `value_states` for the layer `layer_idx`.
        Parameters:
            key_states (`torch.Tensor`):
                The new key states to cache.
            value_states (`torch.Tensor`):
                The new value states to cache.
            layer_idx (`int`):
                The index of the layer to cache the states for.
            cache_kwargs (`Dict[str, Any]`, `optional`):
                Additional arguments for the cache subclass. No additional arguments are used in `OffloadedCache`.
        Return:
            A tuple containing the updated key and value states.
        r   r¿   zWOffloadedCache does not support model usage where layers are skipped. Use DynamicCache.rÀ   )rA   rÂ   r8   r9   r    r[   r>   r  r;   r!  rL   rÄ   )r   r   r    r!   r"   r&  r'  s          r   r'   zOffloadedCache.update¬  s  € ð, ˜Š>Ø×Ò ×!1Ñ!1°"Ñ!5Ñ5Õô ˆt�~‰~Ó Ò*ÜÐvÓwÐwÜ�—‘Ó  IÒ-Ø�N‰N×!Ñ! *Ô-Ø×Ñ×#Ñ# LÔ1Ø× Ñ ×'Ñ'¨
×(9Ñ(9Ô:Ø×%Ñ% iÕ0à'+¨I¡Ñ$ˆJ˜Ü(-¯	©	°:¸zÐ2JÐPRÔ(SˆD�N‰N˜9Ñ%Ü*/¯)©)°\À<Ð4PÐVXÔ*YˆD×Ñ˜YÑ'à�~‰~˜iÑ(¨$×*:Ñ*:¸9Ñ*EÐEÐEr   ©r#   Nr   )rG   rH   rI   rJ   r   rN   r  r!  r   r	   rL   rM   r¸   rP   r?   r   r   rO   r   r'   rÒ   rÍ   rR   rS   s   @r   r  r  X  sÔ   ø„ ñ
õð$h¨ó hðm¨có mðs Sð s¨T°%¸¿¹Ñ2EÑ-Fó sð2) e×&6Ñ&6ó )ð 26ñ&Fà—L‘Lð&Fð —l‘lð&Fð ð	&Fð
 ˜t C¨ H™~Ñ.ð&Fð 
ˆu�|‰|˜UŸ\™\Ð)Ñ	*ó&FðT Ðà„Or   r  c                   óÚ   ‡ — e Zd ZdZdeddfˆ fd„Z	 ddej                  dej                  ded	e	e
eef      deej                  ej                  f   f
d
„Zdde	e   defd„Zd„ Zd„ Zˆ xZS )ÚQuantizedCacheaK  
    A quantizer cache similar to what is described in the [KIVI: A Tuning-Free Asymmetric 2bit Quantization for KV Cache paper](https://arxiv.org/abs/2402.02750).
    It allows the model to generate longer sequence length without allocating too much memory for Key and Value cache by applying quantization.

    The cache has two types of storage, one for original precision and one for the quantized cache. A `residual length` is set as a maximum capacity for the
    original precision cache. When the length goes beyond maximum capacity, the original precision cache is discarded and moved into the quantized cache. The
    quantization is done per-channel with a set `q_group_size` for both Keys and Values, in contrast to what was described in the paper.

    It stores Keys and Values a list of quantized tensors (tuples in case we need to store metadata), one for each layer. Additionally, it stores the Key and
    Value in original precision states as a list of tensors, one for each layer. The size of each tensor
    is `[batch_size, num_heads, seq_len - residual_length, head_dim]`
    Úcache_configr#   Nc                 óH  •— t         ‰| �  «        g | _        g | _        |j                  | _        |j
                  | _        |j                  | _        |j                  | _        |j                  | _        |j                  | _	        |j                  | _
        t         ‰| �  «        y r   )r   r   Ú_quantized_key_cacheÚ_quantized_value_cacher�   r“   r’   r�   r‘   r”   r;   ©r   r/  r   s     €r   r   zQuantizedCache.__init__é  s‡   ø€ Ü‰ÑÔØ8:ˆÔ!Ø:<ˆÔ#à!×'Ñ'ˆŒ
Ø+×;Ñ;ˆÔØ(×5Ñ5ˆÔØ$×-Ñ-ˆŒØ&×1Ñ1ˆŒØ)×7Ñ7ˆÔØ"×)Ñ)ˆŒä‰ÑÕr   r   r    r!   r"   c                 óR  — |dk(  r"| xj                   |j                  d   z  c_         t        | j                  «      |k  rt	        d«      ‚t        | j                  «      |k(  �r| j
                  j                  | j                  |j                  «       | j                  ¬«      «       | j                  j                  | j                  |j                  «       | j                  ¬«      «       | j                  j                  t        j                  d|j                  |j                  ¬«      «       | j                   j                  t        j                  d|j                  |j                  ¬«      «       ||}}||fS | j#                  | j
                  |   «      }| j#                  | j                  |   «      }|| j                  |   |g}|| j                   |   |g}t        j$                  |d¬«      }t        j$                  |d¬«      }| j                  |   j'                  «       dk(  �r| j                  |   j                  d   dz   | j(                  k\  ræ| j                  |j                  «       | j                  ¬«      | j
                  |<   | j                  |j                  «       | j                  ¬«      | j                  |<   t        j                  d|j                  |j                  ¬«      | j                  |<   t        j                  d|j                  |j                  ¬«      | j                   |<   ||fS t        j$                  | j                  |   |gd¬«      | j                  |<   t        j$                  | j                   |   |gd¬«      | j                   |<   ||fS )	Nr   r¿   zWQuantizedCache does not support model usage where layers are skipped. Use DynamicCache.)Úaxis©r¦   r;   rÀ   rŠ   r   )rA   rÂ   r8   r9   r    r1  r[   Ú	_quantizeÚ
contiguousr�   r2  r‘   rL   Úzerosr¦   r;   r>   Ú_dequantizerÄ   rÁ   r“   )	r   r   r    r!   r"   Úkeys_to_returnÚvalues_to_returnÚdequant_keyÚdequant_values	            r   r'   zQuantizedCache.updateø  s  € ð ˜Š>Ø×Ò ×!1Ñ!1°"Ñ!5Ñ5Õäˆt�~‰~Ó Ò*ÜÐvÓwÐwÜ�—‘Ó  IÓ-Ø×%Ñ%×,Ñ,¨T¯^©^¸J×<QÑ<QÓ<SÐZ^×ZgÑZg¨^Ó-hÔiØ×'Ñ'×.Ñ.¨t¯~©~¸l×>UÑ>UÓ>WÐ^b×^mÑ^m¨~Ó/nÔoØ�N‰N×!Ñ!¤%§+¡+¨a°z×7GÑ7GÐPZ×PaÑPaÔ"bÔcØ×Ñ×#Ñ#¤E§K¡K°¸×9IÑ9IÐR\×RcÑRcÔ$dÔeØ/9¸<Ð,ˆNð. Ð/Ð/Ð/ð+ ×*Ñ*¨4×+DÑ+DÀYÑ+OÓPˆKØ ×,Ñ,¨T×-HÑ-HÈÑ-SÓTˆMØ)¨4¯>©>¸)Ñ+DÀjÐQˆNØ -¨t×/?Ñ/?À	Ñ/JÈLÐYÐä"ŸY™Y ~¸2Ô>ˆNÜ$Ÿy™yÐ)9¸rÔBÐà—‘˜yÑ)×-Ñ-Ó/°1Ó4Ø—N‘N 9Ñ-×3Ñ3°BÑ7¸!Ñ;¸t×?SÑ?SÒSà7;·~±~Àn×F_ÑF_ÓFaÐhl×huÑhu°~Ó7v�×)Ñ)¨)Ñ4Ø9=¿¹Ø$×/Ñ/Ó1¸¿¹ð :Hó :�×+Ñ+¨IÑ6ô -2¯K©K¸À×AQÑAQÐZd×ZkÑZkÔ,l�—‘˜yÑ)Ü.3¯k©k¸!À:×CSÑCSÐ\f×\mÑ\mÔ.n�× Ñ  Ñ+ð
 Ð/Ð/Ð/ô -2¯I©I°t·~±~ÀiÑ7PÐR\Ð6]ÐceÔ,f�—‘˜yÑ)Ü.3¯i©i¸×9IÑ9IÈ)Ñ9TÐVbÐ8cÐikÔ.l�× Ñ  Ñ+àÐ/Ð/Ð/r   c                 ót   — t        | j                  «      |k  ry|dk(  r| j                  S | j                  dz
  S )r)   r   r   )r8   r9   rA   r*   s     r   r+   zQuantizedCache.get_seq_length#  s:   € äˆt�~‰~Ó )Ò+Øð %.°¢Nˆt× Ñ ÐM¸×8IÑ8IÈAÑ8MÐMr   c                 ó   — t        d«      ‚)z:Quantizes a key/value using a defined quantization method.z1Make sure to implement `_quantize` in a subclass.r%   )r   rÃ   r5  s      r   r7  zQuantizedCache._quantize,  s   € ä!Ð"UÓVÐVr   c                 ó   — t        d«      ‚)zDDequantizes back the tensor that was quantized by `self._quantize()`z3Make sure to implement `_dequantize` in a subclass.r%   )r   Úq_tensors     r   r:  zQuantizedCache._dequantize0  s   € ä!Ð"WÓXÐXr   r   rF   )rG   rH   rI   rJ   rˆ   r   rL   rM   rN   r   r   rO   r   r	   r'   r+   r7  r:  rR   rS   s   @r   r.  r.  Û  s¤   ø„ ñðÐ%9ð ¸dõ ð( 26ñ)0à—L‘Lð)0ð —l‘lð)0ð ð	)0ð
 ˜t C¨ H™~Ñ.ð)0ð 
ˆu�|‰|˜UŸ\™\Ð)Ñ	*ó)0ñVN¨°©ð N¸có NòWöYr   r.  c                   ó8   ‡ — e Zd ZdZdeddfˆ fd„Zd„ Zd„ Zˆ xZS )ÚQuantoQuantizedCacheaÊ  
    Quantized Cache class that uses `quanto` as a backend to perform quantization. Current implementation supports `int2` and `int4` dtypes only.

    Parameters:
        cache_config (`QuantizedCacheConfig`):
            A configuration containing all the arguments to be used by the quantizer, including axis, qtype and group size.

    Example:

        ```python
        >>> # Run pip install quanto first if you don't have it yet
        >>> from transformers import AutoTokenizer, AutoModelForCausalLM, QuantoQuantizedCache, QuantizedCacheConfig

        >>> model = AutoModelForCausalLM.from_pretrained("Qwen/Qwen2-0.5B-Instruct")
        >>> tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen2-0.5B-Instruct")

        >>> inputs = tokenizer(text="My name is Qwen2", return_tensors="pt")

        >>> # Prepare a cache class and pass it to model's forward
        >>> cache_config = QuantizedCacheConfig(nbits=4)
        >>> past_key_values = QuantoQuantizedCache(cache_config=cache_config)
        >>> outputs = model(**inputs, past_key_values=past_key_values, use_cache=True)
        >>> outputs.past_key_values # access cache filled with key/values from generation
        QuantoQuantizedCache()
        ```
    r/  r#   Nc                 ó*  •— t         ‰| �  |«       t        «       rct        j                  t
        j                  j                  d«      «      }|t        j                  d«      k  rt        d|› d�«      ‚ddlm	}m
}m} | j                  dvrt        d| j                  › �«      ‚| j                  d	vrt        d
| j                  › �«      ‚| j                  d	vrt        d| j                  › �«      ‚| j                  dk(  rn| _         «       | _        y )Nzoptimum-quantoz0.2.5zzYou need optimum-quanto package version to be greater or equal than 0.2.5 to use `QuantoQuantizedCache`. Detected version r
  r   )ÚMaxOptimizerÚqint2Úqint4)ri   rŠ   zA`nbits` for `quanto` backend has to be one of [`2`, `4`] but got )r   rŸ   zE`axis_key` for `quanto` backend has to be one of [`0`, `-1`] but got zG`axis_value` for `quanto` backend has to be one of [`0`, `-1`] but got rŠ   )r   r   r   r   ÚparseÚ	importlibÚmetadataÚImportErrorÚoptimum.quantorF  rG  rH  r�   r    r�   r‘   ÚqtypeÚ	optimizer)r   r/  Úoptimum_quanto_versionrF  rG  rH  r   s         €r   r   zQuantoQuantizedCache.__init__Q  s  ø€ Ü‰Ñ˜Ô&ä&Ô(Ü%,§]¡]´9×3EÑ3E×3MÑ3MÐN^Ó3_Ó%`Ð"Ø%¬¯©°wÓ)?Ò?Ü!ð Qð  Rhð  Qið  ijð  kóð ÷ BÑAà�:‰:˜VÑ#ÜÐ`Ðae×akÑakÐ`lÐmÓnÐnà�=‰= Ñ'ÜÐdÐei×erÑerÐdsÐtÓuÐuà�?‰? 'Ñ)ÜØYÐZ^×ZiÑZiÐYjÐkóð ð #Ÿj™j¨Ašo‘U°5ˆŒ
Ù%›ˆ�r   c                 óÀ   — t        «       rTddlm} | j                  || j                  || j
                  «      \  }} ||| j                  |||| j
                  «      }|S y )Nr   )Úquantize_weight)r   rM  rR  rO  rN  r’   )r   rÃ   r5  rR  ÚscaleÚ	zeropointÚqtensors          r   r7  zQuantoQuantizedCache._quantizej  sW   € ä&Ô(Ý6à#Ÿ~™~¨f°d·j±jÀ$È×HYÑHYÓZÑˆE�9Ù% f¨d¯j©j¸$ÀÀyÐRV×RcÑRcÓdˆGØˆNð )r   c                 ó"   — |j                  «       S r   )Ú
dequantize)r   rU  s     r   r:  z QuantoQuantizedCache._dequantizes  s   € Ø×!Ñ!Ó#Ð#r   ©	rG   rH   rI   rJ   rU   r   r7  r:  rR   rS   s   @r   rD  rD  5  s%   ø„ ñð6( [ð (°Tõ (ò2ö$r   rD  c                   ó8   ‡ — e Zd ZdZdeddfˆ fd„Zd„ Zd„ Zˆ xZS )ÚHQQQuantizedCacheaÕ  
    Quantized Cache class that uses `HQQ` as a backend to perform quantization. Current implementation supports `int2`, `int4`, `int8` dtypes.

    Parameters:
        cache_config (`QuantizedCacheConfig`):
            A configuration containing all the arguments to be used by the quantizer, including axis, qtype and group size.

    Example:

        ```python
        >>> # Run pip install hqq first if you don't have it yet
        >>> from transformers import AutoTokenizer, AutoModelForCausalLM, HQQQuantizedCache, QuantizedCacheConfig

        >>> model = AutoModelForCausalLM.from_pretrained("Qwen/Qwen2-0.5B-Instruct")
        >>> tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen2-0.5B-Instruct")

        >>> inputs = tokenizer(text="My name is Qwen2", return_tensors="pt")

        >>> # Prepare a cache class and pass it to model's forward
        >>> cache_config = QuantizedCacheConfig(nbits=4, axis_key=1, axis_value=1)
        >>> past_key_values = HQQQuantizedCache(cache_config=cache_config)
        >>> outputs = model(**inputs, past_key_values=past_key_values, use_cache=True)
        >>> outputs.past_key_values # access cache filled with key/values from generation
        HQQQuantizedCache()
        ```
    r/  r#   Nc                 ó  •— t         ‰| �  |«       | j                  dvrt        d| j                  › �«      ‚| j                  dvrt        d| j                  › �«      ‚| j
                  dvrt        d| j
                  › �«      ‚t        | _        y )Nr™   zM`nbits` for `HQQ` backend has to be one of [`1`, `2`, `3`, `4`, `8`] but got )r   r   zA`axis_key` for `HQQ` backend has to be one of [`0`, `1`] but got zC`axis_value` for `HQQ` backend has to be one of [`0`, `1`] but got )r   r   r�   r    r�   r‘   ÚHQQQuantizerÚ	quantizerr3  s     €r   r   zHQQQuantizedCache.__init__“  s’   ø€ Ü‰Ñ˜Ô&Ø�:‰:˜_Ñ,ÜØ_Ð`d×`jÑ`jÐ_kÐlóð ð �=‰= Ñ&ÜÐ`Ðae×anÑanÐ`oÐpÓqÐqà�?‰? &Ñ(ÜÐbÐcg×crÑcrÐbsÐtÓuÐuä%ˆ�r   c                 ó  — | j                   j                  ||| j                  | j                  | j                  | j
                  ¬«      \  }}| j                  |d<   | j                   j                  ||| j                  ¬«       ||fS )N)r5  r;   r”   r�   Ú
group_sizer”   )Úmetar;   )r]  Úquantizer;   r”   r�   r’   r  )r   rÃ   r5  rU  r`  s        r   r7  zHQQQuantizedCache._quantize¢  s~   € ØŸ™×/Ñ/ØØØ—;‘;Ø×,Ñ,Ø—*‘*Ø×(Ñ(ð 0ó 
‰ˆ�ð !%× 2Ñ 2ˆˆ_ÑØ�‰×Ñ˜G¨$°t·{±{ÐÔCØ˜ˆ}Ðr   c                 óH   — |\  }}| j                   j                  ||«      }|S r   )r]  rW  )r   rU  Úquant_tensorr`  rÃ   s        r   r:  zHQQQuantizedCache._dequantize¯  s'   € Ø$Ñˆ�dØ—‘×*Ñ*¨<¸Ó>ˆØˆr   rX  rS   s   @r   rZ  rZ  w  s%   ø„ ñð6& [ð &°Tõ &òör   rZ  c                   óæ  ‡ — e Zd ZdZdZdededdfˆ fd„Zed„ «       Zd	e	j                  d
e	j                  de	j                  de	j                  fd„Zd	e	j                  d
e	j                  de	j                  dee	j                  e	j                  f   fd„Zddee   defd„Zdee   fd„Z	 dd	e	j                  de	j                  dedeeeef      dee	j                  e	j                  f   f
d„Zˆ xZS )Ú	SinkCachea«  
    A cache that as described in the [Attention Sinks paper](https://arxiv.org/abs/2309.17453). It allows the model to
    generate beyond the length of its context window, without losing fluency in the conversation. As it discards past
    tokens, the model will lose the ability to generate tokens that depend on the context that was discarded.

    It stores the Key and Value states as a list of tensors, one for each layer. The expected shape for each tensor is
    `[batch_size, num_heads, seq_len, head_dim]`.

    Parameters:
        window_length (`int`):
            The length of the context window.
        num_sink_tokens (`int`):
            The number of sink tokens. See the original paper for more information.

    Example:

        ```python
        >>> from transformers import AutoTokenizer, AutoModelForCausalLM, SinkCache

        >>> model = AutoModelForCausalLM.from_pretrained("Qwen/Qwen2-0.5B-Instruct")
        >>> tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen2-0.5B-Instruct")

        >>> inputs = tokenizer(text="My name is Qwen2", return_tensors="pt")

        >>> # Prepare a cache class and pass it to model's forward
        >>> past_key_values = SinkCache(window_length=256, num_sink_tokens=4)
        >>> outputs = model(**inputs, past_key_values=past_key_values, use_cache=True)
        >>> outputs.past_key_values # access cache filled with key/values from generation
        SinkCache()
        ```
    TÚwindow_lengthÚnum_sink_tokensr#   Nc                 ó’   •— t         ‰| �  «        g | _        g | _        || _        || _        i | _        d | _        d | _        d| _	        y r²   )
r   r   r9   r>   rf  rg  Úcos_sin_rerotation_cacheÚ
_cos_cacheÚ
_sin_cacherA   )r   rf  rg  r   s      €r   r   zSinkCache.__init__Ø  sK   ø€ Ü‰ÑÔØ-/ˆŒØ/1ˆÔØ*ˆÔØ.ˆÔØ(*ˆÔ%ØˆŒØˆŒØˆÕr   c                 óš   — | dd | j                   d   dz  …f   }| d| j                   d   dz  d …f   }t        j                  | |fd¬«      S )N.rŸ   ri   rÀ   )rÂ   rL   rÄ   )ÚxÚx1Úx2s      r   Ú_rotate_halfzSinkCache._rotate_halfã  sZ   € àˆsÐ&�a—g‘g˜b‘k QÑ&Ð&Ð&Ñ'ˆØˆs�A—G‘G˜B‘K 1Ñ$Ñ&Ð&Ñ'ˆÜ�y‰y˜2˜#˜r˜¨Ô+Ð+r   r   ÚcosÚsinc                 ó:   — ||z  | j                  |«      |z  z   }|S r   )rp  )r   r   rq  rr  Úrotated_key_statess        r   Ú_apply_key_rotary_pos_embz#SinkCache._apply_key_rotary_pos_embé  s+   € ð )¨3Ñ.°4×3DÑ3DÀZÓ3PÐSVÑ3VÑWÐØ!Ð!r   c                 óè  — |j                   d   | j                  v�r;|j                  t        j                  «      }|j                  t        j                  «      }|| j
                  |j                   d   z   d  }|| j
                  |j                   d     }|| j
                  |j                   d   z   d  }|| j
                  |j                   d     }||z  ||z  z   }| |z  ||z  z   }	|j                  |j                  «      j                  d«      |	j                  |j                  «      j                  d«      f| j                  |j                   d   <   | j                  |j                   d      S )Nr¿   r   )rÂ   ri  r=   rL   Úfloat32rg  r¦   Ú	unsqueeze)
r   r   rq  rr  Úoriginal_cosÚshifted_cosÚoriginal_sinÚshifted_sinÚrerotation_cosÚrerotation_sins
             r   Ú_get_rerotation_cos_sinz!SinkCache._get_rerotation_cos_sinï  so  € ð ×Ñ˜BÑ t×'DÑ'DÒDà—&‘&œŸ™Ó'ˆCØ—&‘&œŸ™Ó'ˆCð ˜t×3Ñ3°j×6FÑ6FÀrÑ6JÑJÐLÐMˆLØ˜d×2Ñ2°j×6FÑ6FÀrÑ6JÐ5JÐKˆKØ˜t×3Ñ3°j×6FÑ6FÀrÑ6JÑJÐLÐMˆLØ˜d×2Ñ2°j×6FÑ6FÀrÑ6JÐ5JÐKˆKØ)¨KÑ7¸,ÈÑ:TÑTˆNØ*˜]¨[Ñ8¸<È+Ñ;UÑUˆNð ×!Ñ! *×"2Ñ"2Ó3×=Ñ=¸aÓ@Ø×!Ñ! *×"2Ñ"2Ó3×=Ñ=¸aÓ@ðCˆD×)Ñ)¨*×*:Ñ*:¸2Ñ*>Ñ?ð ×,Ñ,¨Z×-=Ñ-=¸bÑ-AÑBÐBr   r!   c                 ól   — t        | j                  «      |k  ry| j                  |   j                  d   S rÇ   )r8   r9   rÂ   r*   s     r   r+   zSinkCache.get_seq_length  s2   € ô ˆt�~‰~Ó )Ò+ØØ�~‰~˜iÑ(×.Ñ.¨rÑ2Ð2r   c                 ó   — | j                   S )zfReturns the maximum sequence length of the cache object, in case of SinkCache it is the window length.)rf  r-   s    r   r.   zSinkCache.get_max_cache_shape  s   € à×!Ñ!Ð!r   r    r"   c                 ó²  — |€i }|j                  d«      }|j                  d«      }|j                  d«      }|duxr |du}|dk(  r"| xj                  |j                  d   z  c_        |rÄ|dk(  r¿|j                  «       dk(  r|| _        || _        n�| j                  €|d   | _        |d   | _        n|| j                  j                  d   | j                  k  rVt        j                  | j                  |d   gd¬	«      | _        t        j                  | j
                  |d   gd¬	«      | _        t        | j                  «      |k  r8| j                  j                  |«       | j                  j                  |«       �n<|j                  d   | j                  |«      z   | j                  k  rht        j                  | j                  |   |gd¬	«      | j                  |<   t        j                  | j                  |   |gd¬	«      | j                  |<   �n¦| j                  |   dd…dd…| j                   | j                  z   |j                  d   z   d…f   }	|r†| j                  || j                  d| j                   | j
                  d| j                   «      \  }
}|�|	d
d|…f   |	d
|d…f   }}	| j!                  |	|
|«      }	|�t        j                  |	fd¬	«      }	| j                  |   dd…dd…d| j                  …f   }t        j                  ||	|gd¬	«      | j                  |<   | j                  |   dd…dd…d| j                  …f   }| j                  |   dd…dd…| j                   | j                  z   |j                  d   z   d…f   }t        j                  |||gd¬	«      | j                  |<   | j                  |   | j                  |   fS )a;  
        Updates the cache with the new `key_states` and `value_states` for the layer `layer_idx`.

        Parameters:
            key_states (`torch.Tensor`):
                The new key states to cache.
            value_states (`torch.Tensor`):
                The new value states to cache.
            layer_idx (`int`):
                The index of the layer to cache the states for.
            cache_kwargs (`Dict[str, Any]`, `optional`):
                Additional arguments for the cache subclass. The following arguments can be used in `SinkCache`: `sin`,
                `cos` and `partial_rotation_size`. These arguments are used with models using RoPE, to recompute the
                rotation as the tokens are shifted.

        Return:
            A tuple containing the updated key and value states.
        Nrr  rq  Úpartial_rotation_sizer   r¿   ri   )r   .rÀ   .rŸ   )ÚgetrA   rÂ   rÁ   rj  rk  rf  rL   rÄ   r8   r9   r[   r>   r+   rg  r  ru  )r   r   r    r!   r"   rr  rq  rƒ  Ú
using_ropeÚkeys_to_keepr}  r~  Ú	keys_passÚ	sink_keysÚsink_valuesÚvalues_to_keeps                   r   r'   zSinkCache.update  sÄ  € ð6 ÐØˆLØ×Ñ˜uÓ%ˆØ×Ñ˜uÓ%ˆØ ,× 0Ñ 0Ð1HÓ IÐØ �_Ò8¨°D¨ˆ
ð ˜Š>Ø×Ò ×!1Ñ!1°"Ñ!5Ñ5Õñ ˜) qš.ð �w‰w‹y˜AŠ~Ø"%�”Ø"%�•à—?‘?Ð*Ø&)¨&¡k�D”OØ&)¨&¡k�D•OØ—_‘_×*Ñ*¨1Ñ-°×0BÑ0BÒBÜ&+§i¡i°·±À#ÀfÁ+Ð0NÐTUÔ&V�D”OÜ&+§i¡i°·±À#ÀfÁ+Ð0NÐTUÔ&V�D”Oô ˆt�~‰~Ó )Ò+à�N‰N×!Ñ! *Ô-Ø×Ñ×#Ñ# LÖ1à×Ñ˜bÑ! D×$7Ñ$7¸	Ó$BÑBÀT×EWÑEWÒWä(-¯	©	°4·>±>À)Ñ3LÈjÐ2YÐ_aÔ(bˆD�N‰N˜9Ñ%Ü*/¯)©)°T×5EÑ5EÀiÑ5PÐR^Ð4_ÐegÔ*hˆD×Ñ˜YÓ'ð  Ÿ>™>¨)Ñ4Ú’1�t×)Ñ)Ð)¨D×,@Ñ,@Ñ@À:×CSÑCSÐTVÑCWÑWÑYÐYñˆLñ
 Ø15×1MÑ1MØ §¡Ð0D°$×2DÑ2DÐ EÀtÇÁÐWkÐY]×YkÑYkÐGló2Ñ.� ð )Ð4à$ SÐ*@Ð+@Ð*@Ð%@ÑAØ$ SÐ*?Ñ*@Ð%@ÑAð #,�Lð  $×=Ñ=¸lÈNÐ\jÓk�Ø(Ð4Ü#(§9¡9¨l¸IÐ-FÈBÔ#O�Lð Ÿ™ yÑ1²!²QÐ8N¸$×:NÑ:NÐ8NÐ2NÑOˆIÜ(-¯	©	°9¸lÈJÐ2WÐ]_Ô(`ˆD�N‰N˜9Ñ%à×*Ñ*¨9Ñ5²aºÐ<R¸d×>RÑ>RÐ<RÐ6RÑSˆKØ!×-Ñ-¨iÑ8Ú’1�t×)Ñ)Ð)¨D×,@Ñ,@Ñ@À<×CUÑCUÐVXÑCYÑYÑ[Ð[ñˆNô +0¯)©)°[À.ÐR^Ð4_ÐegÔ*hˆD×Ñ˜YÑ'à�~‰~˜iÑ(¨$×*:Ñ*:¸9Ñ*EÐEÐEr   rF   r   )rG   rH   rI   rJ   Ú
is_slidingrN   r   Ústaticmethodrp  rL   rM   ru  r	   r  r   r+   r.   r   rO   r   r'   rR   rS   s   @r   re  re  µ  sO  ø„ ñð@ €Jð	 cð 	¸Cð 	ÀDõ 	ð ñ,ó ð,ð
"ØŸ,™,ð"Ø-2¯\©\ð"Ø@EÇÁð"à	�‰ó"ðCØŸ,™,ðCØ-2¯\©\ðCØ@EÇÁðCà	ˆu�|‰|˜UŸ\™\Ð)Ñ	*óCñ,3¨°©ð 3¸có 3ð" X¨c¡]ó "ð 26ñ^Fà—L‘Lð^Fð —l‘lð^Fð ð	^Fð
 ˜t C¨ H™~Ñ.ð^Fð 
ˆu�|‰|˜UŸ\™\Ð)Ñ	*÷^Fr   re  c                   ó†  ‡ — e Zd ZdZdZddej                  dfdedede	e   de
ej                  edf   dej                  d	e	eee
eej                  ef   f      d
dfˆ fd„Z	 ddej                   dej                   dede	eeef      d
eej                   ej                   f   f
d„Zdde	e   d
efd„Zd
e	e   fd„Zd„ Zˆ xZS )ÚStaticCachea¬	  
    Static Cache class to be used with `torch.compile(model)` and `torch.export()`.

    Parameters:
        config (`PretrainedConfig`):
            The configuration file defining the shape-related attributes required to initialize the static cache.
        max_batch_size (`int`):
            The maximum batch size with which the model will be used. Note that a new instance must be instantiated if a
            smaller batch size is used. If you are manually setting the batch size, make sure to take into account the
            number of beams if you are running beam search
        max_cache_len (`int`, *optional*):
            The maximum sequence length with which the model will be used.
        device (`torch.device` or `str`, *optional*):
            The device on which the cache should be initialized. If you're using more than 1 computation device, you
            should pass the `layer_device_map` argument instead.
        dtype (`torch.dtype`, *optional*, defaults to `torch.float32`):
            The default `dtype` to use when initializing the layer.
        layer_device_map (`Optional[Dict[int, Union[str, torch.device, int]]]]`, *optional*):
            Mapping between the layers and its device. This is required when you are manually initializing the cache
            and the model is split between different gpus. You can know which layers mapped to which device by
            checking the associated device_map: `model.hf_device_map`.


    Example:

        ```python
        >>> from transformers import AutoTokenizer, AutoModelForCausalLM, StaticCache

        >>> model = AutoModelForCausalLM.from_pretrained("meta-llama/Llama-2-7b-chat-hf")
        >>> tokenizer = AutoTokenizer.from_pretrained("meta-llama/Llama-2-7b-chat-hf")

        >>> inputs = tokenizer(text="My name is Llama", return_tensors="pt")

        >>> # Prepare a cache class and pass it to model's forward
        >>> # Leave empty space for 10 new tokens, which can be used when calling forward iteratively 10 times to generate
        >>> max_generated_length = inputs.input_ids.shape[1] + 10
        >>> past_key_values = StaticCache(config=model.config, max_batch_size=1, max_cache_len=max_generated_length, device=model.device, dtype=model.dtype)
        >>> outputs = model(**inputs, past_key_values=past_key_values, use_cache=True)
        >>> outputs.past_key_values # access cache filled with key/values from generation
        StaticCache()
        ```
    TNr`   Úmax_batch_sizer«   r;   r¦   Úlayer_device_mapr#   c                 óŠ  •— t         ‰| �  «        || _        |€|j                  n|| _        t        |d«      r|j                  n|j                  |j                  z  | _        || _	        t        |dd «      €|j                  n|j                  | _        g | _        g | _        | j                  | j                  | j                  | j                  f}|�t        j                  |«      nd }t!        |j"                  «      D ]Ä  }|�||   }	n|}	t        j$                  || j                  |	¬«      }
t        j$                  || j                  |	¬«      }t        j&                  j)                  |
«       t        j&                  j)                  |«       | j                  j+                  |
«       | j                  j+                  |«       ŒÆ y )NÚhead_dimÚnum_key_value_headsr6  )r   r   r�  Úmax_position_embeddingsr«   rD   r’  Úhidden_sizeÚnum_attention_headsÚ_dtyperõ   r“  r9   r>   rL   r;   r7   Únum_hidden_layersr9  Ú_dynamoÚmark_static_addressr[   )r   r`   r�  r«   r;   r¦   r�  Úcache_shaperÕ   Úlayer_deviceÚnew_layer_key_cacheÚnew_layer_value_cacher   s               €r   r   zStaticCache.__init__   sŽ  ø€ ô 	‰ÑÔØ,ˆÔØ?LÐ?T˜V×;Ò;ÐZgˆÔô  ' v¨zÔ:ˆF�OŠOÀ×@RÑ@RÐV\×VpÑVpÑ@pð 	Œð ˆŒô �vÐ4°dÓ;ÐCð ×&Ò&à×+Ñ+ð 	Ô ð .0ˆŒØ/1ˆÔà×*Ñ*¨D×,DÑ,DÀd×FXÑFXÐZ^×ZgÑZgÐhˆØ)/Ð);”—‘˜fÔ%ÀˆÜ˜×1Ñ1Ó2ò 	;ˆCØÐ+Ø/°Ñ4‘à%�Ü"'§+¡+¨kÀÇÁÐUaÔ"bÐÜ$)§K¡K°À4Ç;Á;ÐWcÔ$dÐ!ô �M‰M×-Ñ-Ð.AÔBÜ�M‰M×-Ñ-Ð.CÔDØ�N‰N×!Ñ!Ð"5Ô6Ø×Ñ×#Ñ#Ð$9Õ:ñ	;r   r   r    r!   r"   c                 óÎ  — |€i }|j                  d«      }| j                  |   }| j                  |   }|j                  |j                  «      }|j                  |j                  «      }|€&|j                  |«       |j                  |«       ||fS 	 |j                  d||«       |j                  d||«       ||fS # t        $ r ||dd…dd…|f<   ||dd…dd…|f<   Y ||fS w xY w)a'  
        Updates the cache with the new `key_states` and `value_states` for the layer `layer_idx`.
        It is VERY important to index using a tensor, otherwise you introduce a copy to the device.

        Parameters:
            key_states (`torch.Tensor`):
                The new key states to cache.
            value_states (`torch.Tensor`):
                The new value states to cache.
            layer_idx (`int`):
                The index of the layer to cache the states for.
            cache_kwargs (`Dict[str, Any]`, `optional`):
                Additional arguments for the cache subclass. The `StaticCache` needs the `cache_position` input
                to know how where to write in the cache.

        Return:
            A tuple containing the updated key and value states.
        NÚcache_positionri   )r„  r9   r>   r=   r¦   Úcopy_Úindex_copy_r&   )r   r   r    r!   r"   r   Úk_outÚv_outs           r   r'   zStaticCache.updateÌ  sþ   € ð2 ÐØˆLØ%×)Ñ)Ð*:Ó;ˆØ—‘˜yÑ)ˆØ× Ñ  Ñ+ˆØ—]‘] 5§;¡;Ó/ˆ
Ø#—‘ u§{¡{Ó3ˆàÐ!Ø�K‰K˜
Ô#Ø�K‰K˜Ô%ð �eˆ|Ðð;Ø×!Ñ! ! ^°ZÔ@Ø×!Ñ! ! ^°\ÔBð �eˆ|Ðøô 'ò ;à.8�’aš˜NÐ*Ñ+Ø.:�’aš˜NÐ*Ò+à�eˆ|Ðð;ús   Â&B= Â=!C$Ã#C$c                 ób   — | j                   |   d   j                  d¬«      j                  «       S )úMReturns the sequence length of the cached states that were seen by the model.©r   r   rŸ   rÀ   )r9   ÚanyÚsumr*   s     r   r+   zStaticCache.get_seq_lengthþ  s/   € ð
 —‘˜yÑ)¨$Ñ/×3Ñ3¸Ð3Ó;×@Ñ@ÓBÐBr   c                 ó   — | j                   S r   ©r«   r-   s    r   r.   zStaticCache.get_max_cache_shape  ó   € Ø×!Ñ!Ð!r   c                 ó¾   — t        t        | j                  «      «      D ]<  }| j                  |   j                  «        | j                  |   j                  «        Œ> y©ú4Resets the cache values while preserving the objectsN©r7   r8   r9   Úzero_r>   r*   s     r   ÚresetzStaticCache.reset  óK   € äœs 4§>¡>Ó2Ó3ò 	0ˆIà�N‰N˜9Ñ%×+Ñ+Ô-Ø×Ñ˜YÑ'×-Ñ-Õ/ñ	0r   r   rF   )rG   rH   rI   rJ   rK   rL   rw  r   rN   r   r
   r;   rO   r¦   r   r   rM   r   r	   r'   r+   r.   r²  rR   rS   s   @r   rŽ  rŽ  r  s4  ø„ ñ)ðV €Nð (,Ø15Ø"Ÿ]™]ØOSñ*;à ð*;ð ð*;ð   ‘}ð	*;ð
 �e—l‘l C¨Ð-Ñ.ð*;ð �{‰{ð*;ð # 4¨¨U°3¸¿¹ÀcÐ3IÑ-JÐ(JÑ#KÑLð*;ð 
õ*;ðb 26ñ0à—L‘Lð0ð —l‘lð0ð ð	0ð
 ˜t C¨ H™~Ñ.ð0ð 
ˆu�|‰|˜UŸ\™\Ð)Ñ	*ó0ñdC¨°©ð C¸có Cð" X¨c¡]ó "ö0r   rŽ  c                   ór  ‡ — e Zd ZdZdZdZddej                  dfdede	de
e	   deej                  edf   dej                  d	e
ee	eeej                  e	f   f      d
dfˆ fd„Z	 ddej"                  dej"                  de	de
eeef      d
eej"                  ej"                  f   f
d„Zd
e
e	   fd„Zd„ Zˆ xZS )ÚSlidingWindowCachea;  
    Sliding Window Cache class to be used with `torch.compile` for models like Mistral that support sliding window attention.
    Every time when we try to update the cache, we compute the `indices` based on `cache_position >= self.config.sliding_window - 1`,
    if true(which means the cache can not hold all the old key value states and new states together because of the sliding window constraint),
    we need to do a cycle shift based on `indices` to replace the oldest states by the new key value states passed in.

    The `to_shift` is only true once we are above sliding_window. Thus with `sliding_window==64`:

    indices = (slicing + to_shift[-1].int()-1) % self.config.sliding_window
    tensor([ 1,  2,  3,  4,  5,  6,  7,  8,  9, 10, 11, 12, 13, 14, 15, 16, 17, 18,
        19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36,
        37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54,
        55, 56, 57, 58, 59, 60, 61, 62, 63,  0])

    We overwrite the cache using these, then we always write at cache_position (clamped to `sliding_window`)

    Parameters:
        config (`PretrainedConfig`):
            The configuration file defining the shape-related attributes required to initialize the static cache.
        max_batch_size (`int`):
            The maximum batch size with which the model will be used. Note that a new instance must be instantiated if a
            smaller batch size is used.
        max_cache_len (`int`, *optional*):
            The maximum sequence length with which the model will be used.
        device (`torch.device` or `str`, *optional*):
            The device on which the cache should be initialized. If you're using more than 1 computation device, you
            should pass the `layer_device_map` argument instead.
        dtype (`torch.dtype`, *optional*, defaults to `torch.float32`):
            The default `dtype` to use when initializing the layer.
        layer_device_map (`Optional[Dict[int, Union[str, torch.device, int]]]]`, *optional*):
            Mapping between the layers and its device. This is required when you are manually initializing the cache
            and the model is split between different gpus. You can know which layers mapped to which device by
            checking the associated device_map: `model.hf_device_map`.

    Example:

        ```python
        >>> from transformers import AutoTokenizer, AutoModelForCausalLM, SlidingWindowCache

        >>> model = AutoModelForCausalLM.from_pretrained("mistralai/Mistral-7B-Instruct-v0.3")
        >>> tokenizer = AutoTokenizer.from_pretrained("mistralai/Mistral-7B-Instruct-v0.3")

        >>> inputs = tokenizer(text="My name is Mistral", return_tensors="pt")

        >>> # Prepare a cache class and pass it to model's forward
        >>> # Leave empty space for 10 new tokens, which can be used when calling forward iteratively 10 times to generate
        >>> max_generated_length = inputs.input_ids.shape[1] + 10
        >>> past_key_values = SlidingWindowCache(config=model.config, max_batch_size=1, max_cache_len=max_generated_length, device=model.device, dtype=model.dtype)
        >>> outputs = model(**inputs, past_key_values=past_key_values, use_cache=True)
        >>> outputs.past_key_values # access cache filled with key/values from generation
        SlidingWindowCache()
        ```
    TNr`   r�  r«   r;   r¦   r�  r#   c                 ó¢   •— t        |d«      r|j                  €t        d«      ‚t        |j                  |«      }t        ‰| �  ||||||¬«       y )NÚsliding_windowúÑSetting `cache_implementation` to 'sliding_window' requires the model config supporting sliding window attention, please check if there is a `sliding_window` field in the model config and it's not set to None.)r`   r�  r«   r;   r¦   r�  )rD   r·  r    Úminr   r   )r   r`   r�  r«   r;   r¦   r�  r   s          €r   r   zSlidingWindowCache.__init__J  se   ø€ ô �vÐ/Ô0°F×4IÑ4IÐ4QÜð3óð ô
 ˜F×1Ñ1°=ÓAˆÜ‰ÑØØ)Ø'ØØØ-ð 	õ 	
r   r   r    r!   r"   c                 óÄ  — |€i }|j                  d«      }| j                  |   }| j                  |   }|j                  |j                  «      }|j                  |j                  «      }|j
                  d   | j                  kD  rj|d d …d d …| j                   d …d d …f   }|d d …d d …| j                   d …d d …f   }| j                  |xx   |z  cc<   | j                  |xx   |z  cc<   ||fS t        j                  | j                  t        j                  |j                  ¬«      j                  d«      }|j                  d| j                  dz
  «      }|| j                  dz
  k\  }	||	d   j                  «       z   dz
  | j                  z  }
|d d …d d …|
f   }|d d …d d …|
f   }	 |j                  d||«       |j                  d||«       | j                  |   j!                  «        | j                  |   j!                  «        | j                  |xx   |z  cc<   | j                  |xx   |z  cc<   ||fS # t        $ r ||d d …d d …|f<   ||d d …d d …|f<   Y Œ�w xY w)Nr   r   r6  r   rŸ   ri   )r„  r9   r>   r=   r¦   rÂ   r«   rL   ÚonesÚlongr;   ÚcumsumÚclamprN   r¢  r&   r±  )r   r   r    r!   r"   r   r£  r¤  ÚslicingÚto_shiftrë   s              r   r'   zSlidingWindowCache.updatec  sW  € ð ÐØˆLØ%×)Ñ)Ð*:Ó;ˆØ—‘˜yÑ)ˆØ× Ñ  Ñ+ˆØ—]‘] 5§;¡;Ó/ˆ
Ø#—‘ u§{¡{Ó3ˆð ×Ñ Ñ" T×%7Ñ%7Ò7Øšq¢! d×&8Ñ&8Ð%8Ñ%:ºAÐ=Ñ>ˆEØ ¢¢A¨×(:Ñ(:Ð':Ñ'<ºaÐ!?Ñ@ˆEà�N‰N˜9Ó%¨Ñ.Ó%Ø×Ñ˜YÓ'¨5Ñ0Ó'ð ˜|Ð+Ð+ä—*‘*˜T×/Ñ/´u·z±zÈ,×J]ÑJ]Ô^×eÑeÐfgÓhˆØ'×-Ñ-¨a°×1CÑ1CÀaÑ1GÓHˆØ! T×%7Ñ%7¸!Ñ%;Ñ;ˆØ˜X b™\×-Ñ-Ó/Ñ/°!Ñ3°t×7IÑ7IÑIˆà’aš˜G�mÑ$ˆØ’aš˜G�mÑ$ˆð	7Ø×Ñ˜a °Ô<Ø×Ñ˜a °Ô>ð 	�‰�yÑ!×'Ñ'Ô)Ø×Ñ˜Ñ#×)Ñ)Ô+à�‰�yÓ! UÑ*Ó!Ø×Ñ˜Ó# uÑ,Ó#à�eˆ|Ðøô #ò 	7à*4ˆE’!’Q˜Ð&Ñ'Ø*6ˆE’!’Q˜Ð&Ó'ð	7ús   Æ)&H; È;!IÉIc                 ó   — | j                   S r   r«  r-   s    r   r.   z&SlidingWindowCache.get_max_cache_shape–  r¬  r   c                 ó¾   — t        t        | j                  «      «      D ]<  }| j                  |   j                  «        | j                  |   j                  «        Œ> y r   r°  r*   s     r   r²  zSlidingWindowCache.reset™  sK   € Üœs 4§>¡>Ó2Ó3ò 	0ˆIà�N‰N˜9Ñ%×+Ñ+Ô-Ø×Ñ˜YÑ'×-Ñ-Õ/ñ	0r   r   )rG   rH   rI   rJ   r‹  rK   rL   rw  r   rN   r   r
   r;   rO   r¦   r   r   rM   r   r	   r'   r.   r²  rR   rS   s   @r   rµ  rµ    s  ø„ ñ4ðl €JØ€Nð (,Ø15Ø"Ÿ]™]ØOSñ
à ð
ð ð
ð   ‘}ð	
ð
 �e—l‘l C¨Ð-Ñ.ð
ð �{‰{ð
ð # 4¨¨U°3¸¿¹ÀcÐ3IÑ-JÐ(JÑ#KÑLð
ð 
õ
ð< 26ñ1à—L‘Lð1ð —l‘lð1ð ð	1ð
 ˜t C¨ H™~Ñ.ð1ð 
ˆu�|‰|˜UŸ\™\Ð)Ñ	*ó1ðf" X¨c¡]ó "ö0r   rµ  c                   ó¦  ‡ — e Zd ZdZdedefˆ fd„Zdedeee	j                        fd„Zd„ Zdeee	j                     ee	j                     f   fd	„Ze	 dd
eeee	j                            dd fd„«       Zddee   defd„Zd„ Zde	j(                  fd„Zdefd„Zdefd„Zdededdfd„Zeded    dd fd„«       Zdefd„Zde	j                  fd„Zˆ xZS ) ÚEncoderDecoderCachea  
    Base, abstract class for all encoder-decoder caches. Can be used to hold combinations of self-attention and
    cross-attention caches.

    Example:

        ```python
        >>> from transformers import AutoProcessor, AutoModelForCausalLM, DynamicCache, EncoderDecoderCache

        >>> model = AutoModelForCausalLM.from_pretrained("openai/whisper-small")
        >>> processor = AutoProcessor.from_pretrained("openai/whisper-small")

        >>> inputs = processor(audio=YOUR-AUDIO, return_tensors="pt")

        >>> # Prepare cache classes for encoder and decoder and pass it to model's forward
        >>> self_attention_cache = DynamicCache()
        >>> cross_attention_cache = DynamicCache()
        >>> past_key_values = EncoderDecoderCache(self_attention_cache, cross_attention_cache)
        >>> outputs = model(**inputs, past_key_values=past_key_values, use_cache=True)
        >>> outputs.past_key_values # access cache filled with key/values from generation
        EncoderDecoderCache()
        ```

    Úself_attention_cacheÚcross_attention_cachec                 ó  •— t         ‰| �  «        || _        || _        t	        | j                  dd«      | _        i | _        t        t        |j                  «      «      D ],  }t        |j                  |«      dkD  «      | j                  |<   Œ. y )NrK   Fr   )r   r   rÅ  rÆ  rõ   rK   Ú
is_updatedr7   r8   r9   Úboolr+   )r   rÅ  rÆ  r!   r   s       €r   r   zEncoderDecoderCache.__init__º  s†   ø€ Ü‰ÑÔØ$8ˆÔ!Ø%:ˆÔ"Ü% d×&?Ñ&?ÐAQÐSXÓYˆÔàˆŒÜœsÐ#8×#BÑ#BÓCÓDò 	cˆIÜ)-Ð.C×.RÑ.RÐS\Ó.]Ð`aÑ.aÓ)bˆD�O‰O˜IÒ&ñ	cr   r!   r#   c                 ó  — |t        | «      k  rb| j                  j                  |   | j                  j                  |   | j                  j                  |   | j                  j                  |   fS t        dt        | «      › d|› �«      ‚r´   )r8   rÅ  r9   r>   rÆ  r·   r*   s     r   r¸   zEncoderDecoderCache.__getitem__Ä  s‹   € ð
 ”s˜4“yÒ à×)Ñ)×3Ñ3°IÑ>Ø×)Ñ)×5Ñ5°iÑ@Ø×*Ñ*×4Ñ4°YÑ?Ø×*Ñ*×6Ñ6°yÑAð	ð ô ˜_¬S°«Y¨KÐ7eÐfoÐepÐqÓrÐrr   c                 ó,   — t        | j                  «      S r¼   )r8   rÅ  r-   s    r   r½   zEncoderDecoderCache.__len__Ó  s   € ô
 �4×,Ñ,Ó-Ð-r   c                 ó  — d}t        | j                  «      dkD  rOt        | j                  j	                  «       | j                  j	                  «       «      D ]  \  }}|||z   fz  }Œ |S | j                  j	                  «       }|S )z\Converts the `EncoderDecoderCache` instance into  its equivalent in the legacy cache format.rX   r   )r8   rÆ  ÚziprÅ  rÍ   )r   rÌ   Ú	self_attnÚ
cross_attns       r   rÍ   z#EncoderDecoderCache.to_legacy_cacheÚ  sŽ   € àˆÜˆt×)Ñ)Ó*¨QÒ.Ü),Ø×)Ñ)×9Ñ9Ó;¸T×=WÑ=W×=gÑ=gÓ=ió*ò :Ñ%�	˜:ð  ¨ZÑ!7Ð 9Ñ9‘ð:ð Ðð  ×4Ñ4×DÑDÓFˆLØÐr   rÎ   c                 óR  —  | t        «       t        «       ¬«      }|�Št        t        |«      «      D ]s  }||   dd \  }}|j                  j	                  |||«       t        ||   «      dkD  sŒ=||   dd \  }}|j
                  j	                  |||«       d|j                  |<   Œu |S )zUConverts a cache in the legacy cache format into an equivalent `EncoderDecoderCache`.)rÅ  rÆ  Nri   T)r¯   r7   r8   rÅ  r'   rÆ  rÈ  rÐ   s         r   rÒ   z%EncoderDecoderCache.from_legacy_cacheæ  s¿   € ñ
 Ü!-£Ü".£.ô
ˆð Ð&Ü"¤3 Ó#7Ó8ò 7�	Ø+:¸9Ñ+EÀbÀqÐ+IÑ(�
˜LØ×*Ñ*×1Ñ1°*¸lÈIÔVÜ� yÑ1Ó2°QÓ6Ø/>¸yÑ/IÈ!È"Ð/MÑ,�J Ø×/Ñ/×6Ñ6°zÀ<ÐQZÔ[Ø26�E×$Ñ$ YÒ/ð7ð ˆr   c                 ó8   — | j                   j                  |«      S )r)   )rÅ  r+   r*   s     r   r+   z"EncoderDecoderCache.get_seq_lengthù  s   € ð ×(Ñ(×7Ñ7¸	ÓBÐBr   c                 óâ  — t        | j                  d«      r| j                  j                  «        t        | j                  d«      r| j                  j                  «        nnt        | j                  d«      sXt        | j                  d«      sBt	        d| j                  j                  «       › d| j                  j                  «       › d�«      ‚| j                  D ]  }d| j                  |<   Œ y )Nr²  z·Neither self nor cross-attention cache have valid `.reset()` methods. `.reset()` should only be called on compatible cache classes, such as `StaticCache` or `SlidingWindowCache`. Got ú" for the self attention cache and ú for the cross attention cache.F)rD   rÅ  r²  rÆ  r    Ú__str__rÈ  r*   s     r   r²  zEncoderDecoderCache.resetþ  sÑ   € Ü�4×,Ñ,¨gÔ6Ø×%Ñ%×+Ñ+Ô-Ü�4×-Ñ-¨wÔ7Ø×&Ñ&×,Ñ,Õ.Ü˜×2Ñ2°GÔ<ÄWÈT×MgÑMgÐipÔEqÜðà×0Ñ0×8Ñ8Ó:Ð;Ð;]Ø×-Ñ-×5Ñ5Ó7Ð8Ð8WðYóð ð Ÿ™ò 	/ˆIØ).ˆD�O‰O˜IÒ&ñ	/r   r4   c                 óp   — | j                   j                  |«       | j                  j                  |«       y)r6   N)rÅ  r?   rÆ  r*  s     r   r?   z!EncoderDecoderCache.reorder_cache  s*   € à×!Ñ!×/Ñ/°Ô9Ø×"Ñ"×0Ñ0°Õ:r   Úmethodc           	      óö   — t        | j                  t        «      rt        | j                  t        «      sEt	        d|› d| j                  j                  «       › d| j                  j                  «       › d�«      ‚y )Nú`z)` is only defined for dynamic cache, got rÓ  rÔ  )ró   rÅ  r¯   rÆ  r    rÕ  )r   r×  s     r   Úcheck_dynamic_cachez'EncoderDecoderCache.check_dynamic_cache  sw   € ä�t×0Ñ0´,Ô?Ü˜4×5Ñ5´|ÔDäØ�F�8ÐDÀT×E^ÑE^×EfÑEfÓEhÐDið j'Ø'+×'AÑ'A×'IÑ'IÓ'KÐ&LÐLkðmóð ð Er   Úmaximum_lengthc                 ó„   — | j                  | j                  j                  «       | j                  j                  |«       y)zÔCrop the past key values up to a new `maximum_length` in terms of tokens. `maximum_length` can also be
        negative to remove `maximum_length` tokens. This is used in assisted decoding and contrastive search.N)rÚ  rÖ   rG   rÅ  )r   rÛ  s     r   rÖ   zEncoderDecoderCache.crop  s0   € ð 	× Ñ  §¡×!3Ñ!3Ô4Ø×!Ñ!×&Ñ& ~Õ6r   r×   rØ   zList[EncoderDecoderCache]c                 ó"  — | j                  | j                  j                  «       | j                  j                  ||«      }| j                  j                  ||«      }g }t        ||«      D ]   \  }}|j                  t        ||«      «       Œ" |S )rÚ   )rÚ  rÞ   rG   rÅ  rÆ  rÍ  r[   rÄ  )r   r×   rØ   rÅ  rÆ  rÛ   rÎ  rÏ  s           r   rÞ   zEncoderDecoderCache.batch_split#  s�   € ð 	× Ñ  ×!1Ñ!1×!:Ñ!:Ô;Ø#×8Ñ8×DÑDÀ_ÐV`ÓaÐØ $× :Ñ :× FÑ FÀÐXbÓ cÐàˆÜ%(Ð)=Ð?TÓ%Uò 	CÑ!ˆI�zØ�J‰JÔ*¨9°jÓAÕBð	Càˆ
r   rß   c                 óÌ  — t        «       }t        «       }t        t        |d   «      «      D �]  }t        j                  |D �cg c]  }|j
                  j                  |   ‘Œ c}d¬«      }t        j                  |D �cg c]  }|j
                  j                  |   ‘Œ c}d¬«      }|j                  |||«       t        j                  |D �cg c]  }|j                  j                  |   ‘Œ c}d¬«      }t        j                  |D �cg c]  }|j                  j                  |   ‘Œ c}d¬«      }|j                  |||«       �Œ  | ||«      S c c}w c c}w c c}w c c}w rá   )
r¯   r7   r8   rL   rÄ   rÅ  r9   r>   r'   rÆ  )r]   rß   rÅ  rÆ  rÕ   râ   rã   rä   s           r   rå   z%EncoderDecoderCache.from_batch_splits/  s6  € ô  ,›~ÐÜ ,£ÐÜœ˜V A™Y›Ó(ó 	HˆCÜŸ™Ð_eÖ#fÐT[ G×$@Ñ$@×$JÑ$JÈ3Ó$OÒ#fÐlmÔnˆJÜ Ÿ9™9ÐciÖ%jÐX_ g×&BÑ&B×&NÑ&NÈsÓ&SÒ%jÐpqÔrˆLØ ×'Ñ'¨
°LÀ#ÔFäŸ™Ð`fÖ#gÐU\ G×$AÑ$A×$KÑ$KÈCÓ$PÒ#gÐmnÔoˆJÜ Ÿ9™9ÐdjÖ%kÐY` g×&CÑ&C×&OÑ&OÐPSÓ&TÒ%kÐqrÔsˆLØ!×(Ñ(¨°\À3ÖGð	Hñ Ð'Ð)>Ó?Ð?ùò $gùÚ%jùò $hùÚ%ks   Á E
Á? E
Ã E
Ä
 E!
ræ   c                 óº   — | j                  | j                  j                  «       | j                  j                  |«       | j                  j                  |«       y)rè   N)rÚ  rê   rG   rÅ  rÆ  )r   ræ   s     r   rê   z+EncoderDecoderCache.batch_repeat_interleave?  sD   € à× Ñ  ×!=Ñ!=×!FÑ!FÔGØ×!Ñ!×9Ñ9¸'ÔBØ×"Ñ"×:Ñ:¸7ÕCr   rë   c                 óº   — | j                  | j                  j                  «       | j                  j                  |«       | j                  j                  |«       y)rí   N)rÚ  rî   rG   rÅ  rÆ  )r   rë   s     r   rî   z(EncoderDecoderCache.batch_select_indicesE  sD   € à× Ñ  ×!:Ñ!:×!CÑ!CÔDØ×!Ñ!×6Ñ6°wÔ?Ø×"Ñ"×7Ñ7¸Õ@r   r   rF   )rG   rH   rI   rJ   r   r   rN   r   r	   rL   rM   r¸   r½   rÍ   r„   r   rï   rÒ   r+   r²  rP   r?   rO   rÚ  rÖ   rÞ   rå   rê   rî   rR   rS   s   @r   rÄ  rÄ     s\  ø„ ñð2c¨Uð cÈ5õ cðs Sð s¨T°%¸¿¹Ñ2EÑ-Fó sò.ð
  u¨U¯\©\Ñ':¸EÀ%Ç,Á,Ñ<OÐ'OÑ!Pó 
ð àJNñØ& u¨U°5×3DÑ3DÑ-EÑ'FÑGðà	òó ðñ$C¨°©ð C¸có Cò
/ð; e×&6Ñ&6ó ;ð
¨#ó ð7 3ó 7ð
¨3ð 
¸Cð 
ÐD_ó 
ð ð@ tÐ,AÑ'Bð @ÐG\ò @ó ð@ðD¨só DðA¨E¯L©L÷ Ar   rÄ  c                   óŠ  ‡ — e Zd ZdZddej
                  dfdededee   de	ej                  edf   dej                  deeee	eej                  ef   f      d	dfˆ fd
„Zd„ Zd„ Z	 ddej"                  dej"                  dedeeeef      d	eej"                  ej"                  f   f
d„Zd	ee   fd„Zddee   fd„Zd„ Zˆ xZS )ÚHybridCachea:
  
    Hybrid Cache class to be used with `torch.compile` for Gemma2 models that alternate between a local sliding window attention
    and global attention in every other layer. Under the hood, Hybrid Cache leverages ["SlidingWindowCache"] for sliding window attention
    and ["StaticCache"] for global attention. For more information, see the documentation of each subcomponeent cache class.

    Parameters:
        config (`PretrainedConfig):
            The configuration file defining the shape-related attributes required to initialize the static cache.
        max_batch_size (`int`):
            The maximum batch size with which the model will be used. Note that a new instance must be instantiated if a
            smaller batch size is used.
        max_cache_len (`int`, *optional*):
            The maximum sequence length with which the model will be used.
        device (`torch.device` or `str`, *optional*):
            The device on which the cache should be initialized. If you're using more than 1 computation device, you
            should pass the `layer_device_map` argument instead.
        dtype (torch.dtype, *optional*, defaults to `torch.float32`):
            The default `dtype` to use when initializing the layer.
        layer_device_map (`Optional[Dict[int, Union[str, torch.device, int]]]]`, *optional*):
            Mapping between the layers and its device. This is required when you are manually initializing the cache
            and the model is split between different gpus. You can know which layers mapped to which device by
            checking the associated device_map: `model.hf_device_map`.

    Example:

        ```python
        >>> from transformers import AutoTokenizer, AutoModelForCausalLM, HybridCache

        >>> model = AutoModelForCausalLM.from_pretrained("google/gemma-2-2b")
        >>> tokenizer = AutoTokenizer.from_pretrained("google/gemma-2-2b")

        >>> inputs = tokenizer(text="My name is Gemma", return_tensors="pt")

        >>> # Prepare a cache class and pass it to model's forward
        >>> # Leave empty space for 10 new tokens, which can be used when calling forward iteratively 10 times to generate
        >>> max_generated_length = inputs.input_ids.shape[1] + 10
        >>> past_key_values = HybridCache(config=model.config, max_batch_size=1, max_cache_len=max_generated_length, device=model.device, dtype=model.dtype)
        >>> outputs = model(**inputs, past_key_values=past_key_values, use_cache=True)
        >>> outputs.past_key_values # access cache filled with key/values from generation
        HybridCache()
        ```
    Nr`   r�  r«   r;   r¦   r�  r#   c           	      óF  •— t         ‰| �  «        t        |d«      r|j                  €t	        d«      ‚|| _        || _        t        |d«      r|j                  n|j                  |j                  z  | _        || _
        |j                  €|j                  n|j                  | _        t        |d«      r|j                  nd}t        j                  t        |j                   «      D �cg c]  }t#        |dz   |z  «      ‘Œ c}t        j"                  ¬«      | _        g | _        g | _        | j                  | j                  || j                  f}	| j                  | j                  t+        |j                  |«      | j                  f}
|�%t-        |t.        «      rt        j0                  |«      nd }t        |j                   «      D ]×  }|�||   }n|}| j$                  |   s|	n|
}t        j2                  || j                  |¬«      }t        j2                  || j                  |¬«      }t        j4                  j7                  |«       t        j4                  j7                  |«       | j&                  j9                  |«       | j(                  j9                  |«       ŒÙ y c c}w )	Nr·  r¸  r’  Úsliding_window_patternri   r   )r¦   r6  )r   r   rD   r·  r    r«   r�  r’  r•  r–  r—  r“  rä  rL   rÃ   r7   r˜  rÉ  r‹  r9   r>   r¹  ró   rO   r;   r9  r™  rš  r[   )r   r`   r�  r«   r;   r¦   r�  Úlayer_switchrÜ   Úglobal_cache_shapeÚsliding_cache_shaperœ  r›  r�  rž  r   s                  €r   r   zHybridCache.__init__|  sO  ø€ ô 	‰ÑÔÜ�vÐ/Ô0°F×4IÑ4IÐ4QÜð3óð ð
 +ˆÔØ,ˆÔô  ' v¨zÔ:ˆF�OŠOÀ×@RÑ@RÐV\×VpÑVpÑ@pð 	Œð ˆŒà*0×*DÑ*DÐ*LˆF×&Ò&ÐRX×RlÑRlð 	Ô ô 9@ÀÐH`Ô8a�v×4Ò4ÐghˆÜŸ,™,Ü38¸×9QÑ9QÓ3RÖS¨aŒT�1�q‘5˜LÑ(Õ)ÒSÔ[`×[eÑ[eô
ˆŒð .0ˆŒØ/1ˆÔØ"×1Ñ1°4×3KÑ3KÈ]Ð\`×\iÑ\iÐjÐà×ÑØ×$Ñ$Ü�×%Ñ% }Ó5Ø�M‰Mð	
Ðð *0Ð);Ä
È6ÔSVÔ@W”—‘˜fÔ%Ð]aˆÜ�v×/Ñ/Ó0ò 	;ˆAØÐ+Ø/°Ñ2‘à%�ð 59·O±OÀAÒ4FÑ,ÐL_ˆKÜ"'§+¡+¨kÀÇÁÐUaÔ"bÐÜ$)§K¡K°À4Ç;Á;ÐWcÔ$dÐ!Ü�M‰M×-Ñ-Ð.AÔBÜ�M‰M×-Ñ-Ð.CÔDØ�N‰N×!Ñ!Ð"5Ô6Ø×Ñ×#Ñ#Ð$9Õ:ñ	;ùò Ts   Ã'Jc                 óú  — |j                   d   |kD  rV|d d …d d …| d …d d …f   }|d d …d d …| d …d d …f   }| j                  |xx   |z  cc<   | j                  |xx   |z  cc<   ||fS t        j                  |t        j
                  |j                  ¬«      j                  d«      }|j                  d|dz
  «      }||dz
  k\  }	||	d   j                  «       z   dz
  |z  }
|d d …d d …|
f   }|d d …d d …|
f   }||d d …d d …|f<   ||d d …d d …|f<   | j                  |   j                  «        | j                  |   j                  «        | j                  |xx   |z  cc<   | j                  |xx   |z  cc<   ||fS )Nr   r6  r   rŸ   )rÂ   r9   r>   rL   r»  r¼  r;   r½  r¾  rN   r±  )r   r   r!   r   r    r£  r¤  r«   r¿  rÀ  rë   s              r   Ú_sliding_updatezHybridCache._sliding_updateµ  s‹  € Ø×Ñ Ñ" ]Ò2Øšq¢! m ^¡_²aÐ7Ñ8ˆEØ ¢¢A¨ ~¡ºÐ!9Ñ:ˆEà�N‰N˜9Ó%¨Ñ.Ó%Ø×Ñ˜YÓ'¨5Ñ0Ó'ð ˜|Ð+Ð+ä—*‘*˜]´%·*±*À\×EXÑEXÔY×`Ñ`ÐabÓcˆØ'×-Ñ-¨a°ÀÑ1BÓCˆØ! ]°QÑ%6Ñ6ˆØ˜X b™\×-Ñ-Ó/Ñ/°!Ñ3°}ÑDˆØ’aš˜G�mÑ$ˆØ’aš˜G�mÑ$ˆà&0ˆŠa’�NÐ"Ñ#Ø&2ˆŠa’�NÐ"Ñ#à�‰�yÑ!×'Ñ'Ô)Ø×Ñ˜Ñ#×)Ñ)Ô+à�‰�yÓ! UÑ*Ó!Ø×Ñ˜Ó# uÑ,Ó#Ø�eˆ|Ðr   c                 óv   — ||d d …d d …|f<   ||d d …d d …|f<   || j                   |<   || j                  |<   ||fS r   rò   ©r   r   r!   r   r    r£  r¤  r«   s           r   Ú_static_updatezHybridCache._static_updateÑ  óL   € Ø&0ˆŠa’�NÐ"Ñ#Ø&2ˆŠa’�NÐ"Ñ#à$)ˆ�‰�yÑ!Ø&+ˆ×Ñ˜Ñ#Ø�eˆ|Ðr   r   r    r!   r"   c           
      óÎ  — |€i }|j                  d«      }|j                  d«      }| j                  |   j                  |j                  k7  r5| j                  |   j                  |j                  «      | j                  |<   | j                  |   j                  |j                  k7  r5| j                  |   j                  |j                  «      | j                  |<   | j                  |   }| j                  |   }|j                  |j
                  «      }|j                  |j
                  «      }|r| j                  }	n| j                  }	 |	|||||||j                  d   «      S )Nr   r·  ri   )	r„  r9   r;   r=   r>   r¦   ré  rì  rÂ   )
r   r   r    r!   r"   r   r·  r£  r¤  Ú	update_fns
             r   r'   zHybridCache.updateÙ  sH  € ð ÐØˆLØ%×)Ñ)Ð*:Ó;ˆØ%×)Ñ)Ð*:Ó;ˆð �>‰>˜)Ñ$×+Ñ+¨z×/@Ñ/@Ò@Ø(,¯©°yÑ(A×(DÑ(DÀZ×EVÑEVÓ(WˆD�N‰N˜9Ñ%Ø×Ñ˜IÑ&×-Ñ-°×1DÑ1DÒDØ*.×*:Ñ*:¸9Ñ*E×*HÑ*HÈ×I\ÑI\Ó*]ˆD×Ñ˜YÑ'à—‘˜yÑ)ˆØ× Ñ  Ñ+ˆØ—]‘] 5§;¡;Ó/ˆ
Ø#—‘ u§{¡{Ó3ˆáØ×,Ñ,‰Ià×+Ñ+ˆIáØØØØØØØ�K‰K˜‰Nó
ð 	
r   c                 ó   — | j                   S r   r«  r-   s    r   r.   zHybridCache.get_max_cache_shape   r¬  r   c                 ó‚   — |dk7  rt        d«      ‚| j                  |   d   j                  d¬«      j                  «       S ©Nr   z�`get_seq_length` on `HybridCache` may get inconsistent results depending on the layer index. Using the `layer_idx` argument is not supported.r§  rŸ   rÀ   )r    r9   r¨  r©  r*   s     r   r+   zHybridCache.get_seq_length  sK   € ð ˜Š>ÜðCóð ð —‘˜yÑ)¨$Ñ/×3Ñ3¸Ð3Ó;×@Ñ@ÓBÐBr   c                 ó¾   — t        t        | j                  «      «      D ]<  }| j                  |   j                  «        | j                  |   j                  «        Œ> yr®  r°  r*   s     r   r²  zHybridCache.reset  r³  r   r   rF   )rG   rH   rI   rJ   rL   rw  r   rN   r   r
   r;   rO   r¦   r   r   ré  rì  rM   r   r	   r'   r.   r+   r²  rR   rS   s   @r   râ  râ  L  s/  ø„ ñ)ðf (,Ø15Ø"Ÿ]™]ØOSñ7;à ð7;ð ð7;ð   ‘}ð	7;ð
 �e—l‘l C¨Ð-Ñ.ð7;ð �{‰{ð7;ð # 4¨¨U°3¸¿¹ÀcÐ3IÑ-JÐ(JÑ#KÑLð7;ð 
õ7;òrò8ð 26ñ%
à—L‘Lð%
ð —l‘lð%
ð ð	%
ð
 ˜t C¨ H™~Ñ.ð%
ð 
ˆu�|‰|˜UŸ\™\Ð)Ñ	*ó%
ðN" X¨c¡]ó "ñ	C¨°©ó 	Cö0r   râ  c                   ó”  ‡ — e Zd ZdZdZddej                  dfdedede	e   de
ej                  edf   dej                  d	e	eee
eej                  ef   f      d
dfˆ fd„Zd„ Zd„ Zd„ Z	 ddej&                  dej&                  dede	eeef      d
eej&                  ej&                  f   f
d„Zd
e	e   fd„Zdde	e   fd„Zd„ Zˆ xZS )ÚHybridChunkedCachea;
  
    Hybrid Cache class to be used with `torch.compile` for Gemma2 models that alternate between a local sliding window attention
    and global attention in every other layer. Under the hood, Hybrid Cache leverages ["SlidingWindowCache"] for sliding window attention
    and ["StaticCache"] for global attention. For more information, see the documentation of each subcomponeent cache class.

    Parameters:
        config (`PretrainedConfig):
            The configuration file defining the shape-related attributes required to initialize the static cache.
        max_batch_size (`int`):
            The maximum batch size with which the model will be used. Note that a new instance must be instantiated if a
            smaller batch size is used.
        max_cache_len (`int`, *optional*):
            The maximum sequence length with which the model will be used.
        device (`torch.device` or `str`, *optional*):
            The device on which the cache should be initialized. If you're using more than 1 computation device, you
            should pass the `layer_device_map` argument instead.
        dtype (torch.dtype, *optional*, defaults to `torch.bfloat16`):
            The default `dtype` to use when initializing the layer.
        layer_device_map (`Optional[Dict[int, Union[str, torch.device, int]]]]`, *optional*):
            Mapping between the layers and its device. This is required when you are manually initializing the cache
            and the model is split between different gpus. You can know which layers mapped to which device by
            checking the associated device_map: `model.hf_device_map`.

    Example:

        ```python
        >>> from transformers import AutoTokenizer, AutoModelForCausalLM, HybridCache

        >>> model = AutoModelForCausalLM.from_pretrained("google/gemma-2-2b")
        >>> tokenizer = AutoTokenizer.from_pretrained("google/gemma-2-2b")

        >>> inputs = tokenizer(text="My name is Gemma", return_tensors="pt")

        >>> # Prepare a cache class and pass it to model's forward
        >>> # Leave empty space for 10 new tokens, which can be used when calling forward iteratively 10 times to generate
        >>> max_generated_length = inputs.input_ids.shape[1] + 10
        >>> past_key_values = HybridCache(config=model.config, max_batch_size=1, max_cache_len=max_generated_length, device=model.device, dtype=model.dtype)
        >>> outputs = model(**inputs, past_key_values=past_key_values, use_cache=True)
        >>> outputs.past_key_values # access cache filled with key/values from generation
        HybridCache()
        ```
    TNr`   r�  r«   r;   r¦   r�  r#   c                 ó’  •— t         ‰
| �  «        t        |d«      r|j                  €!t	        |j                  «       dd«      | _        n|j                  | _        || _        || _        t	        |d|j                  |j                  z  «      | _
        || _        t        |j                  «       d«      r|j                  | _        nDt	        |dd«      }t        |j                  «      D �cg c]  }t!        |dz   |z  «      ‘Œ c}| _        g | _        g | _        t        |j                  «      D �	cg c]  }	d	‘Œ c}	| _        y c c}w c c}	w )
Nr·  Úattention_chunk_sizei    r’  Úno_rope_layersrä  ri   r   r   )r   r   rD   r·  rõ   Úget_text_configr«   r�  r•  r–  r’  r—  rø  r‹  r7   r˜  rÉ  r9   r>   Úcumulative_length)r   r`   r�  r«   r;   r¦   r�  rå  rÜ   rÅ   r   s             €r   r   zHybridChunkedCache.__init__F  s  ø€ ô 	‰ÑÔÜ�vÐ/Ô0°F×4IÑ4IÐ4QÜ")¨&×*@Ñ*@Ó*BÐDZÐ\`Ó"aˆDÕà"(×"7Ñ"7ˆDÔØ*ˆÔØ,ˆÔÜ ¨
°F×4FÑ4FÈ&×JdÑJdÑ4dÓeˆŒØˆŒä�6×)Ñ)Ó+Ð-=Ô>Ø$×3Ñ3ˆD�Oä" 6Ð+CÀQÓGˆLÜEJÈ6×KcÑKcÓEdÖeÀœt Q¨¡U¨lÑ$:Õ;ÒeˆDŒOà-/ˆŒØ/1ˆÔÜ-2°6×3KÑ3KÓ-LÖ!M¨¢!Ò!MˆÕùò	 fùò "Ns   Ã(D?Ä-	Ec                 ó’  — t        | j                  «      |kD  ry |j                  d   }|j                  }| j                  || j
                  | j                  f}| j                  || j                  | j                  f}| j                  |   r|n|}t        j                  || j                  |¬«      }t        j                  || j                  |¬«      }	t        j                  j                  |«       t        j                  j                  |	«       | j                  j                  |«       | j                  j                  |	«       y )Nr   r6  )r8   r9   rÂ   r;   r�  r«   r’  r·  r‹  rL   r9  r—  r™  rš  r[   r>   )
r   r!   r   r“  r;   ræ  rç  r›  r�  rž  s
             r   Úinitialise_cache_layerz)HybridChunkedCache.initialise_cache_layerc  s  € Üˆt�~‰~Ó Ò*Øà(×.Ñ.¨qÑ1ÐØ×"Ñ"ˆØ"×1Ñ1Ð3FÈ×HZÑHZÐ\`×\iÑ\iÐjÐà×ÑØØ×ÑØ�M‰Mð	
Ðð .2¯_©_¸YÒ-GÑ)ÐM_ˆÜ#Ÿk™k¨+¸T¿[¹[ÐQWÔXÐÜ %§¡¨K¸t¿{¹{ÐSYÔ ZÐÜ�‰×)Ñ)Ð*=Ô>Ü�‰×)Ñ)Ð*?Ô@Ø�‰×ÑÐ1Ô2Ø×Ñ×ÑÐ 5Õ6r   c                 óP  — | j                   |   }| j                   |xx   |j                  d   z  cc<   ||k\  }	|	r½t        j                  |d d …d d …dd …d d …f   |fd¬«      }
t        j                  |d d …d d …dd …d d …f   |fd¬«      }|j                  d   dk(  �r*| j                  |   j                  |
«       | j                  |   j                  |«       | j                  |   | j                  |   fS |	sp||j                  d   z   |kD  r[|dk(  r|}
|}n¯t        j                  |d d …d d …d |…d d …f   |fd¬«      }
t        j                  |d d …d d …d |…d d …f   |fd¬«      }n^| j                  |   j                  d||«       | j                  |   j                  d||«       | j                  |   | j                  |   fS | j                  |   j                  |
d d …d d …| d …d d …f   «       | j                  |   j                  |d d …d d …| d …d d …f   «       |
|fS )Nr¿   r   rÀ   ri   r   )rú  rÂ   rL   rÄ   r9   r¡  r>   r¢  )r   r   r!   r   r    r£  r¤  r«   rú  Úis_fullÚfull_key_statesÚfull_value_statess               r   ré  z"HybridChunkedCache._sliding_updatez  s2  € Ø ×2Ñ2°9Ñ=Ðà×Ñ˜yÓ)¨Z×-=Ñ-=¸bÑ-AÑAÓ)Ø# }Ñ4ˆÙÜ#Ÿi™i¨ªq²!°Q±Rº¨{Ñ);¸ZÐ(HÈbÔQˆOÜ %§	¡	¨5²²A°q±rº1°Ñ+=¸|Ð*LÐRTÔ UÐð ×Ñ Ñ# qÓ(Ø—‘˜yÑ)×/Ñ/°Ô@Ø× Ñ  Ñ+×1Ñ1Ð2CÔDØ—~‘~ iÑ0°$×2BÑ2BÀ9Ñ2MÐMÐMÙÐ.°×1AÑ1AÀ!Ñ1DÑDÀ}ÒTà  AÒ%Ø",�Ø$0Ñ!ä"'§)¡)¨U²1²aÐ9KÐ:KÐ9KÊQÐ3NÑ-OÐQ[Ð,\ÐbdÔ"e�Ü$)§I¡I¨u²QºÐ;MÐ<MÐ;MÊqÐ5PÑ/QÐS_Ð.`ÐfhÔ$iÑ!à�N‰N˜9Ñ%×1Ñ1°!°^ÀZÔPØ×Ñ˜YÑ'×3Ñ3°A°~À|ÔTØ—>‘> )Ñ,¨d×.>Ñ.>¸yÑ.IÐIÐIà�‰�yÑ!×'Ñ'¨ºº1¸}¸n¹oÊqÐ8PÑ(QÔRØ×Ñ˜Ñ#×)Ñ)Ð*;ºAºqÀ=À.Á/ÒSTÐ<TÑ*UÔVð Ð 1Ð1Ð1r   c                 óv   — ||d d …d d …|f<   ||d d …d d …|f<   || j                   |<   || j                  |<   ||fS r   rò   rë  s           r   rì  z!HybridChunkedCache._static_updateœ  rí  r   r   r    r!   r"   c           
      ó~  — |€i }|j                  d«      }| j                  ||«       | j                  |   }| j                  |   }|j	                  |j
                  «      }|j	                  |j
                  «      }| j                  |   r| j                  }n| j                  } ||||||||j                  d   «      S )Nr   ri   )
r„  rü  r9   r>   r=   r¦   r‹  ré  rì  rÂ   )	r   r   r    r!   r"   r   r£  r¤  rï  s	            r   r'   zHybridChunkedCache.update¤  s¾   € ð ÐØˆLØ%×)Ñ)Ð*:Ó;ˆØ×#Ñ# I¨zÔ:à—‘˜yÑ)ˆØ× Ñ  Ñ+ˆØ—]‘] 5§;¡;Ó/ˆ
Ø#—‘ u§{¡{Ó3ˆà�?‰?˜9Ò%Ø×,Ñ,‰Ià×+Ñ+ˆIáØØØØØØØ�K‰K˜‰Nó
ð 	
r   c                 ó   — | j                   S r   r«  r-   s    r   r.   z&HybridChunkedCache.get_max_cache_shapeÄ  r¬  r   c                 ó´   — |dk7  rt        d«      ‚t        | j                  «      dk(  ry| j                  |   d   j                  d¬«      j	                  «       S rò  )r    r8   r9   r¨  r©  r*   s     r   r+   z!HybridChunkedCache.get_seq_lengthÇ  s_   € ð ˜Š>ÜðCóð ô ˆt�~‰~Ó !Ò#ØØ—‘˜yÑ)¨$Ñ/×3Ñ3¸Ð3Ó;×@Ñ@ÓBÐBr   c                 ó*  — t        t        | j                  «      «      D ]<  }| j                  |   j                  «        | j                  |   j                  «        Œ> t        t        | j
                  «      «      D �cg c]  }d‘Œ c}| _        yc c}w )r¯  r   N)r7   r8   r9   r±  r>   rú  )r   r!   rÅ   s      r   r²  zHybridChunkedCache.resetÔ  ss   € äœs 4§>¡>Ó2Ó3ò 	0ˆIà�N‰N˜9Ñ%×+Ñ+Ô-Ø×Ñ˜YÑ'×-Ñ-Õ/ð	0ô .3´3°t×7MÑ7MÓ3NÓ-OÖ!P¨¢!Ò!PˆÕùÒ!Ps   Á>	Br   rF   )rG   rH   rI   rJ   rK   rL   Úbfloat16r   rN   r   r
   r;   rO   r¦   r   r   rü  ré  rì  rM   r   r	   r'   r.   r+   r²  rR   rS   s   @r   rõ  rõ    sD  ø„ ñ)ðZ €Nð (,Ø15Ø"Ÿ^™^ØOSñNà ðNð ðNð   ‘}ð	Nð
 �e—l‘l C¨Ð-Ñ.ðNð �{‰{ðNð # 4¨¨U°3¸¿¹ÀcÐ3IÑ-JÐ(JÑ#KÑLðNð 
õNò:7ò. 2òDð 26ñ
à—L‘Lð
ð —l‘lð
ð ð	
ð
 ˜t C¨ H™~Ñ.ð
ð 
ˆu�|‰|˜UŸ\™\Ð)Ñ	*ó
ð@" X¨c¡]ó "ñC¨°©ó CöQr   rõ  c                   óø   — e Zd ZdZdZej                  dfdededej                  de
ej                  edf   fd„Zd	ed
ej                  dej                  dej                  fd„Zd	edej                  fd„Zd„ Zy)Ú
MambaCacheaœ  
    Cache for mamba model which does not have attention mechanism and key value states.

    Arguments:
        config (`PretrainedConfig):
            The configuration file defining the shape-related attributes required to initialize the static cache.
        max_batch_size (`int`):
            The maximum batch size with which the model will be used. Note that a new instance must be instantiated if a smaller batch size is used.
        dtype (`torch.dtype`, *optional*, defaults to `torch.float16`):
            The default `dtype` to use when initializing the layer.
        device (`torch.device` or `str`, *optional*):
            The device on which the cache should be initialized. Should be the same as the layer.

    Example:

        ```python
        >>> from transformers import AutoTokenizer, MambaForCausalLM, MambaCache

        >>> model = MambaForCausalLM.from_pretrained("state-spaces/mamba-130m-hf")
        >>> tokenizer = AutoTokenizer.from_pretrained("state-spaces/mamba-130m-hf")

        >>> inputs = tokenizer(text="My name is Mamba", return_tensors="pt")

        >>> # Prepare a cache class and pass it to model's forward
        >>> past_key_values = MambaCache(config=model.config, max_batch_size=1, device=model.device, dtype=model.dtype)
        >>> outputs = model(**inputs, past_key_values=past_key_values, use_cache=True)
        >>> outputs.past_key_values
        MambaCache()
        ```
    TNr`   r�  r¦   r;   c                 óø  — || _         || _        |j                  | _        |j                  | _        |j
                  | _        g | _        g | _        |�t        j                  |«      nd }t        |j                  «      D ]ú  }t        j                  | j                   | j                  | j                  || j                  ¬«      }t        j                  | j                   | j                  | j                  || j                  ¬«      }t        j                  j                  |«       t        j                  j                  |«       | j                  j!                  |«       | j                  j!                  |«       Œü y )N©r;   r¦   )r�  r—  Úintermediate_sizeÚ
state_sizeÚssm_state_sizeÚconv_kernelÚconv_kernel_sizeÚconv_statesÚ
ssm_statesrL   r;   r7   r˜  r9  r™  rš  r[   )r   r`   r�  r¦   r;   rÅ   Ú
conv_stateÚ	ssm_states           r   r   zMambaCache.__init__   s/  € ð -ˆÔØˆŒØ!'×!9Ñ!9ˆÔØ$×/Ñ/ˆÔØ &× 2Ñ 2ˆÔà/1ˆÔØ.0ˆŒØ)/Ð);”—‘˜fÔ%ÀˆÜ�v×/Ñ/Ó0ò 	.ˆAÜ',§{¡{Ø×#Ñ#Ø×&Ñ&Ø×%Ñ%ØØ—k‘kô(ˆJô ',§k¡kØ×#Ñ#Ø×&Ñ&Ø×#Ñ#ØØ—k‘kô'ˆIô �M‰M×-Ñ-¨jÔ9Ü�M‰M×-Ñ-¨iÔ8Ø×Ñ×#Ñ# JÔ/Ø�O‰O×"Ñ" 9Õ-ñ'	.r   r!   Únew_conv_stater   r#   c                 ó"  — | j                   |   j                  |j                  k7  r5| j                   |   j                  |j                  «      | j                   |<   | j                   |   }|j                  d| j                  dz
  «      }|j                  dd¬«      }|j                  |j                  |j                  ¬«      |d d …d d …|f<   | j                   |   j                  «        | j                   |xx   |z  cc<   | j                   |   S )Nr   r   rŸ   )ÚshiftsÚdimsr
  )r  r;   r=   r¾  r  Úrollr¦   r±  )r   r!   r  r   r  s        r   Úupdate_conv_statezMambaCache.update_conv_state%  sú   € ð
 ×Ñ˜IÑ&×-Ñ-°×1FÑ1FÒFØ*.×*:Ñ*:¸9Ñ*E×*HÑ*HÈ×I^ÑI^Ó*_ˆD×Ñ˜YÑ'à×%Ñ% iÑ0ˆ
Ø'×-Ñ-¨a°×1FÑ1FÈÑ1JÓKˆà—_‘_¨B°R�_Ó8ˆ
Ø+9×+<Ñ+<ÀJ×DUÑDUÐ]g×]mÑ]mÐ+<Ó+nˆ
’1’a˜Ð'Ñ(Ø×Ñ˜Ñ#×)Ñ)Ô+Ø×Ñ˜Ó# zÑ1Ó#Ø×Ñ 	Ñ*Ð*r   Únew_ssm_statec                 óŠ   — |j                  | j                  |   j                  «      | j                  |<   | j                  |   S r   )r=   r  r;   )r   r!   r  s      r   Úupdate_ssm_statezMambaCache.update_ssm_state6  s9   € Ø%2×%5Ñ%5°d·o±oÀiÑ6P×6WÑ6WÓ%Xˆ�‰˜	Ñ"Ø�‰˜yÑ)Ð)r   c                 ó¾   — t        t        | j                  «      «      D ]<  }| j                  |   j                  «        | j                  |   j                  «        Œ> y r   )r7   r8   r  r±  r  r*   s     r   r²  zMambaCache.reset:  sM   € Üœs 4×#3Ñ#3Ó4Ó5ò 	/ˆIà×Ñ˜YÑ'×-Ñ-Ô/Ø�O‰O˜IÑ&×,Ñ,Õ.ñ	/r   )rG   rH   rI   rJ   rK   rL   r¥   r   rN   r¦   r
   r;   rO   r   rM   rP   r  r  r²  rX   r   r   r  r  Ý  s©   „ ñð> €Nð #Ÿ]™]Ø15ñ#.à ð#.ð ð#.ð �{‰{ð	#.ð
 �e—l‘l C¨Ð-Ñ.ó#.ðJ+Øð+Ø.3¯l©lð+ØLQ×L\ÑL\ð+à	�‰ó+ð"*¨#ð *¸e¿l¹ló *ó/r   r  c                   óR  ‡ — e Zd ZdZdZd ej                  d«      dfdedede	e   de
eej                  f   d	e	ej                     d
e
eej                  f   de	eee
eej                  ef   f      ddfˆ fd„Z	 ddej                  dej                  dede	eeef      deej                  ej                  f   f
d„Zdde	e   defd„Zde	e   fd„Zdd„Zedefd„«       Zdeedf   dej                  deej                  ej                  f   fd„Zdeddfd„Zdeddfd„Zˆ xZS )ÚOffloadedStaticCachea3	  
    Static cache class to be used with `torch.compile(model)` that offloads to the CPU or
    another device.

    Args:
        config (`PretrainedConfig):
            The configuration file defining the shape-related attributes required to initialize
            the static cache.
        max_batch_size (`int`):
            The maximum batch size with which the model will be used.
        max_cache_len (`int`):
            The maximum sequence length with which the model will be used.
        device (`Union[str, torch.device]`):
            The device on which the cache should be initialized. If you're using more than 1 computation device, you
            should pass the `layer_device_map` argument instead.
        dtype (`torch.dtype`, *optional*):
            The default `dtype` to use when initializing the cache.
        offload_device (`Union[str, torch.device]`, *optional*, defaults to `cpu`):
            The device to offload to. Defaults to CPU.
        layer_device_map (`Dict[int, Union[str, torch.device, int]]`, *optional*):
            Mapping between the layers and its device. This is required when you are manually initializing the cache
            and the model is splitted between differents gpus. You can know which layers mapped to which device by
            checking the associated device_map: `model.hf_device_map`.

    Example:

        ```python
        >>> from transformers import AutoTokenizer, AutoModelForCausalLM, OffloadedStaticCache

        >>> model = AutoModelForCausalLM.from_pretrained("openai-community/gpt2")
        >>> tokenizer = AutoTokenizer.from_pretrained("openai-community/gpt2")

        >>> inputs = tokenizer(text="My name is GPT2", return_tensors="pt")

        >>> # Prepare a cache class and pass it to model's forward
        >>> # Leave empty space for 10 new tokens, which can be used when calling forward iteratively 10 times to generate
        >>> max_generated_length = inputs.input_ids.shape[1] + 10
        >>> past_key_values = OffloadedStaticCache(config=model.config, max_batch_size=1, max_cache_len=max_generated_length, device=model.device, dtype=model.dtype)
        >>> outputs = model(**inputs, past_key_values=past_key_values, use_cache=True)
        >>> past_kv_length = outputs.past_key_values # access cache filled with key/values from generation
        ```
    TNr�   r`   r�  r«   r;   r¦   Úoffload_devicer�  r#   c                 ó˜  •— t         t        | �  «        || _        |€|j                  n|| _        |€t        j                  |«      nt        j                  |d   «      | _        t        j                  |«      | _        |�|nt        j                  | _
        t        |d«      r|j                  n|j                  |j                  z  }t        |dd «      €|j                  n|j                   }	||	| j
                  |f}
g | _        g | _        t'        |j(                  «      D ]j  }|dk(  r| j                  n| j                  }| j+                  |
|«      \  }}| j"                  j-                  |«       | j$                  j-                  |«       Œl g | _        g | _        t'        d«      D ]W  }| j+                  |
| j                  «      \  }}| j.                  j-                  |«       | j0                  j-                  |«       ŒY d| _        | j                  j4                  dk(  r$t        j6                  j9                  «       | _        y d | _        y )Nr   r’  r“  ri   r  )r   r   r   r�  r”  r«   rL   r;   r   rw  r—  rD   r’  r•  r–  rõ   r“  r9   r>   r7   r˜  Ú_create_key_value_cache_tensorsr[   Ú_device_key_cacheÚ_device_value_cacherA   Útyper  r  Ú_prefetch_stream)r   r`   r�  r«   r;   r¦   r   r�  r’  r“  r›  rÜ   r9   r>   r   s                 €r   r   zOffloadedStaticCache.__init__o  s
  ø€ ô 	Œe�TÑ#Ô%Ø,ˆÔØ?LÐ?T˜V×;Ò;ÐZgˆÔØ.>Ð.F”e—l‘l 6Ô*ÌEÏLÉLÐYiÐjkÑYlÓLmˆŒÜ#Ÿl™l¨>Ó:ˆÔØ$Ð0‘e´e·m±mˆŒô '.¨f°jÔ&A�6—?’?Àv×GYÑGYÐ]c×]wÑ]wÑGwˆô �vÐ4°dÓ;ÐCð ×&Ò&à×+Ñ+ð 	ð &Ð':¸D×<NÑ<NÐPXÐYˆð .0ˆŒØ/1ˆÔä�v×/Ñ/Ó0ò 	1ˆAà$%¨¢F�T—[’[°×0CÑ0CˆFà%)×%IÑ%IÈ+ÐW]Ó%^Ñ"ˆI�{à�N‰N×!Ñ! )Ô,Ø×Ñ×#Ñ# KÕ0ð	1ð 68ˆÔØ79ˆÔ ä�q“ò 	9ˆAØ%)×%IÑ%IÈ+ÐW[×WbÑWbÓ%cÑ"ˆI�{à×"Ñ"×)Ñ)¨)Ô4Ø×$Ñ$×+Ñ+¨KÕ8ð		9ð ˆÔð 8<·{±{×7GÑ7GÈ6Ò7Q¤§
¡
× 1Ñ 1Ó 3ˆÕÐW[ˆÕr   r   r    r!   r"   c                 ó\  — |dk(  rA| xj                   |j                  d   z  c_         | j                  d   }| j                  d   }nr| j                  �Bt
        j                  j                  | j                  «      j                  | j                  «       | j                  |dz     }| j                  |dz     }| j                  |dz   «       |�|j                  d«      nd}|€™|j                  |«       |j                  |«       |dk(  rn| j                  |   j                  |j                  | j                   «      «       | j                  |   j                  |j                  | j                   «      «       ||fS 	 |j#                  d||«       |j#                  d||«       |dk7  r–|j                  | j                   «      }|j                  | j                   «      }|j                  | j                   «      }	 | j                  |   j#                  d||«       | j                  |   j#                  d||«       ||fS ||fS # t$        $ r ||dd…dd…|f<   ||dd…dd…|f<   Y ŒÂw xY w# t$        $ r8 || j                  |   dd…dd…|f<   || j                  |   dd…dd…|f<   Y ||fS w xY w)a0  
        Updates the cache with the new `key_states` and `value_states` for the layer `layer_idx`.
        It is VERY important to index using a tensor, otherwise you introduce a copy to the device.

        Parameters:
            key_states (`torch.Tensor`):
                The new key states to cache.
            value_states (`torch.Tensor`):
                The new value states to cache.
            layer_idx (`int`):
                The index of the layer to cache the states for.
            cache_kwargs (`Dict[str, Any]`, *optional*):
                Additional arguments for the cache subclass. The `OffloadedStaticCache` needs the
                `cache_position` input to know how where to write in the cache.

        Return:
            A tuple containing the updated key and value states.
        r   r¿   Nr   r   ri   )rA   rÂ   r9   r>   r&  rL   r  Údefault_streamr;   Úwait_streamr#  r$  Ú_prefetch_layerr„  r¡  r=   r   r¢  r&   )r   r   r    r!   r"   r£  r¤  r   s           r   r'   zOffloadedStaticCache.update©  s   € ð4 ˜Š>ð ×Ò ×!1Ñ!1°"Ñ!5Ñ5Õð —N‘N 1Ñ%ˆEØ×$Ñ$ QÑ'‰Eð ×$Ñ$Ð0Ü—
‘
×)Ñ)¨$¯+©+Ó6×BÑBÀ4×CXÑCXÔYà×*Ñ*¨9°q©=Ñ9ˆEØ×,Ñ,¨Y¸©]Ñ;ˆEà×Ñ˜Y¨™]Ô+à?KÐ?W˜×)Ñ)Ð*:Ô;Ð]aˆØÐ!Ø�K‰K˜
Ô#Ø�K‰K˜Ô%ð ˜AŠ~Ø—‘˜yÑ)×/Ñ/°
·±¸d×>QÑ>QÓ0RÔSØ× Ñ  Ñ+×1Ñ1°,·/±/À$×BUÑBUÓ2VÔWð: �eˆ|Ðð1;Ø×!Ñ! ! ^°ZÔ@Ø×!Ñ! ! ^°\ÔBð ˜AŠ~Ø!/×!2Ñ!2°4×3FÑ3FÓ!G�Ø'Ÿ]™]¨4×+>Ñ+>Ó?�
Ø+Ÿ™¨t×/BÑ/BÓC�ðUØ—N‘N 9Ñ-×9Ñ9¸!¸^ÈZÔXØ×$Ñ$ YÑ/×;Ñ;¸A¸~È|Ô\ð �eˆ|Ðˆu�eˆ|Ðøô+ 'ò ;ð /9�’aš˜NÐ*Ñ+Ø.:�’aš˜NÐ*Ó+ð	;ûô +ò Uð GQ�D—N‘N 9Ñ-ªa²°NÐ.BÑCØHT�D×$Ñ$ YÑ/²²1°nÐ0DÒEà�eˆ|ÐðUús%   Å>&I Ç;A I* É!I'É&I'É*;J+Ê*J+c                 ó   — | j                   S )r¦  ©rA   r*   s     r   r+   z#OffloadedStaticCache.get_seq_lengthü  ó   € ð × Ñ Ð r   c                 ó   — | j                   S )z9Returns the maximum sequence length of the cached states.r«  r-   s    r   r.   z(OffloadedStaticCache.get_max_cache_shape	  s   € ð ×!Ñ!Ð!r   c                 óÌ   — d| _         t        t        | j                  «      «      D ]<  }| j                  |   j	                  «        | j
                  |   j	                  «        Œ> y)z5Resets the cache values while preserving the objects.r   N)rA   r7   r8   r9   r±  r>   r*   s     r   r²  zOffloadedStaticCache.reset	  sW   € ð
 ˆÔô œs 4§>¡>Ó2Ó3ò 	0ˆIà�N‰N˜9Ñ%×+Ñ+Ô-Ø×Ñ˜YÑ'×-Ñ-Õ/ñ	0r   c                 ó   — | j                   S r   r,  r-   s    r   rE   z OffloadedStaticCache.seen_tokens	  r-  r   rÂ   .c                 óB  — |t        j                  d«      k(  }t        j                  || j                  ||¬«      }t        j                  || j                  ||¬«      }t         j                  j                  |«       t         j                  j                  |«       ||fS )a8  Creates K/V cache tensors on a device. Pins memory for CPU tensors. Marks them as static
        addresses for non-CPU tensors.

        Args:
            shape (`Tuple[int, ...]`): Shape.
            device (`torch.device`): Device.

        Returns:
            Key and value cache tensors as a tuple.
        r�   )r¦   r;   Ú
pin_memory)rL   r;   r9  r—  r™  rš  )r   rÂ   r;   Úis_cpu_devicer9   r>   s         r   r"  z4OffloadedStaticCache._create_key_value_cache_tensors	  sz   € ð ¤%§,¡,¨uÓ"5Ñ5ˆä—K‘K ¨T¯[©[ÀÐTaÔbˆ	Ü—k‘k %¨t¯{©{À6ÐVcÔdˆô 	�‰×)Ñ)¨)Ô4Ü�‰×)Ñ)¨+Ô6à˜+Ð%Ð%r   c                 ó  — |t        | j                  «      k\  ry| j                  �Dt        j                  j                  | j                  «      5  | j                  |«       ddd«       y| j                  |«       y# 1 sw Y   yxY w)zMPrefetch a layer to the device. Needs to be called in order of layer indices.N)r8   r9   r&  rL   r  r  Ú_prefetch_layer_in_contextr*   s     r   r*  z$OffloadedStaticCache._prefetch_layer4	  sv   € ð œ˜DŸN™NÓ+Ò+Øð × Ñ Ð,Ü—‘×"Ñ" 4×#8Ñ#8Ó9ñ ;Ø×/Ñ/°	Ô:÷;ð ;ð ×+Ñ+¨IÕ6÷;ð ;ús   ÁA<Á<Bc                 óÄ   — | j                   |dz     j                  | j                  |   d¬«       | j                  |dz     j                  | j                  |   d¬«       y)z6Performs the actual copy of the layer to device cache.r   Tr  N)r#  r¡  r9   r$  r>   r*   s     r   r5  z/OffloadedStaticCache._prefetch_layer_in_contextB	  s^   € ð 	×Ñ˜y¨1™}Ñ-×3Ñ3°D·N±NÀ9Ñ4MÐ\`Ð3ÔaØ× Ñ  ¨Q¡Ñ/×5Ñ5°d×6FÑ6FÀyÑ6QÐ`dÐ5Õer   r   rF   r,  )rG   rH   rI   rJ   rK   rL   r;   r   rN   r   r
   rO   r¦   r   r   rM   r   r	   r'   r+   r.   r²  rQ   rE   r"  r*  r5  rR   rS   s   @r   r  r  A  sÞ  ø„ ñ)ðV €Nð (,Ø3?°5·<±<ÀÓ3FØOSñ8\à ð8\ð ð8\ð   ‘}ð	8\ð
 �c˜5Ÿ<™<Ð'Ñ(ð8\ð ˜Ÿ™Ñ$ð8\ð ˜c 5§<¡<Ð/Ñ0ð8\ð # 4¨¨U°3¸¿¹ÀcÐ3IÑ-JÐ(JÑ#KÑLð8\ð 
õ8\ð~ 26ñQà—L‘LðQð —l‘lðQð ð	Qð
 ˜t C¨ H™~Ñ.ðQð 
ˆu�|‰|˜UŸ\™\Ð)Ñ	*óQñf!¨°©ð !¸có !ð" X¨c¡]ó "ó
0ð ð!˜Sò !ó ð!ð
&Ø˜3 ˜8‘_ð&Ø.3¯l©lð&à	ˆu�|‰|˜UŸ\™\Ð)Ñ	*ó&ð47¨ð 7°ó 7ðf°Cð f¸D÷ fr   r  )<rv   Úimportlib.metadatarJ  ro   r…   Údataclassesr   Útypingr   r   r   r   r   r	   r
   rL   Ú	packagingr   Útransformers.pytorch_utilsr   Úconfiguration_utilsr   rö   r   r   r   r   Úhqq.core.quantizer   r\  Ú
get_loggerrG   rB   r   rU   rˆ   r¨   r¯   rû   rþ   r÷   ÚContextr  r	  Úregister_pytree_noderH   ÚfxÚregister_pytree_flatten_specr  r.  rD  rZ  re  rŽ  rµ  rÄ  râ  rõ  r  r  rX   r   r   ú<module>rC     s"  ðÛ Û Û Û 	Ý !ß D× DÑ Dã Ý å Iå 1ß dÓ dñ ÔÝ;à	ˆ×	Ñ	˜HÓ	%€÷Kñ Kð\ ÷]ð ]ó ð]ð@ ô_˜;ó _ó ð_ðD ô$˜ó $ó ð$ôNHT�5ô HTðX9Øó9ð(C°Ló Cðà�[‰[× Ñ ×(Ñ(óò;ñ ˜UÔ#Ø	‡K�K×Ñ×,Ñ,ØØØ Ø ,× 7Ñ 7Ð8¸¸,×:OÑ:OÐ9PÐQØ=ð -ô ð 
‡H�H×Ñ×1Ñ1°,Ð@]Ô^ô@�\ô @ôFWY�\ô WYôt?$˜>ô ?$ôD;˜ô ;ô|zF�ô zFôz[0�%ô [0ô|M0˜ô M0ô`iA˜%ô iAôXG0�%ô G0ôTDQ˜ô DQ÷Na/ñ a/ôHEf˜;õ Efr   