Ë
    S^(hÓ ã                   óª  — d Z ddlZddlZddlmZmZmZ ddlZddl	Z	ddl
mc mZ ddl	mZ ddlmZ ddlmZ ddlmZ dd	lmZmZ dd
lmZ ddlmZmZmZmZ  ej>                  e «      Z!dd e"d«       fd„Z#d„ Z$d„ Z%d„ Z&d„ Z'd„ Z( G d„ dejR                  «      Z* G d„ dejR                  «      Z+ G d„ dejR                  «      Z, G d„ dejR                  «      Z- G d„ dejR                  «      Z. G d„ d ejR                  «      Z/ G d!„ d"ejR                  «      Z0 G d#„ d$ejR                  «      Z1 G d%„ d&ejR                  «      Z2d'Z3 ed(e3«       G d)„ d*e«      «       Z4 G d+„ d,ejR                  «      Z5 G d-„ d.e«      Z6 G d/„ d0ejR                  «      Z7 G d1„ d2ejR                  «      Z8 G d3„ d4ejR                  «      Z9 G d5„ d6ejR                  «      Z: G d7„ d8ejR                  «      Z; G d9„ d:ejR                  «      Z< G d;„ d<ejR                  «      Z= G d=„ d>ejR                  «      Z> G d?„ d@e«      Z? G dA„ dBe«      Z@dCZA edDe3«       G dE„ dFe@«      «       ZBy)GzPyTorch Jukebox model.é    N)ÚListÚOptionalÚTuple)Únn)Ú	LayerNormé   )ÚACT2FN)ÚPreTrainedModel)Úadd_start_docstringsÚlogging)Útqdmé   )ÚATTENTION_PATTERNSÚJukeboxConfigÚJukeboxPriorConfigÚJukeboxVQVAEConfigç        ÚInfc                 ó  — | j                  «       } t        || j                  d«      «      }|dkD  r*| t        j                  | |d¬«      d   ddd…f   k  }|| |<   |dkD  r«t        j
                  | dd¬«      \  }}t        j                  t        j                  |d¬«      d¬«      }||kD  }|ddd…f   j                  «       |dd	d…f<   d|d
<   t        j                  | t        j                  ¬«      j                  d||¬«      }|| |<   | S )aì  
    Filter a distribution of logits using top-k and/or nucleus (top-p) filtering

    Args:
        logits (`torch.Tensor`):
            logits distribution shape (vocabulary size)
        top_k (`int`, *optional*, defaults to 0):
            When `top_k >0` keep only top key tokens with highest probability (top-k filtering).
        top_p (`int`, *optional*, defaults to 0):
            When `top_p>0.0` keep the top tokens with cumulative probability >= `top_p` (nucleus filtering).
    éÿÿÿÿr   ©Údim.Nr   T)Ú
descendingr   r   ).r   ©Údtype)r   ÚindexÚsrc)ÚcloneÚminÚsizeÚtorchÚtopkÚsortÚcumsumÚFÚsoftmaxÚ
zeros_likeÚboolÚscatter_)	ÚlogitsÚtop_kÚtop_pÚfilter_valueÚindices_to_removeÚsorted_logitsÚsorted_indicesÚcumulative_probsÚsorted_indices_to_removes	            úu/var/www/skyplay_api_hub/venv/lib/python3.12/site-packages/transformers/models/deprecated/jukebox/modeling_jukebox.pyÚfilter_logitsr4   %   s  € ð �\‰\‹^€FÜ��v—{‘{ 2“Ó'€Eàˆq‚yà"¤U§Z¡Z°¸À2Ô%FÀqÑ%IÈ#ÈrÉsÈ(Ñ%SÑSÐØ$0ˆÐ Ñ!àˆs‚{Ü(-¯
©
°6ÀdÐPRÔ(SÑ%ˆ�~Ü Ÿ<™<¬¯	©	°-ÀRÔ(HÈbÔQÐð $4°eÑ#;Ð à,DÀSÈ#È2È#ÀXÑ,N×,TÑ,TÓ,VÐ   a¡b Ñ)Ø+,Ð  Ñ(ô "×,Ñ,¨V¼5¿:¹:ÔF×OÑOØ˜.Ð.Fð Pó 
Ðð %1ˆÐ Ñ!Ø€Mó    c           	      óf  — | d   } t        | «      |k  r‘t        j                  t        j                  |t        | «      z
  t        j                  ¬«      j                  | j                  «      | g«      }dg|t        | «      z
  z  t        t        dt        | «      «      «      z   }nzt        t        | «      ||dz  z   z  |z  «      }t        t        ||dz  «      t        | «      |dz  z
  «      }| ||dz  z
  ||dz  z    }t        t        ||dz  z
  ||dz  z   «      «      }|j                  d¬«      |fS )a  
    Extract only the relevant tokens based on the character position. A total of `max_n_lyric_tokens` tokens will be
    returned. If the provided token sequence is smaller, it will be padded, otherwise, only characters ranging from the
    midpoint - `max_n_lyric_tokens//2` to the midpoint + `max_n_lyric_tokens//2` will be returned. This *focuses* on
    the most relevant tokens (in time) for the sequence.

    Args:
        full_tokens (`List[int]`):
            List containing the token ids of the entire lyrics.
        total_length (`int`):
            Total expected length of the music (not all of it is generated, see duration), in samples.
        offset (`int`):
            Starting sample in the music. If the offset is greater than 0, the lyrics will be shifted take that into
            account
        duration (`int`):
            Expected duration of the generated music, in samples. The duration has to be smaller than the total length,
            which represent the overall length of the signal,
    r   r   r   ç       @é   r   )Úlenr!   ÚcatÚzerosÚlongÚtoÚdeviceÚlistÚrangeÚintr   ÚmaxÚ	unsqueeze)Úfull_tokensÚmax_n_lyric_tokensÚtotal_lengthÚoffsetÚdurationÚtokensÚindicesÚmidpoints           r3   Úget_relevant_lyric_tokensrL   K   s?  € ð& ˜a‘.€KÜ
ˆ;ÓÐ,Ò,Ü—‘Ü�[‰[Ð+¬c°+Ó.>Ñ>ÄeÇjÁjÔQ×TÑTÐU`×UgÑUgÓhÐjuÐvó
ˆð �$Ð,¬s°;Ó/?Ñ?Ñ@Ä4ÌÈaÔQTÐU`ÓQaÓHbÓCcÑc‰ä”s˜;Ó'¨6°H¸s±NÑ+BÑCÀlÑRÓSˆÜ”s˜8Ð%7¸1Ñ%<Ó=¼sÀ;Ó?OÐRdÐhiÑRiÑ?iÓjˆØ˜XÐ(:¸aÑ(?Ñ?À(ÐM_ÐcdÑMdÑBdÐeˆÜ”u˜XÐ(:¸aÑ(?Ñ?ÀÐL^ÐbcÑLcÑAcÓdÓeˆØ×Ñ ÐÓ" GÐ+Ð+r5   c                 óv   — g }t        d| |z
  |z   |«      D ]   }||z   | k\  r| |z
  }|j                  |«       Œ" |S ©Nr   )r@   Úappend)rF   Ún_ctxÚ
hop_lengthÚstartsÚstarts        r3   Ú
get_startsrT   m   sR   € Ø€FÜ�q˜,¨Ñ.°Ñ;¸ZÓHò ˆØ�5‰=˜LÒ(à  5Ñ(ˆEØ�‰�eÕð	ð
 €Mr5   c           	      ó>  — |j                   dz
  }|j                  }| |   }|j                  d   |j                  d   }}||k  r\||z
  }	t        j                  |t        j
                  |||z
  |j                  |j                  ¬«      gd¬«      }|j                  d   }nd}	t        |j                  | dz
     |j                  z  «      }
|j                  d   |j                  d   }}|h}i }i }t        t        |||
«      d¬«      D ]ü  }||z   }|j                  |||j                  dd¬«      \  }}t        j                   ||d¬«      }t        j                   ||d¬«      }g }t#        ||«      D ]?  \  }}|j%                  |d d …||…f   g ||¬	«      }|j'                  |d   d d …|f   «       ~ŒA t        j                  |d¬«      }~|j)                  d
t        j*                  ¬«      j-                  «       }~|||<   |||<   Œþ g }t/        |«      D ]†  }|ddd …f   }t1        j
                  |t3        |«      dz   f«      }t5        t        |||
«      «      D ]   }||z   }||   |   }||   |   } ||||…| f<   Œ" |d ||	z
  …d d…f   }|j'                  |«       Œˆ |S )Nr   r   ©r   r>   r   z#Computing lyric to music alignment )ÚdescT)Úget_indicesrG   )Úget_attn_weightsÚcpu©r>   r   é   r   )ÚlevelsrP   Úshaper!   r:   r;   r   r>   rA   Úhop_fractionÚprior_alignment_headÚprior_alignment_layerr   rT   Úget_metadataÚsample_lengthÚchunkÚzipÚforward_tokensrO   r=   ÚfloatÚnumpyr@   Únpr9   Úreversed)!Úmusic_tokensÚlabelsÚpriorÚconfigÚlevelrP   rI   Ú
batch_sizerF   Úpadding_lengthrQ   Úalignment_headÚalignment_layerÚattn_layersÚalignment_hopsÚindices_hopsrS   ÚendÚmetadataÚindices_hopÚ	tokens_bsÚmetadata_bsÚw_hopsÚtokens_iÚ
metadata_iÚw_hopÚweightsÚalignment_hopÚ
alignmentsÚitemrD   Ú	alignmentrJ   s!                                    r3   Úget_alignmentr…   w   sê  € Ø�L‰L˜1Ñ€EØ�K‰K€EØ˜%Ñ €FØ%Ÿ|™|¨A™°·±¸Q±�€JØ�eÒØ Ñ-ˆÜ—‘Ø”U—[‘[ ¨U°\Ñ-AÈÏÉÐ^d×^kÑ^kÔlÐmÐstô
ˆð —|‘| A‘‰àˆä�V×(Ñ(¨%¨°!©Ñ4°u·{±{ÑBÓC€JØ&,×&AÑ&AÀ!Ñ&DÀf×FbÑFbÐcdÑFe�O€NØ"Ð#€KØ€NØ€LÜ”j ¨u°jÓAÐHmÔnò .ˆØ�e‰mˆà %× 2Ñ 2°6¸5À&×BVÑBVÐdhÐqrÐ 2Ó sÑˆ�+Ü—K‘K ¨
¸Ô:ˆ	Ü—k‘k (¨J¸AÔ>ˆØˆÜ$'¨	°;Ó$?ò 	Ñ ˆH�jØ×(Ñ(¨²!°U¸3°Y°,Ñ)?ÀÀZÐbmÐ(ÓnˆEØ�M‰M˜% ™(¢1 nÐ#4Ñ5Ô6Ùð	ô —)‘)˜F¨Ô*ˆØØŸ
™
¨%´u·{±{˜
ÓC×IÑIÓKˆØð *ˆ�UÑØ -ˆ�uÒð'.ð. €JÜ�jÓ!ò 
%ˆà˜Q ¡˜U‘mˆÜ—H‘H˜l¬C°Ó,<¸qÑ,@ÐAÓBˆ	Üœj¨°u¸jÓIÓJò 	:ˆEØ˜%‘-ˆCØ*¨5Ñ1°$Ñ7ˆMØ" 5Ñ)¨$Ñ/ˆGØ,9ˆI�e˜C�i Ð(Ò)ð		:ð
 Ð= ¨~Ñ =Ð=¸sÀ¸sÐBÑCˆ	Ø×Ñ˜)Õ$ð
%ð Ðr5   c                 óœ  — t        j                  |dd«      j                  «       j                  «       }t	        t        |j                  d   «      «      D ]u  }|�Pt	        |«      |   j                  «       \  }}}| › d|› d|› d|› d|d d › d|› �}t        j                  |||   «       ŒUt        j                  | › d|› d|› �||   «       Œw y )Nr   r   r   z/lvl_ú-é   z-sample-)
r!   ÚclamprZ   rh   r?   r@   r^   Úvaluesri   Úsave)	ÚfnameÚlvlÚmetasÚaudÚiÚartistsÚgenresÚlyricsÚpaths	            r3   Úsave_temp_audior•   °   sÍ   € Ü
�+‰+�c˜2˜qÓ
!×
%Ñ
%Ó
'×
-Ñ
-Ó
/€CÜ”%˜Ÿ	™	 !™Ó%Ó&ò =ˆØÐÜ&*¨5£k°!¡n×&;Ñ&;Ó&=Ñ#ˆG�V˜VØ�W˜E #  a¨ y°°&°¸¸6À"À1¸:¸,ÀaÈÀsÐKˆDÜ�G‰G�D˜#˜a™&Õ!ä�G‰G�u�g˜U 3 % x°¨sÐ3°S¸±VÕ<ñ=r5   c                 óœ  — | �|dk(  ry |r||z
  nt        ||z
  d«      }| dk(  r(t        j                  |||¬«      j                  |«      } nî| dk(  r½t        j                  |||¬«      j                  «       } t        j                  |||¬«      j                  «       } | j	                  ||||z  «      d d …d d…| |z  d …f   } t        j
                  j                  j                  | dd¬«      j                  «       j	                  ||«      } n,| d	k(  r't        j                  |||¬«      j                  |«      } | j	                  dd||«      S )
Nr   r   Úautoregressive©r>   Úsummaryr   ©r   r   r   r   )ÚvalueÚprime)	rB   r!   ÚonesÚtrilÚviewr   Ú
functionalÚpadÚ
contiguous)	ÚmaskÚquery_lengthÚkey_value_lengthÚblocksÚspreadr>   ÚsampleÚsample_trG   s	            r3   Úget_maskrª   »   sD  € à€|�| qÒ(ØÙ(.ˆX˜Ò$´CÐ8HÈ<Ñ8WÐYZÓ4[€FØÐÒä�z‰z˜,Ð(8ÀÔH×MÑMÈfÓU‰Ø	�Ò	ä�z‰z˜,¨¸VÔD×IÑIÓKˆÜ�z‰z˜,¨¸VÔD×IÑIÓKˆØ�y‰y˜ v¨|¸vÑ/EÓFÂqÈ#È2È#ÐP`ÐO`ÐdjÑOjÑOlÐGlÑmˆä�H‰H×Ñ×#Ñ#ØØØð $ó ÷
 ‰Z‹\ß‰T�,Ð 0Ó1ñ 	ð 
�ŠÜ�z‰z˜,Ð(8ÀÔH×MÑMÈfÓUˆØ�9‰9�Q˜˜<Ð)9Ó:Ð:r5   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚJukeboxConv1Dc                 óü   •— t         ‰| �  «        || _        || _        t	        j
                  ||«      }t	        j                  |«      }t        j                  |«      | _	        t        j                  |«      | _
        y ©N)ÚsuperÚ__init__Úinput_widthÚoutput_widthr!   Úemptyr;   r   Ú	ParameterÚweightÚbias)Úselfr±   r²   rµ   r¶   Ú	__class__s        €r3   r°   zJukeboxConv1D.__init__×   s\   ø€ Ü‰ÑÔØ&ˆÔØ(ˆÔÜ—‘˜[¨,Ó7ˆÜ�{‰{˜<Ó(ˆÜ—l‘l 6Ó*ˆŒÜ—L‘L Ó&ˆ�	r5   c           	      ó8  — g |j                  «       d d ¢| j                  ‘­}t        j                  | j                  j                  |«      |j                  d|j                  d«      «      | j                  j                  |«      «      } |j                  |Ž }|S )Nr   )r    r²   r!   Úaddmmr¶   Útype_asrŸ   rµ   )r·   Úhidden_statesÚsize_outs      r3   ÚforwardzJukeboxConv1D.forwardà   sŽ   € ØB�]×'Ñ'Ó)¨#¨2Ð.ÐB°×0AÑ0AÑBˆÜŸ™Ø�I‰I×Ñ˜mÓ,Ø×Ñ˜r =×#5Ñ#5°bÓ#9Ó:Ø�K‰K×Ñ Ó.ó
ˆð
 +˜×*Ñ*¨HÐ5ˆØÐr5   ©Ú__name__Ú
__module__Ú__qualname__r°   r¾   Ú__classcell__©r¸   s   @r3   r¬   r¬   Ö   s   ø„ ô'ör5   r¬   c                   ó&   ‡ — e Zd Zdˆ fd„	Zd„ Zˆ xZS )ÚJukeboxResConv1DBlockc                 ó  •— t         ‰| �  «        |j                  |z  }|j                  |z  }|}|| _        t        j                  «       | _        t        j                  ||dd||«      | _	        t        j                  ||ddd«      | _
        y )Nr\   r   r   )r¯   r°   Úres_convolution_multiplierÚres_dilation_growth_rateÚ	res_scaler   ÚReLUÚ
activationÚConv1dÚconv1d_1Úconv1d_2)	r·   rn   Ú
conv_widthÚdepthrÊ   Ú
hidden_dimÚdilationÚpaddingr¸   s	           €r3   r°   zJukeboxResConv1DBlock.__init__ì   sy   ø€ Ü‰ÑÔØ×6Ñ6¸ÑCˆ
Ø×2Ñ2°EÑ9ˆØˆà"ˆŒÜŸ'™'›)ˆŒÜŸ	™	 *¨j¸!¸QÀÈÓRˆŒÜŸ	™	 *¨j¸!¸QÀÓBˆ�r5   c                 ó²   — |}| j                  |«      }| j                  |«      }| j                  |«      }| j                  |«      }|| j                  |z  z   S r®   )rÌ   rÎ   rÏ   rÊ   )r·   r¼   Ú	residualss      r3   r¾   zJukeboxResConv1DBlock.forward÷   sS   € Ø!ˆ	ØŸ™¨Ó6ˆØŸ™ mÓ4ˆØŸ™¨Ó6ˆØŸ™ mÓ4ˆØ˜4Ÿ>™>¨MÑ9Ñ9Ð9r5   )r   ç      ð?r¿   rÄ   s   @r3   rÆ   rÆ   ë   s   ø„ õ	Cö:r5   rÆ   c                   ó&   ‡ — e Zd Zdˆ fd„	Zd„ Zˆ xZS )ÚJukeboxResnet1Dc           	      óp  •— t         ‰	| �  «        |j                  | _        |j                  sdndt        j                  |«      z  }g }t        |«      D ]<  }| j                  €|n|| j                  z  }|j                  t        ||||«      «       Œ> |r|d d d…   }t        j                  |«      | _        y )Nr×   r   )r¯   r°   Úres_dilation_cycleÚdilation_cycleÚconv_res_scaleÚmathÚsqrtr@   rO   rÆ   r   Ú
ModuleListÚresnet_block)
r·   rn   rÐ   Ún_depthÚreverse_dilationrÊ   r¦   rÑ   Úblock_depthr¸   s
            €r3   r°   zJukeboxResnet1D.__init__  s¬   ø€ Ü‰ÑÔØ$×7Ñ7ˆÔØ%×4Ò4‘C¸#ÄÇ	Á	È'Ó@RÑ:Rˆ	àˆÜ˜7“^ò 	]ˆEØ#'×#6Ñ#6Ð#>™%ÀEÈD×L_ÑL_ÑD_ˆKØ�M‰MÔ/°¸
ÀKÐQZÓ[Õ\ð	]ñ Ø™D˜b˜D‘\ˆFÜŸM™M¨&Ó1ˆÕr5   c                 ó8   — | j                   D ]
  } ||«      }Œ |S r®   )rá   ©r·   r¼   Úblocks      r3   r¾   zJukeboxResnet1D.forward  s'   € Ø×&Ñ&ò 	1ˆEÙ! -Ó0‰Mð	1àÐr5   ©Fr¿   rÄ   s   @r3   rÙ   rÙ      s   ø„ õ2ör5   rÙ   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚJukeboxEncoderConvBlockc           
      ó~  •— t         ‰| �  «        g }|dz  }|dz  }	|dkD  r[t        |«      D ]M  }
|j                  t	        j
                  |
dk(  r|n|||||	«      «       |j                  t        |||«      «       ŒO t	        j
                  ||j                  ddd«      | _        t	        j                  |«      | _
        y )Nr8   r   r\   r   )r¯   r°   r@   rO   r   rÍ   rÙ   Ú	embed_dimÚproj_outrà   Údownsample_block)r·   rn   rì   rÒ   rÑ   Údown_tÚstride_tr¦   Úfilter_tÚpad_tr�   r¸   s              €r3   r°   z JukeboxEncoderConvBlock.__init__  s¯   ø€ Ü‰ÑÔØˆØ˜a‘<ˆØ˜A‘ˆØ�AŠ:Ü˜6“]ò J�Ø—‘œbŸi™i°Q¸!²V©	ÀÈZÐYaÐckÐmrÓsÔtØ—‘œo¨f°jÀ%ÓHÕIðJô Ÿ	™	 *¨f×.>Ñ.>ÀÀ1ÀaÓHˆŒÜ "§¡¨fÓ 5ˆÕr5   c                 óZ   — | j                   D ]
  } ||«      }Œ | j                  |«      }|S r®   )rî   rí   ræ   s      r3   r¾   zJukeboxEncoderConvBlock.forward"  s5   € Ø×*Ñ*ò 	1ˆEÙ! -Ó0‰Mð	1àŸ™ mÓ4ˆØÐr5   r¿   rÄ   s   @r3   rê   rê     s   ø„ ô
6ör5   rê   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚJukeboxEncoderc                 óR  •— t         ‰| �  «        || _        t        j                  «       | _        t        t        t        | j                  «      «      ||«      }|D ]J  \  }}	}
| j
                  j                  t        ||dk(  r|j                  n|j                  |||	|
«      «       ŒL y rN   )r¯   r°   r]   r   rà   Úlevel_blocksre   r?   r@   rO   rê   Úconv_input_shaperì   )r·   rn   ÚwidthrÑ   r]   Údowns_tÚ	strides_tÚiteratorr�   rï   rð   r¸   s              €r3   r°   zJukeboxEncoder.__init__*  s–   ø€ Ü‰ÑÔØˆŒÜŸM™M›OˆÔä”tœE $§+¡+Ó.Ó/°¸)ÓDˆØ#+ò 	ÑˆAˆv�xØ×Ñ×$Ñ$Ü'Ø°q¸A²v˜F×3Ò3À6×CSÑCSÐUZÐ\aÐciÐksóõñ	r5   c                 óŽ   — g }t        | j                  «      D ]*  }| j                  |   } ||«      }|j                  |«       Œ, |S r®   )r@   r]   r÷   rO   )r·   r¼   Úall_hidden_statesro   Úlevel_blocks        r3   r¾   zJukeboxEncoder.forward7  sR   € ØÐô ˜4Ÿ;™;Ó'ò 	4ˆEØ×+Ñ+¨EÑ2ˆKÙ'¨Ó6ˆMØ×$Ñ$ ]Õ3ð	4ð
 !Ð r5   r¿   rÄ   s   @r3   rõ   rõ   )  s   ø„ ôö	!r5   rõ   c                   ó&   ‡ — e Zd Zdˆ fd„	Zd„ Zˆ xZS )ÚJukeboxDecoderConvBockc           
      óŽ  •— || _         || _        t        ‰| �  «        g }|dkD  r‡|dz  }	|dz  }
t	        j
                  ||ddd«      | _        t        |«      D ]Q  }|j                  t        ||||«      «       |j                  t	        j                  |||dz
  k  r|n||	||
«      «       ŒS t	        j                  |«      | _        y )Nr   r8   r\   r   )rì   rÒ   r¯   r°   r   rÍ   Úproj_inr@   rO   rÙ   ÚConvTranspose1drà   Úupsample_block)r·   rn   rì   rÒ   rÑ   rï   rð   rã   r¦   rñ   rò   r�   r¸   s               €r3   r°   zJukeboxDecoderConvBock.__init__D  sÄ   ø€ Ø"ˆŒØ$ˆŒÜ‰ÑÔØˆØ�AŠ:Ø !‘|ˆHØ ‘MˆEÜŸ9™9 Y°
¸A¸qÀ!ÓDˆDŒLÜ˜6“]ò �Ø—‘œo¨f°jÀ%ÐIYÓZÔ[Ø—‘Ü×&Ñ&Ø"°!°f¸q±j².¡JÀiÐQYÐ[cÐejóõðô !Ÿm™m¨FÓ3ˆÕr5   c                 óZ   — | j                  |«      }| j                  D ]
  } ||«      }Œ |S r®   )r  r  ræ   s      r3   r¾   zJukeboxDecoderConvBock.forwardV  s5   € ØŸ™ ]Ó3ˆØ×(Ñ(ò 	1ˆEÙ! -Ó0‰Mð	1àÐr5   ©Tr¿   rÄ   s   @r3   r  r  C  s   ø„ õ4ö$r5   r  c                   ó&   ‡ — e Zd Zˆ fd„Zdd„Zˆ xZS )ÚJukeboxDecoderc                 ó�  •— t         ‰
| �  «        || _        t        j                  «       | _        t        t        t        | j                  «      «      ||«      D ]9  \  }}}	| j
                  j                  t        ||j                  ||||	«      «       Œ; t        j                  |j                  |j                  ddd«      | _        y )Nr\   r   )r¯   r°   r]   r   rà   r÷   re   r?   r@   rO   r  rì   rÍ   rø   Úout)r·   rn   rÒ   rÑ   r]   rú   rû   ro   rï   rð   r¸   s             €r3   r°   zJukeboxDecoder.__init__^  s¤   ø€ Ü‰ÑÔØˆŒÜŸM™M›OˆÔÜ'*¬4´°d·k±kÓ0BÓ+CÀWÈiÓ'Xò 	Ñ#ˆE�6˜8Ø×Ñ×$Ñ$Ü& v¨v×/?Ñ/?ÀÈUÐTZÐ\dÓeõð	ô
 —9‘9˜V×-Ñ-¨v×/FÑ/FÈÈ1ÈaÓPˆ�r5   c                 óÎ   — |d   }t        t        | j                  «      «      D ]-  }| j                  |   } ||«      }|dk7  sŒ |sŒ#|||dz
     z   }Œ/ | j	                  |«      }|S )Nr   r   r   )rj   r@   r]   r÷   r  )r·   r¼   Ú
all_levelsÚhidden_statero   rÿ   s         r3   r¾   zJukeboxDecoder.forwardi  sy   € Ø$ RÑ(ˆô œe D§K¡KÓ0Ó1ò 	GˆEØ×+Ñ+¨EÑ2ˆKÙ& |Ó4ˆLà˜‹zšjØ+¨m¸EÀA¹IÑ.FÑF‘ð	Gð —x‘x Ó-ˆØÐr5   r  r¿   rÄ   s   @r3   r	  r	  ]  s   ø„ ô	Q÷r5   r	  c                   ób   ‡ — e Zd Zdefˆ fd„Zd„ Zd„ Zd„ Zd„ Zd„ Z	d„ Z
d	„ Zd
„ Zd„ Zdd„Zˆ xZS )ÚJukeboxBottleneckBlockrn   c                 ó4  •— t         ‰| �  «        |j                  | _        |j                  | _        |j
                  | _        d| _        d| _        d | _	        d | _
        | j                  dt        j                  | j                  | j                  «      «       y )Nr×   FÚcodebook)r¯   r°   Únb_discrete_codesrì   Úcodebook_widthÚlmuÚmuÚ	thresholdÚinitÚcodebook_sumÚcodebook_elemÚregister_bufferr!   r;   )r·   rn   r¸   s     €r3   r°   zJukeboxBottleneckBlock.__init__y  sz   ø€ Ü‰ÑÔØ!'×!9Ñ!9ˆÔØ$×.Ñ.ˆÔØ—*‘*ˆŒØˆŒØˆŒ	Ø ˆÔØ!ˆÔØ×Ñ˜Z¬¯©°T×5KÑ5KÈT×M`ÑM`Ó)aÕbr5   c                 óö   — |j                   \  }}|| j                  k  rZ| j                  |z   dz
  |z  }dt        j                  |«      z  }|j	                  |d«      }|t        j                  |«      |z  z   }|S )Nr   ç{®Gáz„?)r^   r  ri   rß   Úrepeatr!   Ú
randn_like)r·   r¼   r   Úembed_widthÚ	n_repeatsÚstds         r3   Ú_tilezJukeboxBottleneckBlock._tile„  s   € Ø(×.Ñ.Ñˆˆ[Ø�×'Ñ'Ò'Ø×/Ñ/°#Ñ5¸Ñ9¸cÑAˆIØœŸ™ Ó-Ñ-ˆCØ)×0Ñ0°¸AÓ>ˆMØ)¬E×,<Ñ,<¸]Ó,KÈcÑ,QÑQˆMØÐr5   c                 ó(  — | j                   }d| _        | j                  |«      }|t        j                  |j
                  d   «         d | | _        | j                  | _        t        j                  || j                  j                  ¬«      | _
        y )NTr   r˜   )r  r  r#  r!   Úrandpermr^   r  r  r�   r>   r  )r·   r¼   r  Úcodess       r3   Úinit_codebookz$JukeboxBottleneckBlock.init_codebook�  ss   € Ø ×2Ñ2ÐØˆŒ	Ø—
‘
˜=Ó)ˆØœeŸn™n¨U¯[©[¸©^Ó<Ñ=Ð>PÐ?PÐQˆŒØ ŸM™MˆÔÜ"ŸZ™ZÐ(9À$Ç-Á-×BVÑBVÔWˆÕr5   c           	      ól  — | j                   | j                  | j                  }}}t        j                  «       5  t        j
                  ||j                  d   |j                  ¬«      }|j                  d|j                  d|j                  d   «      d«       t        j                  ||«      }|j                  d¬«      }| j                  |«      }	|	t        j                  |	j                  d   «         d | }
| j                  }|| j                  z  d|z
  |z  z   | _        || j                   z  d|z
  |z  z   | _        | j                   j                  |d«      | j"                  k\  j%                  «       }| j                  j                  ||«      | j                   j                  |d«      z  }||z  d|z
  |
z  z   | _        |t        j                  |«      z  }t        j                  |t        j&                  |dz   «      z  «       }|| j"                  k\  j                  «       }t        j                  |«      }t        j(                  j+                  | j                  |z
  «      t-        j.                  t-        j0                  |j                  «      «      z  }d d d «       dœS # 1 sw Y   ŒxY w)	Nr   r˜   r   r   r   r×   g:Œ0âŽyE>)ÚentropyÚ	used_currÚusageÚdk)r  r  r  r!   Úno_gradr;   r^   r>   r)   rŸ   ÚmatmulÚsumr#  r%  r  r  r  r  rg   ÚlogÚlinalgÚnormri   rß   Úprod)r·   r¼   Úlatent_statesr  r  r  Úlatent_states_onehotÚ_codebook_sumÚ_codebook_elemr&  Ú_random_codebookÚold_codebookr+  Ú	norm_codeÚ_codebook_probr)  r*  r,  s                     r3   Úupdate_codebookz&JukeboxBottleneckBlock.update_codebook•  sd  € Ø04·±¸×9LÑ9LÈd×NdÑNdÐ-ˆNˆÜ�]‰]‹_ñ 	hô $)§;¡;Ð/@À-×BUÑBUÐVWÑBXÐan×auÑauÔ#vÐ Ø ×)Ñ)¨!¨]×-?Ñ-?ÀÀ=×CVÑCVÐWXÑCYÓ-ZÐ\]Ô^ä!ŸL™LÐ)=¸}ÓMˆMØ1×5Ñ5¸"Ð5Ó=ˆNØ—J‘J˜}Ó-ˆEØ$¤U§^¡^°E·K±KÀ±NÓ%CÑDÐEWÐFWÐXÐð  Ÿ=™=ˆLØ " T×%6Ñ%6Ñ 6¸#À¹(ÀmÑ9SÑ SˆDÔØ!# d×&8Ñ&8Ñ!8¸CÀ"¹HÈÑ;VÑ!VˆDÔØ×'Ñ'×,Ñ,Ð->ÀÓBÀdÇnÁnÑT×[Ñ[Ó]ˆEà×)Ñ)×.Ñ.Ð/@À.ÓQÐTX×TfÑTf×TkÑTkØ! 1óUñ ˆIð " YÑ/°1°u±9Ð@PÑ2PÑPˆDŒMØ+¬e¯i©i¸Ó.GÑGˆNÜ—y‘y ´%·)±)¸NÈTÑ<QÓ2RÑ!RÓSÐSˆGØ'¨4¯>©>Ñ9×>Ñ>Ó@ˆIÜ—I‘I˜eÓ$ˆEÜ—‘×"Ñ" 4§=¡=°<Ñ#?Ó@Ä2Ç7Á7Ì2Ï7É7ÐS_×SeÑSeÓKfÓCgÑgˆB÷3	hð4 #°ÀUÐRTÑUÐU÷5	hð 	hús   ¹I"J*Ê*J3c                 óÐ  — |j                  ddd«      j                  «       }|j                  d|j                  d   «      }|j                  d   | j                  k(  rlt
        j                  j                  |t        j                  |«      z
  «      t        j                  t        j                  |j                  «      «      z  }||fS |j                  d   d| j                  z  k(  rü|dd | j                  …f   |d| j                  d …f   }}t
        j                  j                  |t        j                  |«      z
  «      t        j                  t        j                  |j                  «      «      z  t
        j                  j                  |t        j                  |«      z
  «      t        j                  t        j                  |j                  «      «      z  z   }||z   }|fS )Nr   r8   r   r   .)Úpermuter¢   rŸ   r^   r  r!   r1  r2  Úmeanri   rß   r3  )r·   r¼   ÚprenormÚx1Úx2s        r3   Ú
preprocessz!JukeboxBottleneckBlock.preprocess³  s‘  € Ø%×-Ñ-¨a°°AÓ6×AÑAÓCˆØ%×*Ñ*¨2¨}×/BÑ/BÀ2Ñ/FÓGˆà×Ñ˜rÑ" d×&9Ñ&9Ò9Ü—l‘l×'Ñ'¨¼¿
¹
À=Ó8QÑ(QÓRÔUW×U\ÑU\Ü—‘˜×+Ñ+Ó,óVñ ˆGð ˜gÐ%Ð%ð × Ñ  Ñ$¨¨D×,?Ñ,?Ñ(?Ò?Ø" 3Ð(=¨$×*=Ñ*=Ð(=Ð#=Ñ>ÀÈcÐSW×SfÑSfÑShÐNhÑ@i�ˆBÜ—|‘|×(Ñ(¨¬e¯j©j¸«nÑ)<Ó=ÄÇÁÌÏÉÐPR×PXÑPXÓHYÓ@ZÑZÜ—‘×!Ñ! "¤u§z¡z°"£~Ñ"5Ó6¼¿¹ÄÇÁÈÏÉÓARÓ9SÑSñˆGð
  ™GˆMà˜gÐ%Ð%r5   c                 óœ   — |\  }}|j                  ||d«      j                  ddd«      j                  «       }|j                  ||«      }||fS )Nr   r   r8   r   )rŸ   r>  r¢   )r·   r4  Údequantised_statesÚx_shaperp   Útimes         r3   Úpostprocessz"JukeboxBottleneckBlock.postprocessÆ  sZ   € Ø"Ñˆ
�DØ/×4Ñ4°ZÀÀrÓJ×RÑRÐSTÐVWÐYZÓ[×fÑfÓhÐØ%×*Ñ*¨:°tÓ<ˆØÐ0Ð0Ð0r5   c                 ó>  — | j                   j                  «       }t        j                  |dz  dd¬«      dt        j                  ||«      z  z
  t        j                  |dz  dd¬«      z   }t        j
                  |d¬«      \  }}t        j                  |«      }||fS )Nr8   r   T©r   Úkeepdimr   r   )r  Útr!   r/  r.  r   r?  )r·   r4  Úcodebook_weightsÚdistanceÚmin_distancerk   Úfits          r3   ÚquantisezJukeboxBottleneckBlock.quantiseÌ  s“   € àŸ=™=Ÿ?™?Ó,Ðä�I‰I�m QÑ&¨B¸Ô=Ø”%—,‘,˜}Ð.>Ó?Ñ?ñ@ä�i‰iÐ(¨!Ñ+°¸DÔAñBð 	ô
 &+§Y¡Y¨x¸RÔ%@Ñ"ˆ�lÜ�j‰j˜Ó&ˆØ˜SÐ Ð r5   c                 óF   — t        j                  || j                  «      }|S r®   )r%   Ú	embeddingr  )r·   rk   rE  s      r3   Ú
dequantisez!JukeboxBottleneckBlock.dequantiseØ  s   € ÜŸ[™[¨°t·}±}ÓEÐØ!Ð!r5   c                 óš   — |j                   \  }}}| j                  |«      \  }}| j                  |«      \  }}|j                  ||«      }|S r®   )r^   rC  rQ  rŸ   )r·   r4  ÚsamplesÚ_Úseq_lenrk   s         r3   ÚencodezJukeboxBottleneckBlock.encodeÜ  sW   € Ø+×1Ñ1Ñˆ��Gð  Ÿ?™?¨=Ó9Ñˆ�qð Ÿ-™-¨Ó6‰ˆ�að $×(Ñ(¨°'Ó:ˆØÐr5   c                 ó¾   — |j                   \  }}| j                  |«      }|j                  ||| j                  «      j	                  ddd«      j                  «       }|S ©Nr   r8   r   )r^   rT  rŸ   r  r>  r¢   )r·   rk   rV  rX  rE  s        r3   ÚdecodezJukeboxBottleneckBlock.decodeé  se   € Ø'×-Ñ-Ñˆ�ð "Ÿ_™_¨\Ó:Ðð ×#Ñ# G¨W°d×6IÑ6IÓJ×RÑRÐSTÐVWÐYZÓ[×fÑfÓhð 	ð "Ð!r5   c           	      ó&  — |j                   \  }}}| j                  |«      \  }}|r| j                  s| j                  |«       | j	                  |«      \  }}| j                  |«      }	|r| j                  ||«      }
ni }
t        j                  j                  |	j                  «       |z
  «      dz  t        j                  |j                   «      z  }||	|z
  j                  «       z   }	| j                  ||	||f«      \  }}	||	|t        d||dœ|
¤ŽfS )Nr8   )rP  Úpn© )r^   rC  r  r'  rQ  rT  r<  r!   r1  r2  Údetachri   r3  rH  Údict)r·   r¼   r<  rV  rW  rX  r@  rk   rP  rE  Úupdate_metricsÚcommit_losss               r3   r¾   zJukeboxBottleneckBlock.forwardõ  s*  € Ø+×1Ñ1Ñˆ��Gð "&§¡°Ó!?Ñˆ�wñ  4§9¢9Ø×Ñ˜}Ô-ð !ŸM™M¨-Ó8Ñˆ�cØ!Ÿ_™_¨\Ó:Ðñ Ø!×1Ñ1°-ÀÓN‰NàˆNô —l‘l×'Ñ'Ð(:×(AÑ(AÓ(CÀmÑ(SÓTÐXYÑYÔ\^×\cÑ\cØ×Ñó]
ñ 
ˆð
 +Ð.@À=Ñ.P×-XÑ-XÓ-ZÑZÐð ,0×+;Ñ+;¸LÐJ\Ð_fÐhoÐ^pÓ+qÑ(ˆÐ(ØÐ/°¼dÐ>iÀsÈwÑ>iÐZhÑ>iÐiÐir5   r  )rÀ   rÁ   rÂ   r   r°   r#  r'  r<  rC  rH  rQ  rT  rY  r\  r¾   rÃ   rÄ   s   @r3   r  r  x  sG   ø„ ð	cÐ1õ 	còòXòVò<&ò&1ò
!ò"òò
"÷jr5   r  c                   ó2   ‡ — e Zd Zˆ fd„Zd„ Zdd„Zd„ Zˆ xZS )ÚJukeboxBottleneckc                 óÞ   •— t         ‰| �  «        || _        t        j                  «       | _        t        | j                  «      D ]&  }| j
                  j                  t        |«      «       Œ( y r®   )	r¯   r°   r]   r   rà   r÷   r@   rO   r  )r·   rn   r]   ro   r¸   s       €r3   r°   zJukeboxBottleneck.__init__  sW   ø€ Ü‰ÑÔØˆŒÜŸM™M›OˆÔÜ˜4Ÿ;™;Ó'ò 	EˆEØ×Ñ×$Ñ$Ô%;¸FÓ%CÕDñ	Er5   c                 ó€   — t        | j                  |«      D ��cg c]  \  }}|j                  |«      ‘Œ }}}|S c c}}w r®   )re   r÷   rY  )r·   Ú	raw_audiorÿ   r¼   rk   s        r3   rY  zJukeboxBottleneck.encode  sG   € äRUÐVZ×VgÑVgÐirÓRs÷
Ù2N°;ÀˆK×Ñ˜}Õ-ð
ˆñ 
ð Ðùó
s   š:c                 ó¢   — |€| j                   }t        | j                  || |«      D ��cg c]  \  }}|j                  |«      ‘Œ }}}|S c c}}w r®   )r]   re   r÷   r\  )r·   rk   Ústart_levelÚ	end_levelrÿ   ÚzÚquantised_audios          r3   r\  zJukeboxBottleneck.decode$  s^   € ØÐØŸ™ˆIä:=¸d×>OÑ>OÐP[Ð\eÐ>fÐhtÓ:u÷
Ù&6 {°AˆK×Ñ˜qÕ!ð
ˆñ 
ð Ðùó
s   «Ac                 ó”  — g g g g f\  }}}}t        | j                  «      D ]   }| j                  | dz
     }||   } ||| j                  ¬«      \  }	}
}}|j	                  |	«       | j                  s|
j                  «       }
|j	                  |
«       |j	                  |«       | j                  sŒ�|j	                  |«       Œ¢ ||||fS )Nr   )r<  )r@   r]   r÷   ÚtrainingrO   r`  )r·   Úinput_audiork   Úquantised_statesÚcommit_lossesÚmetricsro   rÿ   r¼   Úsampled_tokensÚquantised_staterc  Úmetrics                r3   r¾   zJukeboxBottleneck.forward,  sÖ   € ØACÀRÈÈRÀÑ>ˆÐ&¨°wÜ˜4Ÿ;™;Ó'ò 	'ˆEØ×+Ñ+¨U¨F°Q©JÑ7ˆKØ'¨Ñ.ˆMÙCNØ¨t¯}©}ôDÑ@ˆN˜O¨[¸&ð ×Ñ Ô/Ø—=’=ð #2×"8Ñ"8Ó":�Ø×#Ñ# OÔ4Ø× Ñ  Ô-Ø�}‹}Ø—‘˜vÕ&ð	'ð Ð-¨}¸gÐEÐEr5   ©r   N)rÀ   rÁ   rÂ   r°   rY  r\  r¾   rÃ   rÄ   s   @r3   re  re    s   ø„ ôEòóöFr5   re  a?  

    This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
    library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
    etc.)

    This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
    Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
    and behavior.

    Parameters:
        config (`JukeboxConfig`): Model configuration class with all the parameters of the model.
            Initializing with a config file does not load the weights associated with the model, only the
            configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
zÞThe Hierarchical VQ-VAE model used in Jukebox. This model follows the Hierarchical VQVAE paper from [Will Williams, Sam
Ringer, Tom Ash, John Hughes, David MacLeod, Jamie Dougherty](https://arxiv.org/abs/2002.08111).

    c                   óÈ   ‡ — e Zd ZeZdZd„ Zdefˆ fd„Zdd„Zdde	j                  fd„Zdd„Zdd	„Zd
„ Zde	j                  dee	j                  e	j                  f   fd„Zˆ xZS )ÚJukeboxVQVAEÚvqvaec                 ó.  — t        |t        j                  «      r@|j                  j                  j                  dd| j                  j                  z  ¬«       �nt        |t        «      rz| j                  j                  r%|j                  j                  j                  «        nÁ|j                  j                  j                  dd| j                  j                  z  ¬«       n‚t        |t        «      rr| j                  j                  r\|j                  j                  j                  j                  «        |j                  j                  j                  j                  «        t        |t        j                  «      rI|j                  j                  j                  «        |j                  j                  j                  d«       t        |t        j                   «      r2|j                  �%|j                  j                  j                  «        y y y )Nr   ç{®Gáz”?©r?  r"  r×   )Ú
isinstancer   Ú	Embeddingrµ   ÚdataÚnormal_rn   Ú
init_scaler¬   Úzero_outÚzero_rÆ   rÏ   r¶   r   Úfill_ÚLinear©r·   Úmodules     r3   Ú_init_weightszJukeboxVQVAE._init_weights\  sV  € Ü�fœbŸl™lÔ+Ø�M‰M×Ñ×&Ñ&¨C°T¸D¿K¹K×<RÑ<RÑ5RÐ&ÖSÜ˜¤Ô.Ø�{‰{×#Ò#Ø—‘×"Ñ"×(Ñ(Õ*à—‘×"Ñ"×*Ñ*°¸ÀÇÁ×@VÑ@VÑ9VÐ*ÕWÜ˜Ô 5Ô6¸4¿;¹;×;OÒ;OØ�O‰O×"Ñ"×'Ñ'×-Ñ-Ô/Ø�O‰O× Ñ ×%Ñ%×+Ñ+Ô-Ü�fœbŸl™lÔ+Ø�K‰K×Ñ×"Ñ"Ô$Ø�M‰M×Ñ×$Ñ$ SÔ)Ü�fœbŸi™iÔ(¨V¯[©[Ð-DØ�K‰K×Ñ×"Ñ"Õ$ð .EÐ(r5   rn   c                 ór  •— t         ‰| �  |«       |j                  }|j                  }|j                  s~t        ||«      D ��cg c]
  \  }}||z  ‘Œ }}}t        j                  |«      }|j                  |j                  z  |z  |z  |_        |j                  j                  t        «      |_        |j                  | _        |j                  | _        |j                  | _        t        ||«      D ��cg c]
  \  }}||z  ‘Œ c}}| _        t        j                  | j                  «      | _        |j"                  x| _        }t%        |«      D �	cg c]+  }	t        | j                  | j                   |	 dz
     z  «      ‘Œ- c}	| _        |j(                  �|j(                  ndg|z  | _        t+        j,                  «       | _        t+        j,                  «       | _        t%        |«      D ]ª  }	|j2                  | j(                  |	   z  }
|j4                  | j(                  |	   z  }| j.                  j7                  t9        ||
||	dz   |d |	dz    |d |	dz    «      «       | j0                  j7                  t;        ||
||	dz   |d |	dz    |d |	dz    «      «       Œ¬ t=        ||«      | _        y c c}}w c c}}w c c}	w ©Nr   ) r¯   r°   Úres_downs_tÚres_strides_trc   re   ri   r3  Úsample_length_in_secondsÚsampling_rateÚastyperA   r  ÚcommitÚdownsamplesÚcumprodÚhop_lengthsr]   r@   Úmusic_tokens_shapesÚmultipliersr   rà   ÚencodersÚdecodersÚres_conv_widthÚres_conv_depthrO   rõ   r	  re  Ú
bottleneck)r·   rn   rú   rû   ÚstrideÚdownr’  Útop_raw_to_tokensr]   ro   rù   rÑ   r¸   s               €r3   r°   zJukeboxVQVAE.__init__m  s‡  ø€ Ü‰Ñ˜Ô Ø×$Ñ$ˆØ×(Ñ(ˆ	Ø×#Ò#Ü<?À	È7Ó<S×T©L¨F°D˜6 4›<ÐTˆKÑTÜ "§¡¨Ó 4Ðà×/Ñ/°&×2FÑ2FÑFÐJ[Ñ[Ø!ñ$"ˆFÔ ð $*×#7Ñ#7×#>Ñ#>¼sÓ#CˆFÔ à!'×!9Ñ!9ˆÔØ—m‘mˆŒØ#×1Ñ1ˆÔä=@ÀÈGÓ=T×U©\¨V°T˜F D›LÓUˆÔÜŸ:™: d×&6Ñ&6Ó7ˆÔØ%Ÿ}™}Ð,ˆŒ�fäSXÐY_ÓS`ö$
ØJOŒS�×#Ñ# t×'7Ñ'7¸¸À¹
Ñ'CÑCÕDò$
ˆÔ ð 28×1CÑ1CÐ1O˜6×-Ò-ÐVWÐUXÐ[aÑUaˆÔäŸ™›ˆŒÜŸ™›ˆŒÜ˜6“]ò 	ˆEØ×)Ñ)¨D×,<Ñ,<¸UÑ,CÑCˆEØ×)Ñ)¨D×,<Ñ,<¸UÑ,CÑCˆEØ�M‰M× Ñ Ü˜v u¨e°U¸Q±YÀÈÈ%ÐRSÉ)Ð@TÐV_Ð`kÐbgÐjkÑbkÐVlÓmôð �M‰M× Ñ Ü˜v u¨e°U¸Q±YÀÈÈ%ÐRSÉ)Ð@TÐV_Ð`kÐbgÐjkÑbkÐVlÓmõð	ô ,¨F°FÓ;ˆ�ùóA Uùó Vùò$
s   ÁJ(Ã5J.Å0J4c                 óÀ   — |€| j                   }| j                  j                  |||¬«      }| j                  |   |dd }} ||d¬«      }|j	                  ddd«      }|S )N©rj  rk  r   r   F©r  r8   )r]   r›  r\  r˜  r>  )r·   rk   rj  rk  r4  ÚdecoderÚdequantised_states          r3   Ú_decodezJukeboxVQVAE._decode”  sq   € àÐØŸ™ˆIØŸ™×.Ñ.¨|ÈÐ`iÐ.Ójˆà%)§]¡]°;Ñ%?ÀÈqÐQRÐASÐ"ˆÙ#Ð$5À%ÔHÐØ-×5Ñ5°a¸¸AÓ>ÐØ Ð r5   Úreturnc                 ó   — |D �cg c]  }t        j                  ||d¬«      ‘Œ }}g }t        |«      D ]9  }|D �	cg c]  }	|	|   ‘Œ	 }
}	| j                  |
||¬«      }|j	                  |«       Œ; t        j
                  |d¬«      S c c}w c c}	w )a½  
        Transforms the input `music_tokens` to their `raw_audio` representation.

        Args:
            music_tokens (`torch.LongTensor`):
                Tensor of music tokens which will be decoded to raw audio by using the codebook. Each music token
                should be an index to a corresponding `code` vector in the codebook.
            start_level (`int`, *optional*):
                Level at which the decoding process will start. Default to 0.
            end_level (`int`, *optional*):
                Level at which the decoding process will start. Default to None.
            bs_chunks (int, *optional*):
                Number of chunks to process at the same time.
        r   r   r   )r!   rd   r@   r¤  rO   r:   )r·   rk   rj  rk  Ú	bs_chunksÚtokenÚtoken_chunksrE  r�   ÚchunksÚmusic_tokens_ir£  s               r3   r\  zJukeboxVQVAE.decodeŸ  sœ   € ð KWÖWÀœŸ™ E¨9¸!Ö<ÐWˆÐWØÐÜ�yÓ!ò 	9ˆAØ6BÖC¨F˜f Q›iÐCˆNÐCØ $§¡¨^ÈÐ`i Ó jÐØ×%Ñ%Ð&7Õ8ð	9ô �y‰yÐ+°Ô3Ð3ùò Xùò Ds
   …BºBc                 ó.  — |€| j                   }|j                  ddd«      j                  «       }g }t        | j                   «      D ]-  }| j                  |   } ||«      }|j                  |d   «       Œ/ | j                  j                  |«      }	|	|| S )Nr   r8   r   r   )r]   r>  rg   r@   r—  rO   r›  rY  )
r·   rh  rj  rk  rp  r4  ro   ÚencoderÚlatent_staterk   s
             r3   Ú_encodezJukeboxVQVAE._encode¶  s™   € àÐØŸ™ˆIØ×'Ñ'¨¨1¨aÓ0×6Ñ6Ó8ˆØˆÜ˜4Ÿ;™;Ó'ò 	3ˆEØ—m‘m EÑ*ˆGÙ" ;Ó/ˆLØ× Ñ  ¨bÑ!1Õ2ð	3ð —‘×-Ñ-¨mÓ<ˆØ˜K¨	Ð2Ð2r5   c                 óð   — t        j                  ||d¬«      }g }|D ]'  }| j                  |||¬«      }|j                  |«       Œ) t	        |Ž D �	cg c]  }	t        j
                  |	d¬«      ‘Œ }
}	|
S c c}	w )að  
        Transforms the `input_audio` to a discrete representation made out of `music_tokens`.

        Args:
            input_audio (`torch.Tensor`):
                Raw audio which will be encoded to its discrete representation using the codebook. The closest `code`
                form the codebook will be computed for each sequence of samples.
            start_level (`int`, *optional*, defaults to 0):
                Level at which the encoding process will start. Default to 0.
            end_level (`int`, *optional*):
                Level at which the encoding process will start. Default to None.
            bs_chunks (int, *optional*, defaults to 1):
                Number of chunks of raw audio to process at the same time.
        r   r   r   )r!   rd   r¯  rO   re   r:   )r·   rp  rj  rk  r§  Úaudio_chunksÚmusic_tokens_listÚchunk_ir«  Úmusic_tokens_levelrk   s              r3   rY  zJukeboxVQVAE.encodeÃ  s„   € ô —{‘{ ;°	¸qÔAˆØÐØ#ò 	5ˆGØ!Ÿ\™\¨'¸{ÐV_˜\Ó`ˆNØ×$Ñ$ ^Õ4ð	5ô X[Ð\mÐWnÖoÐASœŸ	™	Ð"4¸!Ö<ÐoˆÐoØÐùò ps   ÁA3c           	      óª   — | j                   D �cg c])  }t        j                  d| j                  |g|¢­d¬«      ‘Œ+ }}| j	                  |«      S c c}w )Nr   rZ   )r    r>   )r•  r!   Úrandintr  r\  )r·   Ú	n_samplesÚmusic_tokens_shaperk   s       r3   r¨   zJukeboxVQVAE.sampleÚ  s\   € ð '+×&>Ñ&>ö
à"ô �M‰M˜!˜T×3Ñ3¸9Ð:ZÐGYÑ:ZÐchÖið
ˆð 
ð �{‰{˜<Ó(Ð(ùò	
s   �.Arh  c                 óô  — |j                  ddd«      j                  «       }g }t        | j                  «      D ]-  }| j                  |   } ||«      }|j                  |d   «       Œ/ | j                  |«      \  }}}	}g }
t        | j                  «      D ]C  }| j                  |   } ||||dz    d¬«      }|
j                  |j                  ddd«      «       ŒE t        |	«      }| j                  |z  }|
|fS )a"  
        Forward pass of the VQ-VAE, encodes the `raw_audio` to latent states, which are then decoded for each level.
        The commit loss, which ensure that the encoder's computed embeddings are close to the codebook vectors, is
        computed.

        Args:
            raw_audio (`torch.FloatTensor`):
                Audio input which will be encoded and decoded.

        Returns:
            `Tuple[torch.Tensor, torch.Tensor]`


        Example:
        ```python
        >>> from transformers import JukeboxVQVAE, set_seed
        >>> import torch

        >>> model = JukeboxVQVAE.from_pretrained("openai/jukebox-1b-lyrics").eval()
        >>> set_seed(0)
        >>> zs = [torch.randint(100, (4, 1))]
        >>> model.decode(zs).shape
        torch.Size([4, 8, 1])
        ```
        r   r8   r   r   Fr¡  )
r>  rg   r@   r]   r—  rO   r›  r˜  r/  r‘  )r·   rh  rp  r4  ro   r­  r®  rW  rk   rr  rE  r¢  r£  rc  Úlosss                  r3   r¾   zJukeboxVQVAE.forwardá  s  € ð8  ×'Ñ'¨¨1¨aÓ0×6Ñ6Ó8ˆØˆÜ˜4Ÿ;™;Ó'ò 	3ˆEØ—m‘m EÑ*ˆGÙ" ;Ó/ˆLØ× Ñ  ¨bÑ!1Õ2ð	3ð
 -1¯O©O¸MÓ,JÑ)ˆˆ<˜¨ØÐÜ˜4Ÿ;™;Ó'ò 	JˆEØ—m‘m EÑ*ˆGÙ '¨°U¸UÀQ¹YÐ(GÐTYÔ ZÐØ×%Ñ%Ð&7×&?Ñ&?ÀÀ1ÀaÓ&HÕIð	Jô
 ˜-Ó(ˆØ�{‰{˜[Ñ(ˆà! 4Ð'Ð'r5   rw  ©r   Nr   )rÀ   rÁ   rÂ   r   Úconfig_classÚbase_model_prefixr‰  r°   r¤  r!   ÚTensorr\  r¯  rY  r¨   ÚFloatTensorr   r¾   rÃ   rÄ   s   @r3   ry  ry  Q  su   ø„ ð &€LØÐò%ð"%<Ð1õ %<óN	!ñ4ÐRW×R^ÑR^ó 4ó.3óò.)ð-( ×!2Ñ!2ð -(°u¸U¿\¹\È5Ï<É<Ð=WÑ7X÷ -(r5   ry  c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )Ú
JukeboxMLPc                 ó&  •— t         ‰| �  «        |j                  }t        |j                  |z  «      }t        ||«      | _        t        ||«      | _        t        |j                     | _
        t        j                  |j                  «      | _        y r®   )r¯   r°   Úhidden_sizerA   Úmlp_multiplierr¬   Úc_fcÚc_projr	   Úact_fnÚactr   ÚDropoutÚresid_dropoutÚdropout)r·   rn   rì   rÒ   r¸   s       €r3   r°   zJukeboxMLP.__init__  sp   ø€ ä‰ÑÔØ×&Ñ&ˆ	Ü˜×.Ñ.°Ñ:Ó;ˆ
ä! )¨ZÓ8ˆŒ	Ü# J°	Ó:ˆŒÜ˜&Ÿ-™-Ñ(ˆŒÜ—z‘z &×"6Ñ"6Ó7ˆ�r5   c                 óŽ   — | j                  |«      }| j                  |«      }| j                  |«      }| j                  |«      }|S r®   )rÅ  rÈ  rÆ  rË  )r·   r¼   s     r3   r¾   zJukeboxMLP.forward  s@   € ØŸ	™	 -Ó0ˆØŸ™ Ó/ˆØŸ™ MÓ2ˆØŸ™ ]Ó3ˆØÐr5   r¿   rÄ   s   @r3   rÁ  rÁ    s   ø„ ô	8ör5   rÁ  c                   ó*   ‡ — e Zd Zdˆ fd„	Zˆ fd„Zˆ xZS )ÚJukeboxLayerNormc                 ó†   •— t         ‰| �  |||¬«       t        j                  |«      | _        d| j                  z  | _        y )N)ÚepsÚelementwise_affineiÿÿ  )r¯   r°   ri   r3  rù   Ú	max_numel)r·   Únormalized_shaperÐ  rÑ  r¸   s       €r3   r°   zJukeboxLayerNorm.__init__&  s:   ø€ Ü‰ÑÐ)¨sÐGYÐÔZÜ—W‘WÐ-Ó.ˆŒ
Ø §¡Ñ+ˆ�r5   c                 ó  •— |j                  «       | j                  kD  rPt        j                  || j                  | j
                  | j                  | j                  «      j                  |«      S t        ‰| �)  |«      j                  |«      S r®   )ÚnumelrÒ  r%   Ú
layer_normrÓ  rµ   r¶   rÐ  r»   r¯   r¾   )r·   Úinputr¸   s     €r3   r¾   zJukeboxLayerNorm.forward+  sg   ø€ Ø�;‰;‹=˜4Ÿ>™>Ò)Ü—<‘<  t×'<Ñ'<¸d¿k¹kÈ4Ï9É9ÐVZ×V^ÑV^Ó_×gÑgÐhmÓnÐnä‘7‘? 5Ó)×1Ñ1°%Ó8Ð8r5   )gñhãˆµøä>Tr¿   rÄ   s   @r3   rÎ  rÎ  %  s   ø„ õ,÷
9ð 9r5   rÎ  c                   ó¼   ‡ — e Zd Zdˆ fd„	Zd„ Zd„ Zdd„Zd„ Zd„ Zd„ Z	d„ Z
d	„ Zd
„ Zd„ Zdd„Zdd„Zdd„Zdd„Zed„ «       Zd„ Zdd„Zd„ Zd„ Zdd„Zd„ Zd„ Zˆ xZS )ÚJukeboxAttentionc           	      óD  •— t         ‰| �  «        |j                  | _        |j                  | _        |j
                  | _        t        |j                  | j                  z  «      }||j                  z  | _	        || _
        || _        | j                  dz  | _        |j                  | _        |dk(  r:t        | j                  |«      | _        t        | j                  |dz  «      | _        nt        | j                  |dz  «      | _        t        || j                  «      | _        t%        j&                  |j
                  «      | _        t%        j&                  |j(                  «      | _        || _        |dk(  r| j,                  | _        n(|dk(  r| j0                  | _        n| j2                  | _        | j4                  df| j6                  df| j8                  df| j:                  d f| j<                  df| j>                  df| j4                  d f| j@                  dfd	œ}||   \  | _!        | _"        |jF                  | _#        |jH                  | _$        | jF                  �| j                  | jF                  z  | _%        d
| _&        i | _'        |jP                  | _)        d| _*        y )Ng      Ð¿Úcross_attentionr8   r\   Ú
prime_attnr—   r™   rœ   )Ú
dense_attnÚ
block_attnÚtranspose_block_attnÚprev_block_attnÚsummary_attnÚsummary_spread_attnrÛ  rÜ  r   F)+r¯   r°   rÃ  rì   Ún_headsÚattn_dropoutrË  rA   Úattention_multiplierÚhead_dimrP   rÒ   Úscaler£   r¬   Úc_attnÚc_enc_kvrÆ  r   rÉ  rÊ  Ú	attn_funcÚ
decode_qkvÚqkvÚ	prime_qkvÚfactored_qkvrÝ  rÞ  rß  rà  rá  râ  rÜ  ÚattnÚ	attn_maskr¦   r§   Ú	block_ctxr©   ÚcacheÚnb_relevant_lyric_tokensÚencoder_lenÚrecord_attn)r·   rn   rP   rê  rÒ   ÚATTENTION_MAPr¸   s         €r3   r°   zJukeboxAttention.__init__3  s*  ø€ Ü‰ÑÔØ×+Ñ+ˆŒØ—~‘~ˆŒØ×*Ñ*ˆŒÜ˜×4Ñ4°t·~±~ÑEÓFˆ
à" f§n¡nÑ4ˆŒØˆŒ
Ø$ˆŒØ—]‘] EÑ)ˆŒ
Ø—K‘KˆŒ	àÐ)Ò)Ü'¨¯©¸
ÓCˆDŒKÜ)¨$¯.©.¸*Àq¹.ÓIˆD�Mä'¨¯©¸
ÀQ¹ÓGˆDŒKä# J°·±Ó?ˆŒÜŸJ™J v×':Ñ':Ó;ˆÔÜŸZ™Z¨×(<Ñ(<Ó=ˆÔð #ˆŒØÐ)Ò)Ø—‘ˆD�HØ˜,Ò&Ø—~‘~ˆD�Hà×(Ñ(ˆDŒHð  Ÿ?™?Ð,<Ð=ØŸ?™?Ð,<Ð=Ø%)×%>Ñ%>Ð@PÐ$QØ $× 4Ñ 4°dÐ;Ø!×.Ñ.°	Ð:Ø$(×$<Ñ$<¸iÐ#HØ $§¡°Ð6ØŸ?™?¨GÐ4ñ	
ˆð %2°)Ñ$<Ñ!ˆŒ	�4”>à—m‘mˆŒØ—m‘mˆŒØ�;‰;Ð"Ø!ŸZ™Z¨4¯;©;Ñ6ˆDŒNàˆŒØˆŒ
Ø!×:Ñ:ˆÔØ ˆÕr5   c           
      ó>  — | j                   }| j                  rt        j                  ||z  ||z  «      }n*t        j                  ||«      }|j	                  ||z  «       |j
                  }|j                  «       }| j                  rrt        | j                  |j                  d«      |j                  d«      | j                  | j                  |j                  || j                  «      }|�||z  dd|z
  z  z   }t        j                   |d¬«      j#                  |«      }	| j$                  rJ|	| _        | j(                  dk(  r4| j&                  d d …d d …| j*                  d …d | j*                  …f   | _        | j-                  |	«      }	t        j                  |	|«      }
|
S )Néþÿÿÿr   g    eÍÍÁr   r   rÜ  )rç  ro  r!   r.  Úmul_r   rg   r£   rª   rð  r    r¦   r§   r>   r©   r%   r&   Útyperõ  Úattention_probrê  rô  rä  )r·   Úquery_statesÚ
key_statesÚvalue_statesr¨   rç  Úattention_weightÚattn_weight_typer£   rû  Úcontext_statess              r3   Ú_attnzJukeboxAttention._attni  ss  € Ø—
‘
ˆØ�=Š=Ü$Ÿ|™|¨L¸5Ñ,@À*ÈuÑBTÓUÑä$Ÿ|™|¨L¸*ÓEÐØ×!Ñ! %¨%¡-Ô0Ø+×1Ñ1ÐØ+×1Ñ1Ó3ÐØ�9Š9ô Ø—‘Ø×!Ñ! "Ó%Ø—‘ Ó#Ø—‘Ø—‘Ø ×'Ñ'ØØ—‘ó	ˆDð ÐØ#3°dÑ#:¸TÀQÈÁXÑ=NÑ#NÐ ÜŸ™Ð#3¸Ô<×AÑAÐBRÓSˆØ×ÒØ"0ˆDÔØ�~‰~ Ò-à&*×&9Ñ&9º!ºQÀ×@PÑ@PÑ@RÐTfÐVZ×VfÑVfÐTfÐ:fÑ&g�Ô#Ø×*Ñ*¨>Ó:ˆÜŸ™ n°lÓCˆØÐr5   c                 óÖ   — |j                  dddd«      j                  «       }g |j                  «       d d ¢|j                  d«      |j                  d«      z  ‘­} |j                  |Ž S )Nr   r8   r   r\   rø  r   )r>  r¢   r    rŸ   )r·   r¼   Únew_hidden_states_shapes      r3   Úmerge_headszJukeboxAttention.merge_heads‹  ss   € Ø%×-Ñ-¨a°°A°qÓ9×DÑDÓFˆØ"o M×$6Ñ$6Ó$8¸¸"Ð$=Ð"o¸}×?QÑ?QÐRTÓ?UÐXe×XjÑXjÐkmÓXnÑ?nÑ"oÐØ!ˆ}×!Ñ!Ð#:Ð;Ð;r5   c                 óô   — g |j                  «       d d ¢| j                  ‘|j                  d«      | j                  z  ‘­} |j                  |Ž }|r|j                  dddd«      S |j                  dddd«      S )Nr   r   r8   r\   r   )r    rã  rŸ   r>  )r·   r¼   Úis_keyr  s       r3   Úsplit_headszJukeboxAttention.split_heads�  s’   € ð#
Ø×ÑÓ! # 2Ð&ð#
à�L‰Lð#
ð ×Ñ˜rÓ" d§l¡lÑ2ñ#
Ðð
 +˜×*Ñ*Ð,CÐDˆÙØ ×(Ñ(¨¨A¨q°!Ó4Ð4à ×(Ñ(¨¨A¨q°!Ó4Ð4r5   c                 óº   — | j                  |«      }| j                  |d¬«      }| j                  |«      }| j                  ||||«      }| j                  |«      }|S )NT)r  )r  r  r  )r·   ÚqueryÚkeyr›   r¨   r  s         r3   rÝ  zJukeboxAttention.dense_attnœ  sa   € Ø× Ñ  Ó'ˆØ×Ñ˜s¨4ÐÓ0ˆØ× Ñ  Ó'ˆØŸ™ E¨3°°vÓ>ˆØ×)Ñ)¨.Ó9ˆØÐr5   c                 óü  — | j                   }|j                  \  }}}|r%| j                  ||||«      j                  |d|«      S |j                  d   }	|j                  ||	z  |z  ||«      }|	|k  r6|	}|d d …| d …f   j	                  «       }|d d …| d …f   j	                  «       }|j                  ||z  |z  ||«      }|j                  ||z  |z  ||«      }| j                  ||||«      j                  |||«      S r‹  )rñ  r^   rÝ  rŸ   r¢   )
r·   r
  r  r›   r¨   rñ  rp   rX  rì   r¤   s
             r3   rÞ  zJukeboxAttention.block_attn¤  s  € Ø—N‘Nˆ	Ø).¯©Ñ&ˆ
�G˜YÙØ—?‘? 5¨#¨u°fÓ=×BÑBÀ:ÈqÐR[Ó\Ð\à Ÿ;™; q™>ˆLØ—J‘J˜z¨LÑ8¸IÑEÀyÐR[Ó\ˆEØ˜gÒ%Ø&�Øš!˜g˜X™Y˜,Ñ'×2Ñ2Ó4�Øša ' ¡˜lÑ+×6Ñ6Ó8�Ø—(‘(˜:¨Ñ/°9Ñ<¸iÈÓSˆCØ—J‘J˜z¨GÑ3°yÑ@À)ÈYÓWˆEØ—?‘? 5¨#¨u°fÓ=×BÑBÀ:ÈwÐXaÓbÐbr5   c                 ó’  — | j                   }|j                  \  }}}|rK|dz
  |z  }	|d d …|	d |…d d …f   }|d d …|	d |…d d …f   }| j                  ||||«      j                  |d|«      S |j                  d   }
|j                  ||
|z  ||«      }|j	                  dd«      j                  «       }|j                  ||z  |
|z  |«      }|j                  |||z  ||«      }|j	                  dd«      j                  «       }|j                  ||z  ||z  |«      }|j                  |||z  ||«      }|j	                  dd«      j                  «       }|j                  ||z  ||z  |«      }| j                  ||||«      }|j                  |||
|z  |«      }|j	                  dd«      j                  «       }|j                  ||
|«      }|S )Nr   r8   )rñ  r^   rÝ  rŸ   Ú	transposer¢   )r·   r
  r  r›   r¨   rñ  rp   rX  rì   Ú	block_lenr¤   rÞ  s               r3   rß  z%JukeboxAttention.transpose_block_attn´  sÞ  € Ø—N‘Nˆ	Ø).¯©Ñ&ˆ
�G˜YÙØ  1™¨	Ñ1ˆIØ’a˜Ð- IÐ-ªqÐ0Ñ1ˆCØš!˜YÐ1¨	Ð1²1Ð4Ñ5ˆEØ—?‘? 5¨#¨u°fÓ=×BÑBÀ:ÈqÐR[Ó\Ð\à Ÿ;™; q™>ˆLØ—J‘J˜z¨<¸9Ñ+DÀiÐQZÓ[ˆEØ—O‘O A qÓ)×4Ñ4Ó6ˆEØ—J‘J˜z¨IÑ5°|ÀyÑ7PÐR[Ó\ˆEà—(‘(˜: w°)Ñ';¸YÈ	ÓRˆCØ—-‘-  1Ó%×0Ñ0Ó2ˆCØ—(‘(˜:¨	Ñ1°7¸iÑ3GÈÓSˆCà—J‘J˜z¨7°iÑ+?ÀÈIÓVˆEØ—O‘O A qÓ)×4Ñ4Ó6ˆEØ—J‘J˜z¨IÑ5°wÀ)Ñ7KÈYÓWˆEàŸ™¨°°U¸FÓCˆJØ#Ÿ™¨°YÀÐPYÑ@YÐ[dÓeˆJØ#×-Ñ-¨a°Ó3×>Ñ>Ó@ˆJØ#Ÿ™¨°\À9ÓMˆJàÐr5   c                 ó  — | j                   }|j                  \  }}}|r¹|dz
  |z  }	|	dz
  |z  }
|	dkD  r#|d d …|
|
|z   …d d …f   }|d d …|
|
|z   …d d …f   }n\t        j                  ||||j                  |j
                  ¬«      }t        j                  ||||j                  |j
                  ¬«      }| j                  ||||«      j                  |d|«      S |j                  d   }|j                  ||z  |z  ||«      }|j                  |||z  ||«      d d …d d…d d …d d …f   }t        j                  j                  j                  |d«      }|j                  ||z  |z  ||«      }|j                  |||z  ||«      d d …d d…d d …d d …f   }t        j                  j                  j                  |d«      }|j                  ||z  |z  ||«      }||k  r�||z  }||z  }|}|j                  ||||«      d d …| d …f   }|j                  «       j                  ||z  ||«      }|j                  ||||«      d d …| d …f   }|j                  «       j                  ||z  ||«      }| j                  ||||«      j                  |||«      S )Nr   r   r[   r   ©r   r   r   r   r   r   )rñ  r^   r!   r;   r>   r   rÝ  rŸ   r   r    r¡   r¢   )r·   r
  r  r›   r¨   rñ  rp   rX  rì   rç   Úprev_lr¤   Únb_query_blocksÚnb_key_blockss                 r3   rà  z JukeboxAttention.prev_block_attnÑ  s²  € Ø—N‘Nˆ	Ø).¯©Ñ&ˆ
�G˜YÙØ˜q‘[ YÑ.ˆEØ˜a‘i 9Ñ,ˆFØ�qŠyØš!˜V f¨yÑ&8Ð8º!Ð;Ñ<�Øša ¨&°9Ñ*<Ð!<ºaÐ?Ñ@‘ä—k‘k *¨i¸È5Ï<É<Ð_d×_jÑ_jÔk�ÜŸ™ J°	¸9ÈUÏ\É\Ðaf×alÑalÔm�Ø—?‘? 5¨#¨u°fÓ=×BÑBÀ:ÈqÐR[Ó\Ð\à Ÿ;™; q™>ˆLØ—J‘J˜z¨LÑ8¸IÑEÀyÐR[Ó\ˆEà—(‘(˜: w°)Ñ';¸YÈ	ÓRÒSTÐVYÐWYÐVYÒ[\Ò^_ÐS_Ñ`ˆCÜ—(‘(×%Ñ%×)Ñ)¨#Ð/AÓBˆCØ—(‘(˜:¨Ñ/°9Ñ<¸iÈÓSˆCà—J‘J˜z¨7°iÑ+?ÀÈIÓVÒWXÐZ]Ð[]ÐZ]Ò_`ÒbcÐWcÑdˆEÜ—H‘H×'Ñ'×+Ñ+¨EÐ3EÓFˆEØ—J‘J˜z¨GÑ3°yÑ@À)ÈYÓWˆEà˜gÒ%Ø".°)Ñ";�Ø '¨9Ñ 4�Ø&�Ø—h‘h˜z¨=¸)ÀYÓOÒPQÐTcÐScÑSdÐPdÑe�Ø—n‘nÓ&×+Ñ+¨J¸Ñ,HÈ)ÐU^Ó_�àŸ
™
 :¨}¸iÈÓSÒTUÐXgÐWgÑWhÐThÑi�Ø×(Ñ(Ó*×/Ñ/°
¸_Ñ0LÈiÐYbÓc�à—?‘? 5¨#¨u°fÓ=×BÑBÀ:ÈwÐXaÓbÐbr5   c                 ó*  — | j                   }| j                  }|j                  \  }}}	|r©|d d …|dz
  ||z  dz
  |…d d …f   }t        j                  j
                  j                  |d«      }|d d …|dz
  ||z  dz
  |…d d …f   }t        j                  j
                  j                  |d«      }| j                  ||||«      j                  |d|	«      S |j                  ||||z  |	«      d d …d d…dd d …f   }t        j                  j
                  j                  |d«      }|j                  ||||z  |	«      d d …d d…dd d …f   }t        j                  j
                  j                  |d«      }| j                  ||||«      j                  |||	«      S )Nr   rš   r   )	r¦   rñ  r^   r!   r   r    r¡   rÝ  rŸ   )
r·   r
  r  r›   r¨   r¦   rñ  rp   rX  rì   s
             r3   rá  zJukeboxAttention.summary_attnö  sŽ  € Ø—‘ˆØ—N‘Nˆ	Ø).¯©Ñ&ˆ
�G˜YÙØ’a˜ Q™¨°)Ñ);¸aÑ)?À)ÐKÊQÐNÑOˆCÜ—(‘(×%Ñ%×)Ñ)¨#¨|Ó<ˆCàš!˜Y¨™]¨V°iÑ-?À!Ñ-CÀiÐOÒQRÐRÑSˆEÜ—H‘H×'Ñ'×+Ñ+¨E°<Ó@ˆEØ—?‘? 5¨#¨u°fÓ=×BÑBÀ:ÈqÐR[Ó\Ð\à—(‘(˜: v¨w¸&Ñ/@À)ÓLÊQÐPSÐQSÐPSÐUWÒYZÈ]Ñ[ˆCÜ—(‘(×%Ñ%×)Ñ)¨#¨|Ó<ˆCà—J‘J˜z¨6°7¸fÑ3DÀiÓPÒQRÐTWÐUWÐTWÐY[Ò]^ÐQ^Ñ_ˆEÜ—H‘H×'Ñ'×+Ñ+¨E°<Ó@ˆEØ—?‘? 5¨#¨u°fÓ=×BÑBÀ:ÈwÐXaÓbÐbr5   c                 ó€  — | j                   }| j                  }|j                  \  }}}	|rt        ‚|j	                  ||||z  |	«      d d …d d…| d …d d …f   }t
        j                  j                  j                  |d«      j                  «       }|j	                  |||z  |	«      }|j	                  ||||z  |	«      d d …d d…| d …d d …f   }t
        j                  j                  j                  |d«      j                  «       }|j	                  |||z  |	«      }| j                  ||||«      j	                  |||	«      S )Nr   r  )r¦   r§   r^   ÚNotImplementedErrorrŸ   r!   r   r    r¡   r¢   rÝ  )
r·   r
  r  r›   r¨   r¦   r§   rp   rX  rì   s
             r3   râ  z$JukeboxAttention.summary_spread_attn	  s7  € Ø—‘ˆØ—‘ˆà).¯©Ñ&ˆ
�G˜YÙÜ%Ð%à—(‘(˜: v¨w¸&Ñ/@À)ÓLÊQÐPSÐQSÐPSÐV\ÐU\ÑU]Ò_`ÐM`ÑaˆCÜ—(‘(×%Ñ%×)Ñ)¨#Ð/AÓB×MÑMÓOˆCØ—(‘(˜: v°¡¸	ÓBˆCà—J‘J˜z¨6°7¸fÑ3DÀiÓPÒQRÐTWÐUWÐTWÐZ`ÐY`ÑYaÒcdÐQdÑeˆEÜ—H‘H×'Ñ'×+Ñ+¨EÐ3EÓF×QÑQÓSˆEØ—J‘J˜z¨6°F©?¸IÓFˆEà—?‘? 5¨#¨u°fÓ=×BÑBÀ:ÈwÐXaÓbÐbr5   c                 ón   — | j                   }|d d …d |…f   }|d d …d |…f   }| j                  ||||«      S r®   )Ú_encoder_lenrÝ  )r·   r
  r  r›   r¨   rô  s         r3   rÜ  zJukeboxAttention.prime_attn  sF   € Ø×'Ñ'ˆØ’!�\�k�\�/Ñ"ˆØ’a˜˜+˜�oÑ&ˆØ�‰˜u c¨5°&Ó9Ð9r5   c                 ó
  — |j                   d   }|�t        d«      ‚|j                  dd¬«      \  }}}|rÉ| xj                  |z  c_        | j	                  ||«      \  }}| j                  «       }| j                  «       |kD  r| j                  | «       |dkD  rG| j                  dk7  r5| j                  |d¬«      }| j                  |«      }| j                  |«      }d	}n| j                  d
   }| j                  d   }||||fS )Nr   ú)last_encoder_hidden_states should be Noner\   r8   r   rÝ  T)r
  Fr  r›   )r^   Ú	TypeErrorrd   r©   Ú_append_cacheÚ_suff_cache_lenÚ
_cache_lenÚ_slice_cacherê  Ú_pad_to_block_ctxrò  )	r·   r¼   Úlast_encoder_hidden_statesr¨   Úcurr_ctxr
  r  r›   Úl_caches	            r3   rî  zJukeboxAttention.factored_qkv!  s  € Ø ×&Ñ& qÑ)ˆØ%Ð1ÜÐGÓHÐHà)×/Ñ/°°qÐ/Ó9Ñˆˆs�EÙØ�MŠM˜XÑ%�MØ×+Ñ+¨C°Ó7‰JˆC�Ø×*Ñ*Ó,ˆGØ�‰Ó  7Ò*Ø×!Ñ! 7 (Ô+Ø˜!Š|Ø—>‘> \Ò1Ø ×2Ñ2°5ÀÐ2ÓE�EØ×0Ñ0°Ó5�CØ ×2Ñ2°5Ó9�EØ‘à—j‘j Ñ'�ØŸ
™
 7Ñ+�Ø�c˜5 &Ð(Ð(r5   c                 ó®  — |j                   d   }|�t        d«      ‚|j                  dd¬«      \  }}}|r›| j                  «       | j                  k  r| j                  ||«       | j                  «       | j                  kD  r| j                  d| j                  «       | j                  d   | j                  d   }}| xj                  |z  c_        ||||fS )	Nr   r  r\   r8   r   r   r  r›   )	r^   r  rd   r  r  r  r   rò  r©   ©r·   r¼   r"  r¨   r#  r
  r  r›   s           r3   rí  zJukeboxAttention.prime_qkv8  sË   € Ø ×&Ñ& qÑ)ˆØ%Ð1ÜÐGÓHÐHØ)×/Ñ/°°qÐ/Ó9Ñˆˆs�EÙØ�‰Ó  4×#4Ñ#4Ò4Ø×"Ñ" 3¨Ô.Ø�‰Ó  4×#4Ñ#4Ò4Ø×!Ñ! ! T×%6Ñ%6Ô7ØŸ™ EÑ*¨D¯J©J°wÑ,?�ˆCØ�MŠM˜XÑ%�MØ�c˜5 &Ð(Ð(r5   c                 ó¾  — |j                   d   }|}|r‘| j                  dk(  rN| j                  |j                  |«      «      j	                  dd¬«      \  | j
                  d<   | j
                  d<   | j
                  d   | j
                  d   }}| xj                  |z  c_        n4| j                  |j                  |«      «      j	                  dd¬«      \  }}||||fS )Nr   r   r8   r   r  r›   )r^   r©   ré  r»   rd   rò  r&  s           r3   rë  zJukeboxAttention.decode_qkvF  sÒ   € Ø ×&Ñ& qÑ)ˆØˆÙØ�}‰} Ò!Ø9=¿¹Ø.×6Ñ6°}ÓEó:ç‘%˜˜q�%“/ñ 7�—
‘
˜5Ñ! 4§:¡:¨gÑ#6ð Ÿ™ EÑ*¨D¯J©J°wÑ,?�ˆCØ�MŠM˜XÑ%ŽMàŸ™Ð'A×'IÑ'IÈ-Ó'XÓY×_Ñ_Ð`aÐghÐ_Ói‰JˆC�Ø�c˜5 &Ð(Ð(r5   c                 ód  — |j                   d   }| j                  |«      }| j                  |||¬«      \  }}}}| j                  ||||«      }|j                   d   |k7  r0| j	                  |«      }	|d d …|	|	|z   …d d …f   j                  «       }| j                  |«      }| j                  |«      S )Nr   ©r"  r¨   )r^   rè  rì  rï  Ú_offsetr¢   rÆ  rÊ  )
r·   r¼   r"  r¨   r#  r
  r  r›   Úattention_scoresrG   s
             r3   r¾   zJukeboxAttention.forwardT  sÄ   € Ø ×&Ñ& qÑ)ˆØŸ™ MÓ2ˆØ$(§H¡HØÐ6PÐY_ð %-ó %
Ñ!ˆˆs�E˜6ð  Ÿ9™9 U¨C°¸Ó?ÐØ×!Ñ! !Ñ$¨Ò0Ø—\‘\ (Ó+ˆFØ/²°6¸FÀXÑ<MÐ3MÊqÐ0PÑQ×\Ñ\Ó^ÐØŸ;™;Ð'7Ó8ÐØ×!Ñ!Ð"2Ó3Ð3r5   c                 ó\   — | j                   }|| j                  z  dz   }|| j                  z  S r‹  )rô  r¦   )r·   rô  Úencoder_blockss      r3   r  zJukeboxAttention._encoder_lena  s/   € à×&Ñ&ˆØ%¨¯©Ñ4¸Ñ9ˆØ §¡Ñ+Ð+r5   c                 óZ   — | j                   dk(  ry| j                  |z
  | j                  z  S )NrÝ  r   )rê  r©   rñ  )r·   r#  s     r3   r*  zJukeboxAttention._offsetg  s)   € Ø�>‰>˜\Ò)ØØ—‘ Ñ(¨D¯N©NÑ:Ð:r5   c                 ó  — |j                   d   }|r| j                  |«      nd}||z   | j                  z   dz
  | j                  z  }|| j                  z  |z
  |z
  }|dk(  r|dk(  r|S t        j                  |dd||f«      S )Nr   r   )r^   r*  rñ  r%   r¡   )r·   r¼   r
  rX  rG   Ún_blocksr¡   s          r3   r!  z"JukeboxAttention._pad_to_block_ctxl  sŠ   € Ø×%Ñ% aÑ(ˆÙ*/�—‘˜gÔ&°QˆØ˜fÑ$ t§~¡~Ñ5¸Ñ9¸d¿n¹nÑLˆØ˜Ÿ™Ñ'¨'Ñ1°FÑ:ˆØ�!Š8˜ !šØ Ð ä—5‘5˜¨¨A¨v°sÐ(;Ó<Ð<r5   c                 óZ   — d| j                   vrdS | j                   d   j                  d   S )Nr  r   r   )rò  r^   ©r·   s    r3   r  zJukeboxAttention._cache_lenv  s,   € Ø §¡Ñ+ˆqÐK°·±¸EÑ1B×1HÑ1HÈÑ1KÐKr5   c           	      ó†  — | j                   dz
  | j                  z  dz   | j                  z   }| j                   | j                   dz
  | j                  z  dz   | j                   | j                   | j                  k  r| j                   n|| j                  t        | j                   | j                  «      dœ}|| j
                     S )z´
        Precondition:
            key and value are appended with the current context and self.sample_t reflects the 1-indexed sample
            location in the context.
        r   )rÝ  rÞ  rß  rà  Ú
cross_attnrÜ  )r©   rñ  rô  r   r  rê  )r·   Úprevious_block_lengthÚREQUIRED_CACHE_LENs      r3   r  z JukeboxAttention._suff_cache_leny  s¡   € ð "&§¡°Ñ!2°d·n±nÑ DÀqÑ HÈ4Ï>É>Ñ YÐàŸ-™-ØŸ=™=¨1Ñ,°·±Ñ>ÀÑBØ$(§M¡MØ04·±ÀÇÁÒ0O˜tŸ}š}ÐUjØ×*Ñ*Ü˜dŸm™m¨T×->Ñ->Ó?ñ
Ðð " $§.¡.Ñ1Ð1r5   c                 ó˜   — | j                   d   d d …||…f   | j                   d<   | j                   d   d d …||…f   | j                   d<   y )Nr  r›   )rò  )r·   rS   rw   s      r3   r   zJukeboxAttention._slice_cache‹  sJ   € Ø ŸJ™J uÑ-ªa°°s°¨lÑ;ˆ�
‰
�5ÑØ"Ÿj™j¨Ñ1²!°U¸3°Y°,Ñ?ˆ�
‰
�7Òr5   c                 ó¬  — d| j                   vr|| j                   d<   || j                   d<   nŠ||}}t        j                  | j                   d   |gd¬«      }t        j                  | j                   d   |gd¬«      }| j                   d= | j                   d= ~~|| j                   d<   || j                   d<   | j                   d   | j                   d   fS )Nr  r›   r   r   )rò  r!   r:   )r·   r  r›   Úold_keyÚ	old_values        r3   r  zJukeboxAttention._append_cache�  sÆ   € Ø˜Ÿ
™
Ñ"Ø #ˆD�J‰J�uÑØ"'ˆD�J‰J�wÒà!$ e�YˆGÜ—)‘)˜TŸZ™Z¨Ñ.°Ð8¸aÔ@ˆCÜ—I‘I˜tŸz™z¨'Ñ2°IÐ>ÀAÔFˆEØ—
‘
˜5Ð!Ø—
‘
˜7Ð#ØØØ #ˆD�J‰J�uÑØ"'ˆD�J‰J�wÑØ�z‰z˜%Ñ  $§*¡*¨WÑ"5Ð5Ð5r5   c                 óŒ   — d| _         d| j                  v r| j                  d= d| j                  v r| j                  d= i | _        y )Nr   r  r›   )r©   rò  r2  s    r3   Ú	del_cachezJukeboxAttention.del_cacheŸ  s@   € ØˆŒØ�D—J‘JÑØ—
‘
˜5Ð!Ø�d—j‘jÑ Ø—
‘
˜7Ð#Øˆ�
r5   ©rÝ  rè   ©NFr®   )rÀ   rÁ   rÂ   r°   r  r  r  rÝ  rÞ  rß  rà  rá  râ  rÜ  rî  rí  rë  r¾   Úpropertyr  r*  r!  r  r  r   r  r<  rÃ   rÄ   s   @r3   rÙ  rÙ  2  sŽ   ø„ õ4!òl òD<ó

5òòcò ò:#còJcò&cò$:ó)ó.)ó)ó4ð ñ,ó ð,ò
;ó
=òLò2ó$@ò6ö r5   rÙ  c                   ó(   ‡ — e Zd Zdˆ fd„	Zdd„Zˆ xZS )ÚJukeboxBlockc                 óD  •— t         ‰| �  «        |j                  | _        t	        |||¬«      | _        t        |j                  «      | _        t        |«      | _	        t        |j                  «      | _
        |j                  rd|j                  z  nd| _        || _        y )N©rê  r×   )r¯   r°   rÃ  rù   rÙ  rï  rÎ  Úlayer_norm_0rÁ  ÚmlpÚlayer_norm_1Úattn_res_scaleÚ
num_layersrÊ   rê  )r·   rn   rP   rê  r¸   s       €r3   r°   zJukeboxBlock.__init__©  s€   ø€ Ü‰ÑÔØ×'Ñ'ˆŒ
Ü$ V¨U¸iÔHˆŒ	ä,¨V×-?Ñ-?Ó@ˆÔÜ˜fÓ%ˆŒÜ,¨V×-?Ñ-?Ó@ˆÔØ4:×4IÒ4I˜˜v×0Ñ0Ò0ÈsˆŒØ"ˆ�r5   c                 óø   — |}| j                  |«      }| j                  |||«      }| j                  ||z   «      }| j                  |«      }| j                  dk(  r
||z   |z   }|S || j                  ||z   z  z   }|S )Nr×   )rD  rï  rF  rE  rÊ   )r·   r¼   r"  r¨   rÖ   Úoutput_statesÚoutputs          r3   r¾   zJukeboxBlock.forward´  sŽ   € Ø!ˆ	Ø×)Ñ)¨-Ó8ˆØŸ	™	 -Ð1KÈVÓTˆà×)Ñ)¨)°mÑ*CÓDˆØŸ™ Ó/ˆØ�>‰>˜SÒ Ø Ñ.°Ñ>ˆFð ˆð  §¡°=À=Ñ3PÑ!QÑQˆFØˆr5   r=  rè   r¿   rÄ   s   @r3   rA  rA  ¨  s   ø„ õ	#÷r5   rA  c                   ó2   ‡ — e Zd Zˆ fd„Zd„ Zdd„Zd„ Zˆ xZS )ÚJukeboxLayerStackc           
      ó0  •— t         ‰| �  «        || _        |j                  | _        |j
                  | _        |j                  | _        |j                  | _        | j                  �|| j                  z  | _        |j                  | _
        |j                  | _        t        | j                     }t        j                  «       | _        t!        | j
                  «      D ]/  }| j                  j#                  t%        || ||«      ¬«      «       Œ1 g | _        y )NrC  )r¯   r°   rP   rÃ  rù   rH  r¦   Úattention_patternrñ  ró  rô  rã  r   r   rà   Ú
_attn_modsr@   rO   rA  Úsaved_attn_weights)r·   rn   rP   rO  rÑ   r¸   s        €r3   r°   zJukeboxLayerStack.__init__Ã  sá   ø€ Ü‰ÑÔØˆŒ
Ø×'Ñ'ˆŒ
Ø ×+Ñ+ˆŒØ—m‘mˆŒØ!'×!9Ñ!9ˆÔØ�;‰;Ð"Ø" d§k¡kÑ1ˆDŒNØ!×:Ñ:ˆÔØ—~‘~ˆŒô /¨t×/EÑ/EÑFÐÜŸ-™-›/ˆŒÜ˜4Ÿ?™?Ó+ò 	dˆEØ�O‰O×"Ñ"¤<°¸ÑIZÐ[`ÓIaÔ#bÕcð	dð #%ˆÕr5   c                 óŒ   ‡— ˆfd„}t        | j                  «      D ]  \  }} ||«      |j                  _        Œ ‰sg | _        yy)a-  
        Makes forward prop dump self-attention softmaxes to self.saved_attn_weights.

        Args:
            record_attn (`Union[bool,set]`):
                Either a set of layer indices indicating which layers to store, or a boolean value indicating Whether
                to dump all.
        c                 ó0   •— t        ‰t        «      r‰S | ‰v S r®   )r~  r(   )Ú	layer_idxrõ  s    €r3   Ú_should_record_attnz>JukeboxLayerStack.set_record_attn.<locals>._should_record_attná  s   ø€ Ü˜+¤tÔ,Ø"Ð"Ø Ð+Ð+r5   N)Ú	enumeraterP  rï  rõ  rQ  )r·   rõ  rU  r�   Úlayers    `   r3   Úset_record_attnz!JukeboxLayerStack.set_record_attn×  sI   ø€ ô	,ô
 " $§/¡/Ó2ò 	<‰HˆAˆuÙ%8¸Ó%;ˆE�J‰JÕ"ð	<ñ Ø&(ˆDÕ#ð r5   c                 ó,  — t        | j                  «      D ]{  \  }}|j                  dk(  r ||||¬«      }n ||d |¬«      }|j                  j                  sŒC| j
                  j                  |j                  j                  j                  «       Œ} |S )NrÛ  r)  )	rV  rP  rê  rï  rõ  rQ  rO   rè  rµ   )r·   r¼   r"  r¨   r�   Ú
attn_layers         r3   r¾   zJukeboxLayerStack.forwardì  s�   € ä& t§¡Ó7ò 	N‰MˆAˆzØ×#Ñ#Ð'8Ò8Ù *Ø!Ð>XÐagô!‘ñ !+¨=ÐUYÐbhÔ i�Ø�‰×*Ó*Ø×'Ñ'×.Ñ.¨z¯©×/EÑ/E×/LÑ/LÕMð	Nð Ðr5   c                 óZ   — | j                   D ]  }|j                  j                  «        Œ y r®   )rP  rï  r<  )r·   rZ  s     r3   r<  zJukeboxLayerStack.del_cacheù  s%   € ØŸ/™/ò 	(ˆJØ�O‰O×%Ñ%Õ'ñ	(r5   r>  )rÀ   rÁ   rÂ   r°   rX  r¾   r<  rÃ   rÄ   s   @r3   rM  rM  Â  s   ø„ ô%ò()ó*ö(r5   rM  c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚJukeboxPositionalEmbeddingc                 ó€   •— t         ‰| �  «        t        j                  t	        j
                  ||f«      «      | _        y r®   )r¯   r°   r   r´   r!   r³   Úpos_emb)r·   rì   rù   r¸   s      €r3   r°   z#JukeboxPositionalEmbedding.__init__ÿ  s,   ø€ Ü‰ÑÔÜ—|‘|¤E§K¡K°¸EÐ0BÓ$CÓDˆ�r5   c                 ó   — | j                   }|S r®   )r_  )r·   r_  s     r3   r¾   z"JukeboxPositionalEmbedding.forward  s   € Ø—,‘,ˆØˆr5   r¿   rÄ   s   @r3   r]  r]  þ  s   ø„ ôEör5   r]  c                   ó|   ‡ — e Zd Z	 	 	 	 	 dˆ fd„	Z	 	 	 	 	 	 dd„Zd„ Z	 	 	 	 	 	 	 	 d	d„Zd„ Z	 	 	 	 	 	 	 	 	 d
d„Zˆ xZ	S )Ú JukeboxConditionalAutoregressivec                 óN  •— t         ‰| �  «        |j                  | _        |j                  | _        |�|n|j
                  | _        |�|n|j                  | _        t        j                  | j                  |j                  «      | _
        t        j                  |j                  «      | _        || _        || _        |s9t        j                   t#        j$                  d|j                  f«      «      | _        t)        | j
                  |j                  «      | _        t        j                  |j                  «      | _        t/        || j
                  ¬«      | _        || _        |j4                  | _        |j8                  rd| _        d| _        nd| _        d| _        |s†t        j>                  |j                  | j                  d¬«      | _         | j<                  r%| j                  jB                  | j@                  _!        t"        j                  jE                  «       | _#        yy)a`  
        Autoregressive model on either lyric tokens or music tokens, or both. The attention pattern should be properly
        set fro each configuration.

        Args:
            config (`JukeboxPriorConfig`):
                Model configuration class with all the parameters of the model. Initializing with a config file does
                not load the weights associated with the model, only the configuration. Check out the
                [`~PreTrainedModel.from_pretrained`] method to load the model weights.
            n_ctx (`int`, *optional*):
                Number of tokens or lyrics tokens provided in a single pass.
            embed_dim (`int`, *optional*):
                Either equals to the dimension of the codebook, or the sum of n_vocab (lyrics) and codebook dimension,
                if the model combines lyrics and music tokens, or simply n_vocab if the model is a seperate encoder
            audio_conditioning (`bool`, *optional*, defaults to `False`):
                Whether or not the prior supports conditionning on audio.
            metadata_conditioning (`bool`, *optional*, defaults to `False`):
                Whether or not the prior supports conditionning on artitst, genres, lyrics and timing.
            is_encoder (`bool`, *optional*, defaults to `False`):
                Whether the model is an encoder only model.
        Nr   )rP   FT©r¶   )$r¯   r°   rÃ  rù   rH  rP   Úmusic_vocab_sizerì   r   r  Úembed_tokensrÉ  Úemb_dropoutÚembed_tokens_dropoutÚmetadata_conditioningÚaudio_conditioningr´   r!   r³   Ústart_tokenr]  r_  Úpos_emb_dropoutrM  ÚtransformerÚ
is_encoderró  rô  Úmerged_decoderÚadd_cond_after_transformerÚshare_embed_tokens_fc_proj_outr†  Úfc_proj_outrµ   ÚCrossEntropyLossrº  )r·   rn   rP   rì   rj  ri  rn  r¸   s          €r3   r°   z)JukeboxConditionalAutoregressive.__init__	  s›  ø€ ô> 	‰ÑÔØ×'Ñ'ˆŒ
Ø ×+Ñ+ˆŒØ#Ð/‘U°V·\±\ˆŒ
Ø&/Ð&;™À×AXÑAXˆŒÜŸL™L¨¯©¸×9KÑ9KÓLˆÔÜ$&§J¡J¨v×/AÑ/AÓ$BˆÔ!Ø%:ˆÔ"Ø"4ˆÔÙ$Ü!Ÿ|™|¬E¯K©K¸¸F×<NÑ<NÐ8OÓ,PÓQˆDÔÜ1°$·*±*¸f×>PÑ>PÓQˆŒÜ!Ÿz™z¨&×*<Ñ*<Ó=ˆÔä,¨V¸4¿:¹:ÔFˆÔØ$ˆŒØ!×:Ñ:ˆÔà× Ò à.3ˆDÔ+Ø27ˆDÕ/à.2ˆDÔ+Ø26ˆDÔ/áÜ!Ÿy™y¨×);Ñ);¸T¿^¹^ÐRWÔXˆDÔØ×2Ò2Ø*.×*;Ñ*;×*BÑ*B�× Ñ Ô'ÜŸ™×1Ñ1Ó3ˆD�Ið	 r5   c                 óB  — |j                   d   }t        j                  «       5  |j                  |d«      j	                  «       }ddd«       | j
                  snt        j                  |d| j                  f|j                  | j                  j                  d   j                  j                  j                  j                  ¬«      }|}	| j                  |«      }
t        j                   |
dd…dd…f   |
dd…dd…f   fd¬«      }
| j"                  r$|j                  || j                  «      |
dd…df<   n| j$                  |
dd…df<   | j'                  |
«      | j)                  | j+                  «       «      z   |z   }
| j                  |
|¬«      }
| j,                  r|
|z   }
|
}| j.                  r|
S | j1                  |
«      }
t3        j4                  «       }|rå|
dd…d| j6                  …f   j9                  d| j:                  «      }|
dd…| j6                  d…f   j9                  d| j:                  «      } |||	dd…d| j6                  …f   j9                  d«      «      t=        j>                  d«      z  } |||	dd…| j6                  d…f   j9                  d«      «      t=        j>                  d«      z  }||f}nH ||
j                  d| j:                  «      |	j                  d«      «      t=        j>                  d«      z  }|r||
fS |r||fS |dfS # 1 sw Y   �ŒÑxY w)	zŸ
        Args:
            tokens (`torch.tensor`):
                Can represent music tokens, lyrics tokens or both, depending on the configuration.
        r   r   Nr   r[   r   )r"  r7   ) r^   r!   r-  rŸ   r<   rj  r;   rù   r>   rm  rP  rE  rÅ  rµ   r   rf  r:   ri  rk  rh  rl  r_  rp  rn  rr  r   rs  rô  Úreshaperì   ri   r0  )r·   rI   rj  ri  r"  Ú	get_predsÚget_actsÚget_sep_lossrp   Útargetr¼   ÚactivationsÚloss_fnÚlyric_hidden_statesÚtoken_hidden_statesÚ
lyric_lossÚmusic_token_lossrº  s                     r3   r¾   z(JukeboxConditionalAutoregressive.forwardH  s  € ð  —\‘\ !‘_ˆ
Ü�]‰]‹_ñ 	8Ø—[‘[ ¨RÓ0×5Ñ5Ó7ˆF÷	8ð ×&Ò&Ü!&§¡Ø˜Q §
¡
Ð+Ø—}‘}Ø×&Ñ&×1Ñ1°!Ñ4×8Ñ8×=Ñ=×DÑD×JÑJô"Ðð ˆØ×)Ñ)¨&Ó1ˆäŸ	™	 =²°B±C°Ñ#8¸-ÊÈ3ÈBÈ3ÈÑ:OÐ"PÐVWÔXˆØ×%Ò%Ø"7×"<Ñ"<¸ZÈÏÉÓ"TˆMš!˜Q˜$Òà"&×"2Ñ"2ˆMš!˜Q˜$Ñð ×%Ñ% mÓ4°t×7KÑ7KÈDÏLÉLËNÓ7[Ñ[Ð^pÑpð 	ð ×(Ñ(ØÐ6Pð )ó 
ˆð ×*Ò*Ø)Ð,>Ñ>ˆMà#ˆØ�?Š?Ø Ð à×(Ñ(¨Ó7ˆÜ×%Ñ%Ó'ˆÙØ"/²Ð3E°T×5EÑ5EÐ3EÐ0EÑ"F×"NÑ"NÈrÐSW×SaÑSaÓ"bÐØ"/²°4×3CÑ3CÑ3EÐ0EÑ"F×"NÑ"NÈrÐSW×SaÑSaÓ"bÐá Ð!4°fºQÐ@RÀ$×BRÑBRÐ@RÐ=RÑ6S×6[Ñ6[Ð\^Ó6_Ó`Ôce×ciÑciÐjmÓcnÑnˆJÙ&Ð':¸FÂ1Àd×FVÑFVÑFXÐCXÑ<Y×<aÑ<aÐbdÓ<eÓfÔik×ioÑioÐpsÓitÑtÐàÐ 0Ð1‰Dá˜=×-Ñ-¨b°$·.±.ÓAÀ6Ç;Á;ÈrÃ?ÓSÔVX×V\ÑV\Ð]`ÓVaÑaˆDáØ˜Ð&Ð&ÙØ˜Ð$Ð$à˜�:Ð÷g	8ñ 	8ús   ¤!LÌLc                 óF  — |dk(  r²t        j                  |d| j                  | j                  j                  j
                  ¬«      j                  | j                  j                  j                  «      }| j                  r$|j                  || j                  «      |d d …df<   n%| j                  |d d …df<   n| j                  |«      }|j                  || j                  | j                  fk(  r|d d …||dz   …d d …f   }n|}|| j                  «       ||dz    z   |z   }||fS )Nr   r   r   )r!   r³   rù   rf  rµ   r   r=   r>   ri  rŸ   rk  r^   rP   r_  )r·   r©   r·  rI   rj  ri  r¼   Úconds           r3   Úget_embz(JukeboxConditionalAutoregressive.get_embŽ  s	  € Ø�qŠ=Ü!ŸK™K¨	°1°d·j±jÈ×HYÑHY×H`ÑH`×HfÑHfÔg×jÑjØ×!Ñ!×(Ñ(×/Ñ/óˆMð ×)Ò)Ø&;×&@Ñ&@ÀÈDÏJÉJÓ&W�ša ˜dÒ#à&*×&6Ñ&6�ša ˜dÒ#à ×-Ñ-¨fÓ5ˆMØ×#Ñ#¨	°4·:±:¸t¿z¹zÐ'JÒJØ%¢a¨°H¸q±LÐ)@Â!Ð&CÑD‰Dà%ˆDà%¨¯©«°xÀ(ÈQÁ,Ð(OÑOÐRVÑVˆØ˜dÐ"Ð"r5   c
           	      óR  — |	€| j                   }	| j                  s†t        j                  |d| j                  f| j
                  j                  d   j                  j                  j                  j                  ¬«      j                  | j                  j                  «      }t        j                  «       5  g }
d }|rg }t        t!        d|	«      d¬«      }|D ]è  }|j#                  d|	› d�d¬	«       | j%                  |||||«      \  }}| j                  ||d¬
«      }| j&                  r||z   }| j                  |«      }|rj)                  |j+                  «       «       ||z  }t-        |||¬«      }t        j.                  j1                  |¬«      j3                  «       }|
j)                  |j+                  «       «       Œê ~| j
                  j5                  «        t        j6                  |
d¬«      }|rt        j6                  d¬«      }d d d «       |rfS S # 1 sw Y   ŒxY w)Nr   r   r   F©ÚleavezAncestral sampling ú music tokensT©Úrefreshr)  ©r+   r,   ©r*   r   )rP   rj  r!   r;   rù   rm  rP  rE  rÅ  rµ   r   r=   rr  r>   r-  r   r@   Úset_descriptionr‚  rp  rO   r   r4   ÚdistributionsÚCategoricalr¨   r<  r:   )r·   r·  rj  ri  r"  Útempr+   r,   rv  Úsample_tokensrt  rI   ÚpredsÚiterr©   r¼   r�  s                    r3   r¨   z'JukeboxConditionalAutoregressive.sample¡  s  € ð Ð Ø ŸJ™JˆMà×&Ò&Ü!&§¡Ø˜A˜tŸz™zÐ*°$×2BÑ2B×2MÑ2MÈaÑ2P×2TÑ2T×2YÑ2Y×2`Ñ2`×2fÑ2fô"ç‰b�×!Ñ!×(Ñ(Ó)ð ô �]‰]‹_ñ !	0ØˆNØˆFÙØ�äœ˜a Ó/°uÔ=ˆDØ ò 6�Ø×$Ñ$Ð':¸=¸/ÈÐ%WÐaeÐ$ÔfØ&*§l¡lØ˜i¨Ð1CÐEZó'Ñ#�˜tð !%× 0Ñ 0Ø!Ð>XÐaeð !1ó !�ð ×2Ò2Ø$1°DÑ$8�MØ $× 0Ñ 0°Ó ?�ÙØ—L‘L ×!4Ñ!4Ó!6Ô7à -°Ñ 4�Ü -¨mÀ5ÐPUÔ V�ä×,Ñ,×8Ñ8ÀÐ8ÓN×UÑUÓW�Ø×%Ñ% f§l¡l£nÕ5ð'6ð* Ø×Ñ×&Ñ&Ô(ä—Y‘Y˜~°1Ô5ˆFÙÜŸ	™	 %¨QÔ/�÷C!	0ñD Ø˜5�=Ð àˆM÷K!	0ð !	0ús   Â5EHÈH&c                 óH   — ||z   dz
  |z  }g |g|dz
  z  ¢|dz
  |z  dz   ‘}|S r‹  r_  )r·   ÚlengthÚ
chunk_sizeÚn_passesÚchunk_sizess        r3   Úsplit_chunksz-JukeboxConditionalAutoregressive.split_chunksÜ  sE   € Ø˜ZÑ'¨!Ñ+°
Ñ:ˆØU˜˜¨°1©Ñ5ÐU¸À¹
ÀjÑ7PÐSTÑ7TÐUˆØÐr5   c                 óÔ  — |€| j                   }|j                  d   }t        j                  «       5  |j	                  |d«      j                  «       }d d d «       t        j                  |dd¬«      }t        |«      }| j                  s|t        j                  |d| j                  f| j                  j                  d   j                  j                  j                  j                   ¬«      j#                  |j$                  «      }t        j                  «       5  |	rg }|
€t'        |«      }
| j)                  t'        |«      |
«      }g }d}d }t+        |dd¬«      D ]Ì  }g g }}t-        |||z   «      D ]A  }| j/                  |||||«      \  }}||   }|j1                  |«       |j1                  |«       ŒC ||z   }t        j2                  |d¬«      t        j2                  |d¬«      }}~~|	s~| j                  ||d	¬
«      }|	r$| j4                  r|z   }~|j1                  |«       ŒÌ~ŒÎ |	r9t        j2                  |d¬«      }| j7                  |«      }j1                  |«       |d   }t+        t-        t'        |«      |«      dt'        t-        t'        |«      |«      «      › d�d¬«      }|D ]Å  }| j/                  |||||«      \  }}| j                  ||d	¬
«      }| j4                  r||z   }| j7                  |«      }|	rj1                  |«       ||z  }t9        |||¬«      }t        j:                  j=                  |¬«      j?                  «       }|j1                  |jA                  «       «       |}ŒÇ ~~| j                  jC                  «        t        j2                  |d¬«      }|	rt        j2                  d¬«      }d d d «       |	rfS S # 1 sw Y   �Œ€xY w# 1 sw Y   ŒxY w)Nr   r   r   r   r   zPreparing past key valueF)rW   r…  Tr)  ú	Sampling r†  r‰  rŠ  )"rP   r^   r!   r-  rŸ   r<   Úsplitr?   rj  r;   rù   rm  rP  rE  rÅ  rµ   r   r=   r>   r9   r—  r   r@   r‚  rO   r:   rp  rr  r4   rŒ  r�  r¨   r   r<  )r·   r·  Úlyric_and_music_tokensrj  ri  r"  rŽ  r+   r,   rv  r”  r�  rp   Úsampled_audior�  r–  Úx_primesrS   r¨  Úcurrent_chunk_sizeÚsampled_audio_primeÚconds_primer©   Úx_primeÚ
cond_primeÚinput_tokensÚ
itereratorr¼   r�  rk   s                                 r3   Úprimed_samplez.JukeboxConditionalAutoregressive.primed_sampleá  sõ  € ð Ð Ø ŸJ™JˆMà+×1Ñ1°!Ñ4ˆ
Ü�]‰]‹_ñ 	XØ%;×%@Ñ%@ÀÈRÓ%P×%UÑ%UÓ%WÐ"÷	Xô Ÿ™Ð$:¸AÀ1ÔEˆÜ˜]Ó+ˆà×&Ò&Ü!&§¡Ø˜A˜tŸz™zÐ*°$×2BÑ2B×2MÑ2MÈaÑ2P×2TÑ2T×2YÑ2Y×2`Ñ2`×2fÑ2fô"ç‰bÐ'×.Ñ.Ó/ð ô �]‰]‹_ñ M	0ÙØ�ð Ð!Ü  Ó/�
Ø×+Ñ+¬C°Ó,>À
ÓKˆKØˆHØˆEØˆEä&*¨;Ð=WÐ_dÔ&eò  Ð"Ø35°r [Ð#Ü % e¨UÐ5GÑ-GÓ Hò 3�HØ*.¯,©,Ø  )¨UÐ4FÐH]ó+Ñ'�G˜Zð *¨(Ñ3�EØ'×.Ñ.¨wÔ7Ø×&Ñ& zÕ2ð3ð Ð 2Ñ2�Ü&+§i¡iÐ0CÈÔ&KÌUÏYÉYÐWbÐhiÔMj˜�Ø'ØÙ Ø"Ø×*Ñ*¨7ÐOiÐrvÐ*Ów�áØ×6Ò6Ø")¨JÑ"6˜Ø"Ø—O‘O GÕ,áð/ ñ2 ÜŸ)™) H°!Ô4�Ø×*Ñ*¨7Ó3�Ø—‘˜WÔ%ð )¨Ñ,ˆLäÜ”c˜-Ó(¨-Ó8Ø ¤¤U¬3¨}Ó+=¸}Ó%MÓ!NÐ OÈ}Ð]ØôˆJð
 'ò ,�Ø&*§l¡lØ˜i¨Ð7IÐK`ó'Ñ#�˜tð !%× 0Ñ 0Ø!Ð>XÐaeð !1ó !�ð ×2Ò2Ø$1°DÑ$8�MØ $× 0Ñ 0°Ó ?�ÙØ—L‘L Ô/à -°Ñ 4�Ü -¨mÀ5ÐPUÔ V�ä$×2Ñ2×>Ñ>ÀmÐ>ÓT×[Ñ[Ó]�Ø×$Ñ$ \×%7Ñ%7Ó%9Ô:Ø+‘ð',ð* ˜lØ×Ñ×&Ñ&Ô(ä Ÿ9™9 ]¸Ô:ˆLÙÜŸ	™	 %¨QÔ/�÷[M	0ñ\ Ø Ð&Ð&àÐ÷y	Xñ 	Xú÷M	0ð M	0ús   ²!OÄJ'OÏOÏO')NNFFF)NNNFFF)NNNr×   r   r   FN)	NNNr×   r   r   FNN)
rÀ   rÁ   rÂ   r°   r¾   r‚  r¨   r—  r¥  rÃ   rÄ   s   @r3   rb  rb    s‚   ø„ ð ØØ Ø#Øõ=4ðD  Ø"Ø#'ØØØóDòL#ð,  Ø"Ø#'ØØØØØó9òvð  Ø"Ø#'ØØØØØØ÷n r5   rb  c                   ó*   ‡ — e Zd ZdZˆ fd„Zdd„Zˆ xZS )ÚJukeboxMusicTokenConditionerzç
    The `JukeboxMusicTokenConditioner` takes music tokens as an input (coresponding to the codes of the VQVAE's
    codebook) and upsamples it using a single layer of decoder convolution block (the same is used in the VQVAE).
    c           	      ót  •— t         ‰| �  «        t        j                  |j                  |j
                  «      | _        |j                  |_        t        ||j
                  |j                  |j                  |j                  |   |j                  |   d¬«      | _        t        |j
                  «      | _        y )NF)rã   )r¯   r°   r   r  re  rÃ  rf  rì   r  r™  rš  rŒ  r�  Ú	upsamplerrÎ  rÖ  )r·   rn   ro   r¸   s      €r3   r°   z%JukeboxMusicTokenConditioner.__init__X  s˜   ø€ Ü‰ÑÔÜŸL™L¨×)@Ñ)@À&×BTÑBTÓUˆÔØ!×2Ñ2ˆÔä/ØØ×ÑØ×!Ñ!Ø×!Ñ!Ø×Ñ˜uÑ%Ø× Ñ  Ñ'Ø"ô
ˆŒô +¨6×+=Ñ+=Ó>ˆ�r5   c                 óê   — |€d}|j                  «       }| j                  |«      }||z   }|j                  ddd«      }| j                  |«      }|j                  ddd«      }| j	                  |«      }|S )a?  
        Args:
            music_tokens (`torch.LongTensor`):
                Music tokens form the uper level in range(nb_discrete_codes)
            raw_audio_conditionning (`torch.LongTensor`, *optional*):
                Audio used when primed sampling, raw audio information that conditions the generation
        r   r   r8   r   )r<   rf  r>  r©  rÖ  )r·   rk   Úraw_audio_conditionningr¼   s       r3   r¾   z$JukeboxMusicTokenConditioner.forwardh  s…   € ð #Ð*Ø&)Ð#à#×(Ñ(Ó*ˆØ×)Ñ)¨,Ó7ˆØ%Ð(?Ñ?ˆð &×-Ñ-¨a°°AÓ6ˆØŸ™ }Ó5ˆØ%×-Ñ-¨a°°AÓ6ˆØŸ™¨Ó6ˆØÐr5   r®   ©rÀ   rÁ   rÂ   Ú__doc__r°   r¾   rÃ   rÄ   s   @r3   r§  r§  R  s   ø„ ñô
?÷ r5   r§  c                   ó,   ‡ — e Zd ZdZdˆ fd„	Zdd„Zˆ xZS )ÚJukeboxRangeEmbeddinga“  
    The `JukeboxRangeEmbedding` interpolate the given [pos_start, pos_end] to obtain an equivalent of time positional
    embedding of length `n_ctx`.

    Binning process : For each pos in position tensor, find its bin [start,end) mapped to [0,1,...,bins-1] [start,end)
    -> [0,1) -> [0, bins) -> floor -> [0,...,bins-1] NOTE: Open ended interval on right, so start <= pos < end, not <=
    end
    c                 ó    •— t         ‰| �  «        || _        || _        t	        j
                  ||«      | _        |\  | _        | _        || _	        y r®   )
r¯   r°   Ún_timerì   r   r  ÚembÚpos_minÚpos_maxr‰   )r·   r±  rì   r@   Ú	out_widthr‰   r¸   s         €r3   r°   zJukeboxRangeEmbedding.__init__‰  sC   ø€ Ü‰ÑÔØˆŒØ"ˆŒÜ—<‘< 	¨9Ó5ˆŒØ%*Ñ"ˆŒ�d”lØˆ�
r5   c                 óz  — t        |j                  «      dk(  st        d|j                  › �«      ‚| j                  |k  j	                  «       sE|| j
                  k  j	                  «       r(t        d| j                  › d| j
                  › d|› �«      ‚|j                  «       }|�B| j                  r&|j                  | j                  | j
                  «      }|j                  «       }| j                  }|dk7  rPt        j                  d|t        j                  |j                  ¬«      j                  d|«      |z  }|||z
  |z  z   }n|}|| j                  z
  | j
                  | j                  z
  z  }| j                  |z  j                  «       j                  «       j!                  «       }| j#                  |«      S )	Nr8   z Expected shape with 2 dims, got z
Range is [ú,z), got r   r   rV   )r9   r^   r  r³  Úallr´  rg   r‰   r±  r!   Úaranger>   rŸ   rì   Úfloorr<   r`  r²  )r·   Ú	pos_startÚpos_endr±  ÚinterpolationÚpositionÚnormalised_positionÚbins_s           r3   r¾   zJukeboxRangeEmbedding.forward‘  sr  € ä�9—?‘?Ó# qÒ(ÜÐ>¸y¿¹Ð>OÐPÓQÐQØ—‘ 	Ñ)×.Ñ.Ô0°iÀ$Ç,Á,Ñ6N×5SÑ5SÔ5UÜ˜j¨¯©¨°a¸¿¹°~ÀWÈYÈKÐXÓYÐYà—O‘OÓ%ˆ	ØÐØ�zŠzØ!Ÿ-™-¨¯©°d·l±lÓC�à—m‘m“oˆGà—‘ˆØ�QŠ;ä—‘˜Q ¬e¯k©kÀ)×BRÑBRÔS×XÑXÐYZÐ\bÓcÐflÑlð ð ! G¨iÑ$7¸=Ñ#HÑH‰Hà ˆHð  (¨$¯,©,Ñ6¸4¿<¹<È$Ï,É,Ñ;VÑWÐØ—‘Ð"5Ñ5×<Ñ<Ó>×CÑCÓE×LÑLÓNˆØ�x‰x˜‹Ðr5   rè   r®   r¬  rÄ   s   @r3   r¯  r¯    s   ø„ ñõ÷r5   r¯  c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚJukeboxLabelConditionerc                 ó.  •— t         ‰| �  «        |j                  }|j                  }|j                  }|j
                  \  }}|j                  }|j                  | _        t        j                  ||«      | _
        t        j                  ||«      | _        || _        | j                  rm|j                  |z  |j                  |z  f}	d|j                  |z  f}
d}t        d||	|«      | _        t        |||
|«      | _        t        ||||d¬«      | _        y y )Nr   )r   r×   r   T)r‰   )r¯   r°   rÃ  Útiming_dimsr�  Úmetadata_dimsrP   Úmax_nb_genresr   r  Úbow_genre_embÚ
artist_embÚinclude_time_signalÚmin_durationÚmax_durationr¯  Útotal_length_embÚabsolute_pos_embÚrelative_pos_emb)r·   rn   rÉ  rì   rÄ  r�  Ú	nb_genresÚ
nb_artistsr¸  Útotal_length_rangeÚabsolute_pos_rangeÚrelative_pos_ranger¸   s               €r3   r°   z JukeboxLabelConditioner.__init__°  s  ø€ Ü‰ÑÔà×&Ñ&ˆ	Ø×(Ñ(ˆØ×,Ñ,ˆØ &× 4Ñ 4Ñˆ	�:Ø#Ÿ\™\Ðà#×1Ñ1ˆÔÜŸ\™\¨)°YÓ?ˆÔÜŸ,™, z°9Ó=ˆŒØ#6ˆÔ Ø×#Ò#Ø"(×"5Ñ"5¸Ñ"EÀv×GZÑGZÐ]jÑGjÐ!kÐØ"% v×':Ñ':¸]Ñ'JÐ!KÐØ!+ÐÜ$9¸!¸[ÐJ\Ð^gÓ$hˆDÔ!Ü$9Ø" KÐ1CÀYó%ˆDÔ!ô %:Ø" KÐ1CÀYÐVZô%ˆDÕ!ð $r5   c                 ó\  — |d d …dd…f   }|d d …dd…f   }|d d …dd…f   }|d d …dd…f   }|d d …dd …f   }| j                  |«      }|dk\  j                  «       j                  d«      }| j                  |j	                  d«      «      |z  j                  dd¬«      }	|	|z   }
| j                  rx|||z   }}|j                  «       }|j                  «       }|j                  «       }| j                  |«      | j                  ||«      z   | j                  ||z  ||z  «      z   }|
|fS d }|
|fS )Nr   r   r8   r\   r   TrJ  )
rÈ  rg   rC   rÇ  r‰   r/  rÉ  rÌ  rÍ  rÎ  )r·   rx   rF   rG   r“  ÚartistÚgenrerÈ  r£   Ú	genre_embÚ	start_embrS   rw   r_  s                 r3   r¾   zJukeboxLabelConditioner.forwardÉ  sa  € Ø¢ 1 Q 3 Ñ'ˆØš!˜Q˜q˜S˜&Ñ!ˆØš!˜Q˜q˜S˜&Ñ!ˆØš!˜Q˜q˜S˜&Ñ!ˆØš˜A™B˜‘ˆð —_‘_ VÓ,ˆ
à˜‘
×!Ñ!Ó#×-Ñ-¨aÓ0ˆØ×'Ñ'¨¯©°A«Ó7¸$Ñ>×CÑCÈÐSWÐCÓXˆ	Ø 
Ñ*ˆ	ð ×#Ò#Ø ¨&¡�3ˆEØ'×-Ñ-Ó/ˆLØ—K‘K“MˆEØ—)‘)“+ˆCà×%Ñ% lÓ3Ø×'Ñ'¨¨sÓ3ñ4à×'Ñ'¨°Ñ(<¸cÀLÑ>PÓQñRð ð ˜'Ð!Ð!ð ˆGØ˜'Ð!Ð!r5   r¿   rÄ   s   @r3   rÂ  rÂ  ¯  s   ø„ ôö2"r5   rÂ  c                   ó"  ‡ — e Zd ZdZeZd„ Zddefˆ fd„Zdd„Zd„ Z	d	„ Z
d
„ Zd„ Zd„ Zdd„Zdd„Zd„ Z	 	 	 	 	 	 	 	 dd„Zdd„Zd„ Zg dddfd„Z	 	 ddej,                  deeej2                        dee   dee   deej,                     f
d„Zˆ xZS )ÚJukeboxPrioruˆ  
    The JukeboxPrior class, which is a wrapper around the various conditioning and the transformer. JukeboxPrior can be
    seen as language models trained on music. They model the next `music token` prediction task. If a (lyric) `encoderÃ¹
    is defined, it also models the `next character` prediction on the lyrics. Can be conditionned on timing, artist,
    genre, lyrics and codes from lower-levels Priors.

    Args:
        config (`JukeboxPriorConfig`):
            Model configuration class with all the parameters of the model. Initializing with a config file does not
            load the weights associated with the model, only the configuration. Check out the
            [`~PreTrainedModel.from_pretrained`] method to load the model weights.
        level (`int`, *optional*):
            Current level of the Prior. Should be in range `[0,nb_priors]`.
        nb_priors (`int`, *optional*, defaults to 3):
            Total number of priors.
        vqvae_encoder (`Callable`, *optional*):
            Encoding method of the VQVAE encoder used in the forward pass of the model. Passing functions instead of
            the vqvae module to avoid getting the parameters.
        vqvae_decoder (`Callable`, *optional*):
            Decoding method of the VQVAE decoder used in the forward pass of the model. Passing functions instead of
            the vqvae module to avoid getting the parameters.
    c                 óB  — | j                   j                  }t        |t        j                  «      r,|j
                  j                  j                  dd|z  ¬«       �nt        |t        «      rh| j                   j                  r&|j
                  j                  j                  «        �nÈ|j
                  j                  j                  dd|z  ¬«       �nœt        |t        «      r,|j                  j                  j                  dd|z  ¬«       �n`t        |t        «      r6|j                  j
                  j                  j                  dd|z  ¬«       �nt        |t        «      rAt!        |d«      r5|j"                  j
                  j                  j                  dd|z  ¬«       nÉt        |t        «      r7t!        |d«      r+|j$                  j                  j                  dd|z  ¬«       n‚t        |t&        «      rr| j                   j                  r\|j(                  j*                  j                  j                  «        |j(                  j,                  j                  j                  «        t        |t        j.                  «      rI|j,                  j                  j                  «        |j
                  j                  j1                  d«       t        |t        j2                  «      r2|j,                  �%|j,                  j                  j                  «        y y y )Nr   r|  r}  r  Úlm_headrk  r×   )rn   r‚  r~  r   r  rµ   r€  r�  r¬   rƒ  r„  r]  r_  r¯  r²  rb  ÚhasattrrÜ  rk  rÆ   rÏ   Úweigthr¶   r   r…  r†  )r·   rˆ  r‚  s      r3   r‰  zJukeboxPrior._init_weights  s'  € Ø—[‘[×+Ñ+ˆ
ä�fœbŸl™lÔ+Ø�M‰M×Ñ×&Ñ&¨C°T¸JÑ5FÐ&ÖGÜ˜¤Ô.Ø�{‰{×#Ò#Ø—‘×"Ñ"×(Ñ(Ö*à—‘×"Ñ"×*Ñ*°¸À
Ñ9JÐ*ÖKÜ˜Ô :Ô;Ø�N‰N×Ñ×'Ñ'¨S°d¸ZÑ6GÐ'ÖHÜ˜Ô 5Ô6Ø�J‰J×Ñ×"Ñ"×*Ñ*°¸À
Ñ9JÐ*ÖKÜ˜Ô @ÔAÄgÈfÐV_ÔF`Ø�N‰N×!Ñ!×&Ñ&×.Ñ.°C¸TÀJÑ=NÐ.ÕOÜ˜Ô @ÔAÄgÈfÐVcÔFdØ×Ñ×#Ñ#×+Ñ+°¸$ÀÑ:KÐ+ÕLÜ˜Ô 5Ô6¸4¿;¹;×;OÒ;OØ�O‰O×"Ñ"×'Ñ'×-Ñ-Ô/Ø�O‰O× Ñ ×%Ñ%×+Ñ+Ô-Ü�fœbŸl™lÔ+Ø�K‰K×Ñ×"Ñ"Ô$Ø�M‰M×Ñ×$Ñ$ SÔ)Ü�fœbŸi™iÔ(¨V¯[©[Ð-DØ�K‰K×Ñ×"Ñ"Õ$ð .EÐ(r5   Nrn   c           
      óê  •— t         ‰	| �  |«       || _        || _        || _        |�|n|j
                  | _        d| j
                  › �| _        |j                  | _        |j                  dkD  | _	        |j                  | _        |j                  | _
        | j
                  dk7  | _        | j
                  dz
  | _        | j                  rt        || j
                  «      | _        |j                  | _        | j                  rt!        || j                   ¬«      | _        |j$                  | _        |j$                  r¯|j                  |j                  g| _        d|j(                  g| _        |j,                  | _        |j                  | _        t1        ||j                  |j                  z   |j(                  |j2                  z   | j                  xs | j                  d¬«      | _        �nO|j6                  }| j                  dk7  rø| j                  rì|j,                  | _        |j,                  | _        |j(                  | _        t1        || j                  | j<                  ddd¬«      | _        tA        |j,                  |j,                  «      | j>                  _!        tE        |j,                  «      | j>                  _#        tI        jJ                  |j,                  |j(                  d¬	«      | j>                  _&        nd| _        t1        || j                  xs | j                  | j                  ¬
«      | _        |j                  | _'        | j                  | jN                  z   | _(        tS        |jT                  |jV                  «      D ��cg c]
  \  }}||z  ‘Œ c}}| _,        | j
                  dk7  r| jX                  | j
                     nd | _-        t]        j^                  | jX                  d || j
                  z
   «      | _0        | j                  | j`                  z  | _1        td        jg                  d| j
                  › d| jZ                  › d| j`                  › d| jb                  › �«       y c c}}w )Nzpriors.r   r   )rÉ  T)rP   rì   rj  ri  F)rP   rì   rj  ri  rn  rd  )rj  ri  zLevel:z, Cond downsample:z, Raw to tokens:z, Sample length:)4r¯   r°   Úvqvae_encoderÚvqvae_decoderr]   ro   r½  rP   ró  Úlyric_conditioningÚencoder_loss_fractionrj  Ú
cond_levelr§  Úconditioner_blocksri  rÂ  Úmetadata_embeddingÚis_encoder_decoderÚinput_shapesÚlyric_vocab_sizeÚembed_dim_shiftrÃ  rù   rb  re  rm   Úencoder_configÚlyric_acts_widthÚencoder_widthÚencoder_dimr­  r¬   r  rÎ  Úfinal_layer_normr   r†  rÜ  Únext_token_prediction_loss_dimsÚtotal_loss_dimsre   r�  rŒ  r’  Úcond_downsampleri   r3  Úraw_to_tokensrc   ÚloggerÚinfo)
r·   rn   ro   Ú	nb_priorsrà  rá  rë  rœ  r�  r¸   s
            €r3   r°   zJukeboxPrior.__init__  s·  ø€ Ü‰Ñ˜Ô ð +ˆÔØ*ˆÔàˆŒØ#Ð/‘U°V·\±\ˆŒ
à#*¨4¯:©:¨,Ð!7ˆÔà—\‘\ˆŒ
à"(×"AÑ"AÀAÑ"EˆÔØ(.×(GÑ(GˆÔ%Ø%+×%AÑ%AˆÔ"ð #'§*¡*°¡/ˆÔØŸ*™* q™.ˆŒØ×"Ò"Ü&BÀ6È4Ï:É:Ó&VˆDÔ#ð &,×%AÑ%AˆÔ"Ø×%Ò%Ü&=¸fÐ^b×^uÑ^uÐZuÔ&vˆDÔ#ð #)×";Ñ";ˆÔØ×$Ò$à!'×!@Ñ!@À&Ç,Á,Ð OˆDÔØ$% v×'>Ñ'>Ð#?ˆDÔ Ø×+Ñ+ˆDŒJà,2×,KÑ,KˆDÔ)ä9ØØ×5Ñ5¸¿¹ÑDØ ×1Ñ1°F×4KÑ4KÑKØ$(×$;Ñ$;Ò$Y¸t×?YÑ?YØ&*ôˆDŽJð $×2Ñ2ˆNà×,Ñ,°Ò1°d×6MÒ6MØ(6×(BÑ(B�Ô%Ø%+×%7Ñ%7�Ô"Ø#)×#:Ñ#:�Ô Ü?Ø"Ø×7Ñ7Ø"×.Ñ.Ø',Ø*/Ø#ô �”ô (5°^×5OÑ5OÐQW×QcÑQcÓ'd�—‘Ô$Ü0@À×ASÑASÓ0T�—‘Ô-Ü')§y¡y°×1CÑ1CÀV×E\ÑE\ÐchÔ'i�—‘Õ$à01�Ô-ô :ØØ$(×$;Ñ$;Ò$Y¸t×?YÑ?YØ&*×&@Ñ&@ôˆDŒJð 06¯|©|ˆÔ,Ø#×<Ñ<¸t×?cÑ?cÑcˆÔä=@À×AUÑAUÐW]×WiÑWiÓ=j×k©\¨V°T˜F D›LÓkˆÔØ?C¿z¹zÈQº˜t×/Ñ/°·
±
Ò;ÐTXˆÔÜŸW™W T×%5Ñ%5Ð6N¸	ÀDÇJÁJÑ8NÐ%OÓPˆÔØ!ŸZ™Z¨$×*<Ñ*<Ñ<ˆÔä�‰Ø�T—Z‘Z�LÐ 2°4×3GÑ3GÐ2HÐHXÐY]×YkÑYkÐXlð mØ×)Ñ)Ð*ð,õ	
ùó ls   ÎQ/Fc                 ó  — |j                  «       }||d d …df<   t        | j                  «      |d d …df<   t        || j                  z  «      t        || j                  z  «      z   |d d …dd…f<   | j	                  |«      \  }}|r||fS |S r[  )r   rA   rc   ró  Úset_metadata_lyric_tokens)r·   rl   rS   rF   rG   rX   rx   rJ   s           r3   rb   zJukeboxPrior.get_metadatau  s•   € Ø—<‘<“>ˆØ%ˆ’�A�‰ä˜T×/Ñ/Ó0ˆ’�A�‰ô ˜v¨×(:Ñ(:Ñ:Ó;¼cÀ%È$×J\ÑJ\ÑB\Ó>]Ñ]ˆ’�A�a�C�Ñð !×:Ñ:¸8ÓDÑˆ�'ÙØ˜WÐ$Ð$àˆOr5   c                 óf  — | j                   dkD  �rt        j                  |j                  d   | j                   ft        j                  |j
                  ¬«      }g }t        |j                  d   «      D ]}  }|j                  «       dd…d| j                  j                  z   d…f   }||df   ||df   ||df   }}}t        || j                   |||«      \  }	}
|	||dd…f<   |j                  |
«       Œ t        j                  |dd…dd| j                  j                  z   …f   |fd¬«      |fS |dfS )	z�
        Processes the full labels to only retrieve the relevant lyric tokens and keep the metadata conditioning tokens.
        r   rV   Nr   r   r8   r   r   )ró  r!   r;   r^   r<   r>   r@   r   ræ  rÆ  rL   rO   r:   )r·   rl   Útokens_listÚindices_listÚidxrD   rF   rG   rH   rI   rJ   s              r3   rø  z&JukeboxPrior.set_metadata_lyric_tokens†  sJ  € ð ×(Ñ(¨1Ó,ÜŸ+™+Ø—‘˜a‘ $×"?Ñ"?Ð@ÌÏ
É
Ð[a×[hÑ[hôˆKð ˆLÜ˜VŸ\™\¨!™_Ó-ò -�Ø$Ÿl™l›nªQ°°D×4KÑ4K×4YÑ4YÑ0YÑ0[Ð-[Ñ\�Ø17¸¸Q¸±ÀÈÈQÈÁÐQWÐX[Ð]^ÐX^ÑQ_ h˜f�Ü";Ø ×!>Ñ!>ÀÈfÐV^ó#‘�˜ð '-�˜C¢˜FÑ#Ø×#Ñ# GÕ,ð-ô —	‘	˜6¢!Ð%P q¨4×+BÑ+B×+PÑ+PÑ'PÐ%PÐ"PÑQÐS^Ð_ÐegÔhØðð ð
 ˜4�<Ðr5   c                 ó®  — | j                   dk7  rÃ|| j                   dz
     }|dd…|| j                  z  || j                  z  …f   }| j                  | j                  z  |d   j                  d   z
  }|dkD  rVt	        j
                  d|«      j                  |j                  «      }t	        j                  ||fd¬«      j                  «       }|g}|S d}|S )zE
        Extracts current level's conditioning music tokens.
        r   r   Nr   r   )
ro   rò  rP   r^   r!   r;   r=   r>   r:   r<   )r·   rk   rS   rw   Úmusic_tokens_condÚmissing_cond_lenÚ	init_condÚmusic_tokens_condss           r3   Úget_music_tokens_condsz#JukeboxPrior.get_music_tokens_condsŸ  sæ   € ð �:‰:˜Š?Ø ,¨T¯Z©Z¸!©^Ñ <ÐØ,ªQ°¸×9MÑ9MÑ0MÐPSÐW[×WkÑWkÑPkÐ0kÐ-kÑlˆLØ#Ÿz™z¨T×-AÑ-AÑAÐDUÐVXÑDY×D_ÑD_Ð`bÑDcÑcÐØ !Ò#Ü!ŸK™K¨Ð+;Ó<×?Ñ?Ð@Q×@XÑ@XÓY�	Ü$)§I¡IÐ/@À)Ð.LÐRTÔ$U×$ZÑ$ZÓ$\Ð!Ø"3Ð!4Ðð "Ð!ð "&ÐØ!Ð!r5   c                 óò  — |d   j                   d   }t        t        |«      «      D ]3  }||   t        | j                  |   «      z   j                  |d«      ||<   Œ5 t        t        |«      «      D ]W  }||   �Œ	t        j                  || j                  |   | j                  f|d   j                  |d   j                  ¬«      ||<   ŒY t        j                  |d¬«      t        j                  |d¬«      fS )zÅ
        Shifts the input tokens to account for the dictionary merge. The embed_dim_shift give by how much the music
        tokens should be shifted by. It is equal to `lyric_vocab_size`.
        r   r   rV   r   r   )r^   r@   r9   rA   rê  rŸ   r!   r;   rè  rù   r   r>   r:   )r·   rI   Úcondsrp   r�   s        r3   Úprior_preprocesszJukeboxPrior.prior_preprocess¯  sì   € ð
 ˜A‘Y—_‘_ QÑ'ˆ
Ü”s˜6“{Ó#ò 	XˆAØ ™¤S¨×)=Ñ)=¸aÑ)@Ó%AÑA×GÑGÈ
ÐTVÓWˆF�1ŠIð	Xô ”s˜5“zÓ"ò 	ˆAØ�Q‰xÑÜ Ÿ;™;Ø ×!2Ñ!2°1Ñ!5°t·z±zÐBÈ&ÐQRÉ)Ï/É/ÐbhÐijÑbk×brÑbrô��a’ð	ô �y‰y˜ QÔ'¬¯©°5¸aÔ)@Ð@Ð@r5   c                 óš  — |j                   d   }| j                  d   |j                   d   | j                  d   z
  f}t        t        j                  ||d¬«      «      }t        t        |«      «      D ]R  }t        | j                  |   «      }||   |z
  j                  |d«      ||<   t        j                  ||   d¬«      ||<   ŒT |d   S )zü
        Shifts back the input tokens if the model uses an encoder decoder architecture. As the embedding layer is
        shared, `prior_embed_dim_shift` shifts the music token ids by `lyric_vocab_size`. Only returns the music
        tokens.
        r   r   r   r   )r   )r^   rè  r?   r!   rš  r@   r9   rA   rê  rŸ   r‰   )r·   rI   rp   Údimsr�   Ú
bins_shifts         r3   Úprior_postprocesszJukeboxPrior.prior_postprocessÀ  sÇ   € ð —\‘\ !‘_ˆ
Ø×!Ñ! !Ñ$ f§l¡l°1¡o¸×8IÑ8IÈ!Ñ8LÑ&LÐMˆÜ”e—k‘k &¨$°AÔ6Ó7ˆô ”s˜6“{Ó#ò 	6ˆAÜ˜T×1Ñ1°!Ñ4Ó5ˆJØ ™ ZÑ/×5Ñ5°jÀ"ÓEˆF�1‰IÜŸ™ F¨1¡I°1Ô5ˆF�1ŠIð	6ð
 �b‰zÐr5   c                 ó¢   — |d| j                   dz    }d}t        t        t        || j                  g«      «      «      D ]  \  }} |||«      }Œ |S )zj
        Embeds the upper level music tokens and upsamples them to provide as audio conditioning.
        Nr   )rä  rj   r?   re   rå  )r·   r  rj  rþ  Úconditioner_blocks        r3   rf  zJukeboxPrior.embed_tokensÒ  sj   € ð 0Ð0E°$·/±/ÀAÑ2EÐFÐØ!ÐÜ4<¼TÄ#ÐFXÐ[_×[rÑ[rÐZsÓBtÓ=uÓ4vò 	ZÑ0ÐÐ0Ù!2Ð3DÐFXÓ!YÑð	Zà!Ð!r5   c                 ó¼   — |€| j                   }|€| j                  }t        j                  «       5  | j	                  ||||¬«      }ddd«       |S # 1 sw Y   S xY w)zi
        Encodes the hidden states (raw audio) using the VQVAE's encoder. Returns latent_states.
        N©rj  rk  r§  )ro   r]   r!   r-  rà  )r·   r¼   rj  rk  r§  r4  s         r3   rY  zJukeboxPrior.encodeÜ  sj   € ð ÐØŸ*™*ˆKØÐØŸ™ˆIä�]‰]‹_ñ 	Ø ×.Ñ.Ø¨;À)ÐW`ð /ó ˆM÷	ð Ð÷		ð Ðúó   ±AÁAc                 ó¼   — |€| j                   }|€| j                  }t        j                  «       5  | j	                  ||||¬«      }ddd«       |S # 1 sw Y   S xY w)zK
        Usamples the sequence of codebook vectors to a raw audio.
        Nr  )ro   r]   r!   r-  rá  )r·   rk   rj  rk  r§  rK  s         r3   r\  zJukeboxPrior.decodeë  sh   € ð ÐØŸ*™*ˆKØÐØŸ™ˆIÜ�]‰]‹_ñ 	Ø×'Ñ'Ø¨+ÀÐV_ð (ó ˆF÷	ð ˆ÷		ð ˆúr  c                 ó  — |�3|j                   d   | j                  z
  }|dd…d|…f   |dd…|d…f   }}nd\  }}| j                  r| j                  |«      nd\  }}| j                  r| j                  |«      n|}|||fS )z“
        Converts the input tokens to input_embeddings. Splits the lyrics form the rest of the metadata. Lyric tokens
        can be None.
        Nr   )NN)r^   ró  ri  ræ  rj  rf  )r·   r  rx   Ún_labelsÚlyric_tokensri  Úmetadata_posrj  s           r3   Úget_condzJukeboxPrior.get_condù  s    € ð
 ÐØ—~‘~ aÑ(¨4×+HÑ+HÑHˆHØ%-ªa°°(°¨lÑ%;¸XÂaÈÉÀlÑ=S�l‰Hà%/Ñ"ˆH�là15×1KÒ1KˆD×#Ñ# HÔ-ÐQ]ñ 	,Ð˜|ð GK×F]ÒF]˜T×.Ñ.Ð/AÔBÐcoÐØ!Ð#8¸,ÐFÐFr5   c
                 óÖ  — |du xs |j                   d   dk(  }
dddœ|
   }t        j                  |› d|› d|› d	|› d
|› �	«       t        j                  «       5  | j                  ||«      \  }}}| j                  r{|
r| j                  |gd|g«      \  }}n| j                  ||gd|g«      \  }}|	�|	| j                  z  }	| j                  j                  |||||||||	¬«	      }| j                  |«      }n^| j                  |d¬«      }|
r$| j                  j                  ||||||||	¬«      }n%| j                  j                  ||||||||||	¬«
      }ddd«       |S # 1 sw Y   |S xY w)a  
        Ancestral/Prime sampling a window of tokens using the provided conditioning and metadatas.

        Args:
            n_samples (`int`):
                Number of samples to generate.
            music_tokens (`List[torch.LongTensor]`, *optional*):
                Previously gemerated tokens at the current level. Used as context for the generation.
            music_tokens_conds (`List[torch.FloatTensor]`, *optional*):
                Upper-level music tokens generated by the previous prior model. Is `None` if the generation is not
                conditionned on the upper-level tokens.
            metadata (`List[torch.LongTensor]`, *optional*):
                List containing the metatdata tensor with the artist, genre and the lyric tokens.
            temp (`float`, *optional*, defaults to 1.0):
                Sampling temperature.
            top_k (`int`, *optional*, defaults to 0):
                Top k probabilities used for filtering.
            top_p (`float`, *optional*, defaults to 0.0):
                Top p probabilities used for filtering.
            chunk_size (`int`, *optional*):
                Size of the chunks used to prepare the cache of the transformer.
            sample_tokens (`int`, *optional*):
                Number of tokens to sample.

        Nr   r   Ú	AncestralÚPrimed)TFz
 sampling z samples with temp=z, top_k=z, top_p=)rŽ  r+   r,   r”  r�  T)r¨   )rŽ  r+   r,   r�  )r^   rô  rõ  r!   r-  r  rç  r  ró  rm   r¥  r	  Úget_encoder_statesr¨   )r·   r·  rk   r  rx   rŽ  r+   r,   r”  r�  Úno_past_contextÚnamerj  ri  r  r›  r"  s                    r3   r¨   zJukeboxPrior.sample	  sé  € ðJ '¨$Ð.ÒL°,×2DÑ2DÀQÑ2GÈ1Ñ2LˆØ!¨(Ñ3°OÑDˆÜ�‰�t�f˜J y kÐ1DÀTÀFÈ(ÐSXÐRYÐYaÐbgÐahÐiÔjä�]‰]‹_ñ 3	àFJÇmÁmÐTfÐhpÓFqÑCÐÐ 5°|Ø×&Ò&Ù"ØAE×AVÑAVØ%˜¨Ð/AÐ(BóBÑ>Ð*Ñ,>ð BF×AVÑAVØ% |Ð4°tÐ=OÐ6PóBÑ>Ð*Ð,>ð !Ð,Ø! T×%BÑ%BÑB�MØ#Ÿz™z×7Ñ7ØØ*Ø&Ø)ØØØØ)Ø"/ð  8ó 
 �ð  $×5Ñ5°lÓC‘à-1×-DÑ-DÀ\ÐZ^Ð-DÓ-_Ð*Ù"Ø#'§:¡:×#4Ñ#4Ø!Ø*Ø-Ø2Ø!Ø#Ø#Ø&3ð $5ó 	$‘Lð $(§:¡:×#;Ñ#;Ø!Ø$Ø*Ø-Ø2Ø!Ø#Ø#Ø#-Ø&3ð $<ó $�L÷Q3	ðh Ð÷i3	ðh Ðús   ÁC<EÅE(c                 ó0  — | j                   dk7  r„| j                  rx|r*| j                  j                  |j                  «      | _        | j                  |ddd«      }| j                  j                  |«      }| j                  j                  |«      }|S d}|S )z›
        Retrieve the last hidden_states of the lyric encoder that will be attended to by the decoder. Forwards through
        the lyric encoder.
        r   N)ró  râ  r­  r=   r>   r  rï  )r·   r  r¨   Ú
lyric_actsr"  s        r3   r  zJukeboxPrior.get_encoder_statesh  sŠ   € ð
 ×(Ñ(¨AÒ-°$×2IÒ2IÙØ#Ÿ|™|Ÿ™¨|×/BÑ/BÓC�”ØŸ™ l°D¸$ÀÓEˆJØŸ™×-Ñ-¨jÓ9ˆJØ)-¯©×)FÑ)FÀzÓ)RÐ&ð *Ð)ð *.Ð&Ø)Ð)r5   c                 óX  — | j                   r|| j                  j                  |«      }t        j                  j                  |j                  d| j                  «      |j                  d«      «      t        j                  d«      z  }|S t        j                  d|j                  ¬«      }|S )zW
        Computes the loss for the lyric encoder: next lyric token prediction.
        r   r7   r   r˜   )râ  r­  rÜ  r   r    Úcross_entropyrŸ   rî  ri   r0  r!   Útensorr>   )r·   r"  Útarget_lyricsÚencoder_losss       r3   Úget_encoder_losszJukeboxPrior.get_encoder_lossw  s’   € ð ×"Ò"Ø)-¯©×)=Ñ)=Ð>XÓ)YÐ&ÜŸ=™=×6Ñ6Ø*×/Ñ/°°D×4DÑ4DÓEÀ}×GYÑGYÐZ\ÓG]óä—‘�s“ñˆLð
 Ðô !Ÿ<™<¨Ð4N×4UÑ4UÔVˆLØÐr5   c                 óœ  — |r%| j                   j                  j                  |«       | j                  ||«      \  }}}| j                  r6| j                  ||gd|g«      \  }	}| j                  |	||d|¬«      \  \  }
}}n<| j                  |«      }| j                  ||«      }
| j                  |||||¬«      \  }}| j                  |
z  | j                  z  | j                  z  }||| j                  z  | j                  z  z  }|j                  «       j                  «       |
j                  «       j                  «       |j                  «       j                  «       dœ}|r!|j                  «       j                  «       |d<   |rG| j                   j                  j                  }| j                   j                  j                  d«       |S ||fS )z¢
        Applies a forward pass using the conditioning tokens. Different from the classic forward as it does not use the
        vqvae's encoding layers.
        NT)rx  rv  )rv  )Úbpdr!  Únext_token_prediction_lossr�  F)rm   rm  rX  r  rç  r  r  r"  rã  ró  rñ  rð  r`  r   rQ  )r·   rk   r  rx   rv  rY   rj  ri  r  rI   r!  r%  r�  r"  rº  rs  rQ  s                    r3   rf   zJukeboxPrior.forward_tokens„  sÞ  € ñ Ø�J‰J×"Ñ"×2Ñ2Ð3CÔDØBFÇ-Á-ÐPbÐdlÓBmÑ?ÐÐ1°<à×"Ò"Ø)-×)>Ñ)>Ø˜|Ð,¨tÐ5GÐ.Hó*Ñ&ˆFÐ&ð AEÇ
Á
ØÐ*Ð,AÐPTÐ`ið AKó AÑ=Ñ6ˆ\Ð5¹ð *.×)@Ñ)@ÀÓ)NÐ&Ø×0Ñ0Ð1KÈ\ÓZˆLØ04·
±
ØØ"Ø%Ø*Ø#ð 1;ó 1Ñ-Ð&¨ð ×)Ñ)¨LÑ8¸4×;XÑ;XÑXÐ[_×[oÑ[oÑoˆØÐ*¨T×-QÑ-QÑQÐTX×ThÑThÑhÑhˆð .×4Ñ4Ó6×<Ñ<Ó>Ø(×/Ñ/Ó1×7Ñ7Ó9Ø*D×*KÑ*KÓ*M×*SÑ*SÓ*Uñ
ˆñ
 Ø$Ÿ|™|›~×3Ñ3Ó5ˆG�GÑÙØ!%§¡×!7Ñ!7×!JÑ!JÐØ�J‰J×"Ñ"×2Ñ2°5Ô9Ø%Ð%à˜�=Ð r5   r¼   rx   r\  rv  r¥  c                 ó¶   — |j                   d   }| j                  ||¬«      ^}}| j                  ||||¬«      \  }}	|r| j                  |g|¢«      }
nd}
|
||	fS )a÷  
        Encode the hidden states using the `vqvae` encoder, and then predicts the next token in the `forward_tokens`
        function. The loss is the sum of the `encoder` loss and the `decoder` loss.

        Args:
            hidden_states (`torch.Tensor`):
                Hidden states which should be raw audio
            metadata (`List[torch.LongTensor]`, *optional*):
                List containing the metadata conditioning tensorwith the lyric and the metadata tokens.
            decode (`bool`, *optional*, defaults to `False`):
                Whether or not to decode the encoded to tokens.
            get_preds (`bool`, *optional*, defaults to `False`):
                Whether or not to return the actual predicitons of the model.
        r   )r§  )rk   r  rx   rv  N)r^   rY  rf   r\  )r·   r¼   rx   r\  rv  rp   rk   r  rº  rs  rE  s              r3   r¾   zJukeboxPrior.forward±  s‚   € ð* #×(Ñ(¨Ñ+ˆ
Ø,0¯K©K¸ÐQ[¨KÓ,\Ð)ˆÐ)Ø×+Ñ+Ø%Ø1ØØð	 ,ó 
‰ˆˆgñ Ø!%§¡¨lÐ-PÐ=OÐ-PÓ!QÑà!%ÐØ! 4¨Ð0Ð0r5   )Nr\   NNrè   )NNr   )NNNr×   r   r   NN)FF)rÀ   rÁ   rÂ   r­  r   r¼  r‰  r°   rb   rø  r  r  r	  rf  rY  r\  r  r¨   r  r"  rf   r!   r¾  r   r   Ú
LongTensorr(   r¾   rÃ   rÄ   s   @r3   rÚ  rÚ  ç  sò   ø„ ñð. &€Lò%ñ6W
Ð1õ W
órò" ò2"ò Aò"ò$"óóòGð& ØØØØØØØó]ó~*òð 02¸DÈEÐdió+!ðb "'Ø$)ñ!1à—|‘|ð!1ð ˜4 × 0Ñ 0Ñ1Ñ2ð!1ð ˜‘ð	!1ð
 ˜D‘>ð!1ð 
ˆe�l‰lÑ	÷!1r5   rÚ  c                   ó4   ‡ — e Zd ZdZeZdZdZd„ Zˆ fd„Z	ˆ xZ
S )ÚJukeboxPreTrainedModelz†
    An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained
    models.
    ÚjukeboxFc                 ó|   — t        |t        «      st        |t        «      r|j                  |j                  «       y y r®   )r~  rÚ  ry  Úapplyr‰  r‡  s     r3   r‰  z$JukeboxPreTrainedModel._init_weightsß  s-   € Ü�fœlÔ+¬z¸&Ä,Ô/OØ�L‰L˜×-Ñ-Õ.ð 0Pr5   c                 ó$   •— t        ‰| �  |i |¤Ž y r®   )r¯   r°   )r·   ÚinputsÚkwargsr¸   s      €r3   r°   zJukeboxPreTrainedModel.__init__ã  s   ø€ Ü‰Ñ˜&Ð+ FÓ+r5   )rÀ   rÁ   rÂ   r­  r   r¼  r½  Úsupports_gradient_checkpointingr‰  r°   rÃ   rÄ   s   @r3   r)  r)  Õ  s*   ø„ ñð
 !€LØ!ÐØ&+Ð#ò/÷,ð ,r5   r)  a"  
            labels (`List[torch.LongTensor]` of length `n_sample`, and shape `(self.levels, self.config.max_nb_genre + lyric_sequence_length)` :
                List of metadata such as `artist_id`, `genre_id` and the full list of lyric tokens which are used to
                condition the generation.
            sampling_kwargs (`Dict[Any]`):
                Various additional sampling arguments that are used by the `_sample` function. A detail list of the
                arguments can bee seen in the [`_sample`] function documentation.
ao  The bare JUKEBOX Model used for music generation. 4 sampling techniques are supported : `primed_sample`, `upsample`,
    `continue_sample` and `ancestral_sample`. It does not have a `forward` method as the training is not end to end. If
    you want to fine-tune the model, it is recommended to use the `JukeboxPrior` class and train each prior
    individually.
    c                   ó°  ‡ — e Zd ZdgZˆ fd„Zd„ Zdd„Zdd„Zd„ Zd„ Z	d„ Z
d	„ Z ej                  «       	 	 	 	 	 	 	 	 	 	 	 dd
eej                     fd„«       Z ed«      dd
eej                     fd„«       Z ede«      d
eej                     fd„«       Z ede«      d
eej                     fd„«       Z ede«      d
eej                     fd„«       Zˆ xZS )ÚJukeboxModelrA  c           	      ó,  •— t         ‰| �  |«       |j                  }t        |«      | _        | j                  |«       t        j                  t        |j                  «      D �cg c]  }t        |j                  |   |«      ‘Œ c}«      | _        y c c}w r®   )r¯   r°   Úvqvae_configry  rz  Úset_shared_paramsr   rà   r@   rö  rÚ  Úprior_configsÚpriors)r·   rn   r4  ro   r¸   s       €r3   r°   zJukeboxModel.__init__ü  st   ø€ Ü‰Ñ˜Ô Ø×*Ñ*ˆÜ! ,Ó/ˆŒ
Ø×Ñ˜vÔ&Ü—m‘mÜKPÐQW×QaÑQaÓKbÖcÀ%Œ\˜&×.Ñ.¨uÑ5°uÕ=Òcó
ˆ�ùÚcs   Á$ Bc                 óò   — |j                   D ]h  }|j                  |_        |j                  |_        |j                  |_        |j                  |_        |j
                  |_        |j                  |_        Œj y)zÄ
        Initialises the parameters that are shared. This has to be done here because the list of `JukeboxPriorConfig`
        is nest, and is thus unreachable in the `from_dict` function
        N)r6  r�  rÄ  rÊ  rË  rÆ  ri  )r·   Úmodel_configrn   s      r3   r5  zJukeboxModel.set_shared_params	  so   € ð
 #×0Ñ0ò 	NˆFØ#/×#=Ñ#=ˆFÔ Ø!-×!9Ñ!9ˆFÔØ".×";Ñ";ˆFÔØ".×";Ñ";ˆFÔØ#/×#=Ñ#=ˆFÔ Ø+7×+MÑ+MˆFÕ(ñ	Nr5   c                 ó>   — | j                   j                  ||||«      S r®   )rz  r\  )r·   rk   rj  rk  r§  s        r3   r\  zJukeboxModel.decode	  s   € Ø�z‰z× Ñ  ¨{¸IÀyÓQÐQr5   c                 ó>   — | j                   j                  ||||«      S r®   )rz  rY  )r·   rp  rj  rk  r§  s        r3   rY  zJukeboxModel.encode	  s   € Ø�z‰z× Ñ  ¨k¸9ÀiÓPÐPr5   c                 ó6  — ||z   dz
  |z  }t        |t        j                  «      rt        j                  ||d¬«      S t        |t        «      r5t	        t        |D �cg c]  }t        j                  ||d¬«      ‘Œ c}Ž «      S |€d g|z  S t        d«      ‚c c}w )Nr   r   r   zUnknown input type)r~  r!   r¾  rš  r?   re   r  )r·   Úobjr·  Ú
split_sizer•  rƒ   s         r3   Úsplit_batchzJukeboxModel.split_batch	  s�   € Ø 
Ñ*¨QÑ.°:Ñ=ˆÜ�cœ5Ÿ<™<Ô(Ü—;‘;˜s J°AÔ6Ð6Ü˜œTÔ"ÜœÈsÖSÀtœeŸk™k¨$°
ÀÖBÒSÐTÓUÐUØˆ[Ø�6˜HÑ$Ð$äÐ0Ó1Ð1ùò	 Ts   ÁBc           	      óÎ   — | j                   |   }||   }	|j                  }
|	j                  d   }||
|z
  k  r||z   |d<   d}n|
|d<   ||
z
  |z   }| j                  |||||||«      S )Nr   r�  r   )r7  rP   r^   Úsample_single_window)r·   rk   rl   rG   Úsampling_kwargsro   Útokens_to_sampleÚmax_batch_sizerm   rt  rP   Únb_sampled_tokensrS   s                r3   Úsample_partial_windowz"JukeboxModel.sample_partial_window$	  s“   € ð —‘˜EÑ"ˆØ% eÑ,ˆØ—‘ˆØ*×0Ñ0°Ñ3ÐØ˜uÐ'7Ñ7Ò7Ø/@ÐCSÑ/SˆO˜OÑ,Ø‰Eà/4ˆO˜OÑ,Ø%¨Ñ-Ð0@Ñ@ˆEà×(Ñ(¨°v¸vÀÐX]Ð_dÐftÓuÐur5   c                 óü  — | j                   |   }|d   j                  d   }	|j                  }
||
z   }||   d d …||…f   }|j                  dd «      }d|v r||z
  }|j                  d   }||j                  d   z
  }t        j                  d|› d|› d||z   › d|› d�	«       |dk  r|S |j                  |||«      }|j                  ||| j                  |«      }| j                  ||	|«      }| j                  ||	|«      }| j                  ||	|«      }g }t        t        |||«      d	¬
«      }|D ]ˆ  \  }}}ddg|j                  d   dk(     }|j                  d|› d|› d|› d| j                  |j                  z  › �d¬«        |j                  d|j                  d   |||dœ|¤Ž}|j                  |«       ŒŠ t!        j"                  |d¬«      }|d d …| d …f   }t!        j"                  ||   |gd¬«      ||<   |S )Nr   r�  r   r™  z tokens for [r·  z]. Conditioning on z tokensFr„  r  r  z[prior level z] z
 Sampling z tokens out of Tr‡  )r·  rk   r  rx   r   r_  )r7  r^   rP   Úgetrô  rõ  r  rb   rF   r?  r   re   r‹  ró  r¨   rO   r!   r:   )r·   rk   rl   rG   rB  ro   rS   rD  rm   r·  rP   rw   Úprevious_sampled_tokensr�  Úconditioning_tokensÚ
new_tokensr  rx   r²  Úmusic_tokens_conds_listÚmetadata_listrI   rü   r«  Úmusic_tokens_conds_ir~   r  r}   rt  Úmusic_tokens_news                                 r3   rA  z!JukeboxModel.sample_single_window5	  sˆ  € Ø—‘˜EÑ"ˆØ  ‘O×)Ñ)¨!Ñ,ˆ	Ø—‘ˆØ�e‰mˆà".¨uÑ"5²a¸¸s¸°lÑ"CÐà'×+Ñ+¨O¸TÓBˆØ˜oÑ-Ø %™KˆMà5×;Ñ;¸AÑ>ÐØ"Ð%<×%BÑ%BÀ1Ñ%EÑEˆ
ä�‰Ø˜� m°E°7¸!¸EÀMÑ<QÐ;Rð SØ#Ð$ Gð-ô	
ð
 ˜Š?àÐð #×9Ñ9¸,ÈÈsÓSÐð ×%Ñ% f¨e°T×5FÑ5FÈÓOˆà ×,Ñ,Ð-DÀiÐQ_Ó`ÐØ"&×"2Ñ"2Ð3EÀyÐR`Ó"aÐØ×(Ñ(¨°9¸nÓMˆØˆÜœÐ-Ð/FÈÓVÐ^cÔdˆØ@Hò 	$Ñ<ˆNÐ0°*Ø Ð*¨>×+?Ñ+?ÀÑ+BÀaÑ+GÑHˆDØ×$Ñ$Ø ˜w b¨¨¨j¸¸ð HØ×%Ñ%¨×)<Ñ)<Ñ<Ð=ð?àð %ô ð
 $�u—|‘|ð Ø(×.Ñ.¨qÑ1Ø+Ø#7Ø#ñ	ð
 "ñˆHð �M‰M˜(Õ#ð	$ô Ÿ™ 6¨qÔ1ˆð *ª!¨j¨[©\¨/Ñ:ÐÜ#Ÿi™i¨°eÑ)<Ð>NÐ(OÐUVÔWˆ�UÑØÐr5   c	           
      óô   — || j                   |   j                  k\  rDt        || j                   |   j                  |«      }	|	D ]  }
| j                  ||||||
|«      }Œ |S | j	                  |||||||«      }|S r®   )r7  rP   rT   rA  rF  )r·   rk   rl   rG   rB  ro   rF   rQ   rD  rü   rS   s              r3   Úsample_levelzJukeboxModel.sample_levelp	  s™   € ð ˜4Ÿ;™; uÑ-×3Ñ3Ò3Ü! ,°·±¸EÑ0B×0HÑ0HÈ*ÓUˆHØ!ò �Ø#×8Ñ8Ø  &¨&°/À5È%ÐQ_ó ‘ðð Ðð  ×5Ñ5Ø˜f f¨o¸uÀlÐTbóˆLð Ðr5   r¥  c                 ót  — | j                   d   }|�|}n<t        |	| j                  j                  z  «      |j                  z  |j                  z  }|€t        t        | j                   «      «      }|| _        |D �]"  }|t        | j                   «      dz
  k(  rdn|||dœ}|| j                   |   j                  z  }t        | j                  j                  |   | j                   |   j                  z  «      }||k7  r|n|}| j                  |||   ||||||«      }|sŒ¦| j                  j                  ||   j                  «       t        j                  «       5  t        | j                   «      |z
  dz
  }| j                  j!                  |d|dz    |||   j"                  d   ¬«      }ddd«       d|› �}t$        j&                  j)                  |«      st%        j*                  |«       t-        |||j/                  «       ¬«       |
s�Œ”| j                   d   €�Œ¥| j                   d   j0                  dkD  s�ŒÃt        j                  «       5  t3        ||d   | j                   d   | j                  «      }ddd«       t        j4                  d	i|› d
�«       �Œ% |S # 1 sw Y   Œ÷xY w# 1 sw Y   Œ5xY w)aH  
        Core sampling function used to generate music tokens. Iterates over the provided list of levels, while saving
        the generated raw audio at each step.

        Args:
            music_tokens (`List[torch.LongTensor]`):
                A sequence of music tokens of length `self.levels` which will be used as context to continue the
                sampling process. Should have `self.levels` tensors, each corresponding to the generation at a certain
                level.
            labels (`List[torch.LongTensor]`):
                List of length `n_sample`, and shape `(self.levels, 4 + self.config.max_nb_genre +
                lyric_sequence_length)` metadata such as `artist_id`, `genre_id` and the full list of lyric tokens
                which are used to condition the generation.
            sample_levels (`List[int]`):
                List of the desired levels at which the sampling will be done. A level is equivalent to the index of
                the prior in the list of priors
            metas (`List[Any]`, *optional*):
                Metadatas used to generate the `labels`
            chunk_size (`int`, *optional*, defaults to 32):
                Size of a chunk of audio, used to fill up the memory in chuncks to prevent OOM erros. Bigger chunks
                means faster memory filling but more consumption.
            sampling_temperature (`float`, *optional*, defaults to 0.98):
                Temperature used to ajust the randomness of the sampling.
            lower_batch_size (`int`, *optional*, defaults to 16):
                Maximum batch size for the lower level priors
            max_batch_size (`int`, *optional*, defaults to 16):
                Maximum batch size for the top level priors
            sample_length_in_seconds (`int`, *optional*, defaults to 24):
                Desired length of the generation in seconds
            compute_alignments (`bool`, *optional*, defaults to `False`):
                Whether or not to compute the alignment between the lyrics and the audio using the top_prior
            sample_tokens (`int`, *optional*):
                Precise number of tokens that should be sampled at each level. This is mostly useful for running dummy
                experiments
            offset (`int`, *optional*, defaults to 0):
                Audio offset used as conditioning, corresponds to the starting sample in the music. If the offset is
                greater than 0, the lyrics will be shifted take that intoaccount
            save_results (`bool`, *optional*, defaults to `True`):
                Whether or not to save the intermediate results. If `True`, will generate a folder named with the start
                time.
            sample_length (`int`, *optional*):
                Desired length of the generation in samples.

        Returns: torch.Tensor

        Example:

        ```python
        >>> from transformers import AutoTokenizer, JukeboxModel, set_seed
        >>> import torch

        >>> metas = dict(artist="Zac Brown Band", genres="Country", lyrics="I met a traveller from an antique land")
        >>> tokenizer = AutoTokenizer.from_pretrained("openai/jukebox-1b-lyrics")
        >>> model = JukeboxModel.from_pretrained("openai/jukebox-1b-lyrics", min_duration=0).eval()

        >>> labels = tokenizer(**metas)["input_ids"]
        >>> set_seed(0)
        >>> zs = [torch.zeros(1, 0, dtype=torch.long) for _ in range(3)]
        >>> zs = model._sample(zs, labels, [0], sample_length=40 * model.priors[0].raw_to_tokens, save_results=False)
        >>> zs[0]
        tensor([[1853, 1369, 1150, 1869, 1379, 1789,  519,  710, 1306, 1100, 1229,  519,
              353, 1306, 1379, 1053,  519,  653, 1631, 1467, 1229, 1229,   10, 1647,
             1254, 1229, 1306, 1528, 1789,  216, 1631, 1434,  653,  475, 1150, 1528,
             1804,  541, 1804, 1434]])
        ```
        r   Nr   g®Gáz®ï?)rŽ  r”  r�  )rj  r§  zjukebox/level_)rŽ   r�   r‚   z/lyric_alignments.pt)r7  rA   rn   r�  ró  r@   r9   rF   r_   rP   rQ  rz  r=   r>   r!   r-  r\  r^   Úosr”   ÚexistsÚmakedirsr•   rg   ró  r…   r‹   )r·   rk   rl   Úsample_levelsrŽ   r”  Úsampling_temperatureÚlower_batch_sizerD  rŽ  Úcompute_alignmentsr�  rG   Úsave_resultsrc   Ú	top_priorrF   ro   rB  Útotal_token_to_samplerQ   rj  rh  Úlogdirr‚   s                            r3   Ú_samplezJukeboxModel._sample€	  s™  € ðj —K‘K ‘Nˆ	ØÐ$Ø(‰Lô Ð,¨t¯{©{×/HÑ/HÑHÓIÈY×MdÑMdÑdØ×'Ñ'ñ(ˆLð Ð Ü!¤# d§k¡kÓ"2Ó3ˆMð )ˆÔØ"ó %	\ˆEà %¬¨T¯[©[Ó)9¸AÑ)=Ò =™ÐCWØ(Ø!.ñˆOð %1°D·K±KÀÑ4F×4TÑ4TÑ$TÐ!Ü˜TŸ[™[×5Ñ5°eÑ<¸t¿{¹{È5Ñ?Q×?WÑ?WÑWÓXˆJØ16¸-Ò1GÑ-È^ˆNØ×,Ñ,ØØ�u‘ØØØØ%ØØó	ˆLò Ø—
‘
—‘˜l¨5Ñ1×8Ñ8Ô9ä—]‘]“_ñ Ü"% d§k¡kÓ"2°UÑ":¸QÑ">�KØ $§
¡
× 1Ñ 1Ø$ [ u¨q¡yÐ1¸{ÐVbÐchÑVi×VoÑVoÐpqÑVrð !2ó !�I÷ð
 *¨%¨Ð1�Ü—w‘w—~‘~ fÔ-Ü—K‘K Ô'Ü ¨°UÀ	ÇÁÓ@QÕRÛ%¨$¯+©+°a©.Ò*DÈÏÉÐUVÉ×IpÑIpÐstÔItÜŸ™›ñ iÜ%2°<ÀÈÁÈDÏKÉKÐXYÉNÐ\`×\gÑ\gÓ%h˜
÷iä—J‘J ¨jÐ9¸f¸XÐEYÐ;ZÖ[ðK%	\ðN Ð÷ð ú÷ið iús   ÅAJ"É)J.Ê"J+	Ê.J7	a  
        Generates music tokens based on the provided `labels. Will start at the desired prior level and automatically
        upsample the sequence. If you want to create the audio, you should call `model.decode(tokens)`, which will use
        the VQ-VAE decoder to convert the music tokens to raw audio.

        Args:
            labels (`List[torch.LongTensor]`) :
                List of length `n_sample`, and shape `(self.levels, 4 + self.config.max_nb_genre +
                lyric_sequence_length)` metadata such as `artist_id`, `genre_id` and the full list of lyric tokens
                which are used to condition the generation.
            n_samples (`int`, *optional*, default to 1) :
                Number of samples to be generated in parallel.
        c           
      ób  — |j                  dt        t        t        | j                  «      «      «      «      }t        t        | j                  «      «      D �cg c]6  }t        j                  |dt
        j                  |d   j                  ¬«      ‘Œ8 }} | j                  |||fi |¤Ž}|S c c}w )aR  
        Example:

        ```python
        >>> from transformers import AutoTokenizer, JukeboxModel, set_seed

        >>> model = JukeboxModel.from_pretrained("openai/jukebox-1b-lyrics", min_duration=0).eval()
        >>> tokenizer = AutoTokenizer.from_pretrained("openai/jukebox-1b-lyrics")

        >>> lyrics = "Hey, are you awake? Can you talk to me?"
        >>> artist = "Zac Brown Band"
        >>> genre = "Country"
        >>> metas = tokenizer(artist=artist, genres=genre, lyrics=lyrics)
        >>> set_seed(0)
        >>> music_tokens = model.ancestral_sample(metas.input_ids, sample_length=400)

        >>> with torch.no_grad():
        ...     model.decode(music_tokens)[:, :10].squeeze(-1)
        tensor([[-0.0219, -0.0679, -0.1050, -0.1203, -0.1271, -0.0936, -0.0396, -0.0405,
            -0.0818, -0.0697]])
        ```
        rV  r   rV   )
Úpopr?   r@   r9   r7  r!   r;   r<   r>   r^  )r·   rl   r·  rB  rV  rW  rk   s          r3   Úancestral_samplezJukeboxModel.ancestral_sample
  s™   € ðN (×+Ñ+¨O¼TÄ%ÌÈDÏKÉKÓHXÓBYÓ=ZÓ[ˆäZ_Ô`cÐdh×doÑdoÓ`pÓZqö
ØUVŒE�K‰K˜	 1¬E¯J©J¸vÀa¹y×?OÑ?OÖPð
ˆð 
ð $�t—|‘| L°&¸-Ñ[È?Ñ[ˆØÐùò	
s   Á;B,az  Generates a continuation of the previously generated tokens.

        Args:
            music_tokens (`List[torch.LongTensor]` of length `self.levels` ) :
                A sequence of music tokens which will be used as context to continue the sampling process. Should have
                `self.levels` tensors, each corresponding to the generation at a certain level.
        c           
      óž   — |j                  dt        t        t        | j                  «      «      «      «      } | j
                  |||fi |¤Ž}|S )NrV  ©r`  r?   r@   r9   r7  r^  ©r·   rk   rl   rB  rV  s        r3   Úcontinue_samplezJukeboxModel.continue_sample9
  sH   € ð (×+Ñ+¨O¼TÄ%ÌÈDÏKÉKÓHXÓBYÓ=ZÓ[ˆØ#�t—|‘| L°&¸-Ñ[È?Ñ[ˆØÐr5   a„  Upsamples a sequence of music tokens using the prior at level `level`.

        Args:
            music_tokens (`List[torch.LongTensor]` of length `self.levels` ) :
                A sequence of music tokens which will be used as context to continue the sampling process. Should have
                `self.levels` tensors, each corresponding to the generation at a certain level.
        c           
      ó¤   — |j                  dt        t        t        | j                  «      dz
  «      «      «      } | j
                  |||fi |¤Ž}|S )NrV  r   rc  rd  s        r3   ÚupsamplezJukeboxModel.upsampleH
  sN   € ð (×+Ñ+¨O¼TÄ%ÌÈDÏKÉKÓHXÐ[\ÑH\ÓB]Ó=^Ó_ˆØ#�t—|‘| L°&¸-Ñ[È?Ñ[ˆØÐr5   a'  Generate a raw audio conditioned on the provided `raw_audio` which is used as conditioning at each of the
        generation levels. The audio is encoded to music tokens using the 3 levels of the VQ-VAE. These tokens are
        used: as conditioning for each level, which means that no ancestral sampling is required.

        Args:
            raw_audio (`List[torch.Tensor]` of length `n_samples` ) :
                A list of raw audio that will be used as conditioning information for each samples that will be
                generated.
        c           
      óÔ  — |j                  dt        t        t        | j                  «      «      «      «      }| j
                  j                  |j                  «      j                  «        t        j                  «       5  | j
                  j                  |dt        | j                  «      |j                  d   ¬«      }d d d «        | j                  ||fi |¤Ž}|S # 1 sw Y   Œ xY w)NrV  r   r  )r`  r?   r@   r9   r7  rz  r=   r>   rg   r!   r-  rY  r^   r^  )r·   rh  rl   rB  rV  rk   s         r3   r¥  zJukeboxModel.primed_sampleW
  s¿   € ð (×+Ñ+¨O¼TÄ%ÌÈDÏKÉKÓHXÓBYÓ=ZÓ[ˆØ�
‰
�‰�i×&Ñ&Ó'×-Ñ-Ô/Ü�]‰]‹_ñ 	ØŸ:™:×,Ñ,Ø q´C¸¿¹Ó4DÐPY×P_ÑP_Ð`aÑPbð -ó ˆL÷	ð $�t—|‘| L°&¸-Ñ[È?Ñ[ˆØÐ÷	ð 	ús   Á?A CÃC'r»  )Né    g\�Âõ(\ï?é   rj  é   FNr   TN)r   )rÀ   rÁ   rÂ   Ú_no_split_modulesr°   r5  r\  rY  r?  rF  rA  rQ  r!   r-  r   r'  r^  r   ra  Ú JUKEBOX_SAMPLING_INPUT_DOCSTRINGre  rg  r¥  rÃ   rÄ   s   @r3   r2  r2  ñ  sb  ø„ ð (Ð(Ðô
òNóRóQò	2òvò"8òvð  €U‡]�]ƒ_ð ØØ!ØØØ!#Ø ØØØØñHð  
ˆe×ÑÑ	ò!Hó ðHñT ð	óñÈ$Èu×O_ÑO_ÑJ`ò óðñ> ð	ð 	)ó	ðÈ$Èu×O_ÑO_ÑJ`ò ó	ðñ
 ð	ð 	)ó	ðÀ4È×HXÑHXÑCYò ó	ðñ
 ð	ð 	)óðÀTÈ%×JZÑJZÑE[ò óôr5   r2  )Cr­  rÞ   rS  Útypingr   r   r   rh   ri   r!   Útorch.nn.functionalr   r    r%   Útorch.nnr   ÚFusedLayerNormrz  r	   Úmodeling_utilsr
   Úutilsr   r   Úutils.loggingr   Úconfiguration_jukeboxr   r   r   r   Ú
get_loggerrÀ   rô  rg   r4   rL   rT   r…   r•   rª   ÚModuler¬   rÆ   rÙ   rê   rõ   r  r	  r  re  ÚJUKEBOX_START_DOCSTRINGry  rÁ  rÎ  rÙ  rA  rM  r]  rb  r§  r¯  rÂ  rÚ  r)  rm  r2  r_  r5   r3   ú<module>ry     sC  ðñ ã Û 	ß (Ñ (ã Û ß Ð Ý Ý 0å "Ý .ß 3Ý "ß lÓ lð 
ˆ×	Ñ	˜HÓ	%€ð !"¨¹EÀ%»L¸=ó #òL,òDò6òr=ò;ô6�B—I‘Iô ô*:˜BŸI™Iô :ô*�b—i‘iô ô*˜bŸi™iô ô(!�R—Y‘Yô !ô4˜RŸY™Yô ô4�R—Y‘Yô ô6[j˜RŸY™Yô [jô|'F˜Ÿ	™	ô 'FðTÐ ñ" ðð óôv(�?ó v(óðv(ôr�—‘ô ô(
9�~ô 
9ôs�r—y‘yô sôl�2—9‘9ô ô49(˜Ÿ	™	ô 9(ôx §¡ô ôG  r§y¡yô G ôT
* 2§9¡9ô *ôZ-˜BŸI™Iô -ô`5"˜bŸi™iô 5"ôpk1�?ô k1ô\,˜_ô ,ð$$Ð  ñ ðð
 óôrÐ)ó róñrr5   