Ë
    S^(h¬# ã                   óV  — d dl Z d dlZd dlZd dlZd dlmZmZ d dlmZ d dl	m
Z
 d dlmZ d dlmZ d dlmZmZmZmZmZmZmZmZ d dlZdd	lmZ dd
lmZ ddlmZm Z m!Z!m"Z" ddl#m$Z$ ddl%m%Z%  e"jL                  e'«      Z( e
jR                  «       jT                  Z+ ee,«      jZ                  jZ                  a.t\        jZ                  jZ                  a/e G d„ d«      «       Z0dddddddddddddœZ1de2de3fd „Z4de2de5fd!„Z6d"e2dee2   fd#„Z7dqd"e2d$e5de2fd%„Z8	 	 	 drd&e2d"e2d'eee2ef      d(eee2ef      d)e3de2fd*„Z9	 	 	 drd+ee2ejt                  f   d"e2d'eee2ef      d(eee2ef      d)e3f
d,„Z;d&e2d-e0d.e0dee2e2f   fd/„Z<d0„ Z=d1ee2ejt                  f   de2fd2„Z>d3d4d5d6d7d8d9œZ? ej€                  d:ej‚                  ¬;«      ZBd<„ ZC	 	 	 dsd1ee2ejt                  f   d-e0d.e0d=ee2   d>e3d?ee2   fd@„ZD	 dtdAeee2ejt                  f      dBeee2      deee2ejt                  f      fdC„ZEdtdDe2dBeee2      dee2eeee   f   f   fdE„ZF ej€                  dFej‚                  ¬;«      ZG	 dtdDe2dGeee2eeee   f   f      de2fdH„ZHdI„ ZI ej€                  dJ«      ZJdtdDe2dBeee2      dee2ee2   f   fdK„ZKdtdBeee2      fdL„ZL	 dudMee2ejt                  f   dBeee2      dNe3fdO„ZM	 	 dud-e0d.e0dBeee2      dPe3fdQ„ZNd-e0d.e0fdR„ZOg dS¢dTgdUgdVgdWgdXgdYgdZœZPd-e0d.e0d[ee2ee2   f   fd\„ZQd]ZR	 	 dvd^ee2ejt                  f   d-e0d.e0d=eee2ejt                  f      dBeee2      f
d_„ZSd`„ ZT	 	 	 	 dwdDe2d.e0d>e3dBeee2      daee2   dbe3fdc„ZUddefde„ZV G df„ dge$«      ZW	 	 	 	 dxdhe2diee2   djee   dkee   dlee2   defdm„ZXdne2de3fdo„ZYdp„ ZZy)yé    N)ÚArgumentParserÚ	Namespace)Ú	dataclass)Údate)Úchain)ÚPath)ÚAnyÚCallableÚDictÚListÚOptionalÚPatternÚTupleÚUnioné   )Úauto)Úmodel_type_to_module_name)Úis_flax_availableÚis_tf_availableÚis_torch_availableÚloggingé   )ÚBaseTransformersCLICommand)Úadd_fast_image_processorc                   óô   — e Zd ZU dZeed<   eed<   dZee   ed<   dZee   ed<   dZ	ee   ed<   dZ
ee   ed<   dZee   ed	<   dZee   ed
<   dZee   ed<   dZee   ed<   dZee   ed<   dZee   ed<   d„ Zy)ÚModelPatternsaÄ  
    Holds the basic information about a new model for the add-new-model-like command.

    Args:
        model_name (`str`): The model name.
        checkpoint (`str`): The checkpoint to use for doc examples.
        model_type (`str`, *optional*):
            The model type, the identifier used internally in the library like `bert` or `xlm-roberta`. Will default to
            `model_name` lowercased with spaces replaced with minuses (-).
        model_lower_cased (`str`, *optional*):
            The lowercased version of the model name, to use for the module name or function names. Will default to
            `model_name` lowercased with spaces and minuses replaced with underscores.
        model_camel_cased (`str`, *optional*):
            The camel-cased version of the model name, to use for the class names. Will default to `model_name`
            camel-cased (with spaces and minuses both considered as word separators.
        model_upper_cased (`str`, *optional*):
            The uppercased version of the model name, to use for the constant names. Will default to `model_name`
            uppercased with spaces and minuses replaced with underscores.
        config_class (`str`, *optional*):
            The tokenizer class associated with this model. Will default to `"{model_camel_cased}Config"`.
        tokenizer_class (`str`, *optional*):
            The tokenizer class associated with this model (leave to `None` for models that don't use a tokenizer).
        image_processor_class (`str`, *optional*):
            The image processor class associated with this model (leave to `None` for models that don't use an image
            processor).
        image_processor_fast_class (`str`, *optional*):
            The fast image processor class associated with this model (leave to `None` for models that don't use a fast
            image processor).
        feature_extractor_class (`str`, *optional*):
            The feature extractor class associated with this model (leave to `None` for models that don't use a feature
            extractor).
        processor_class (`str`, *optional*):
            The processor class associated with this model (leave to `None` for models that don't use a processor).
    Ú
model_nameÚ
checkpointNÚ
model_typeÚmodel_lower_casedÚmodel_camel_casedÚmodel_upper_casedÚconfig_classÚtokenizer_classÚimage_processor_classÚimage_processor_fast_classÚfeature_extractor_classÚprocessor_classc           
      ó  — | j                   €/| j                  j                  «       j                  dd«      | _         | j                  €?| j                  j                  «       j                  dd«      j                  dd«      | _        | j
                  €…| j                  j                  d«      }t        t        |D �cg c]  }|j                  d«      ‘Œ c}Ž «      }|D �cg c]  }|d   j                  «       |dd  z   ‘Œ }}dj                  |«      | _        | j                  €?| j                  j                  «       j                  dd«      j                  dd«      | _        | j                  €| j
                  › d�| _        y y c c}w c c}w )Nú ú-Ú_r   r   Ú ÚConfig)r   r   ÚlowerÚreplacer    r!   ÚsplitÚlistr   ÚupperÚjoinr"   r#   )ÚselfÚwordsÚws      úf/var/www/skyplay_api_hub/venv/lib/python3.12/site-packages/transformers/commands/add_new_model_like.pyÚ__post_init__zModelPatterns.__post_init__]   sP  € Ø�?‰?Ð"Ø"Ÿo™o×3Ñ3Ó5×=Ñ=¸cÀ3ÓGˆDŒOØ×!Ñ!Ð)Ø%)§_¡_×%:Ñ%:Ó%<×%DÑ%DÀSÈ#Ó%N×%VÑ%VÐWZÐ\_Ó%`ˆDÔ"Ø×!Ñ!Ð)à—O‘O×)Ñ)¨#Ó.ˆEÜœ°uÖ =°! §¡¨¥Ò =Ð>Ó?ˆEà38Ö9¨a�Q�q‘T—Z‘Z“\ A a b EÓ)Ð9ˆEÐ9Ø%'§W¡W¨U£^ˆDÔ"Ø×!Ñ!Ð)Ø%)§_¡_×%:Ñ%:Ó%<×%DÑ%DÀSÈ#Ó%N×%VÑ%VÐWZÐ\_Ó%`ˆDÔ"Ø×ÑÐ$Ø#'×#9Ñ#9Ð":¸&Ð AˆDÕð %ùò !>ùâ9s   Â<FÃ  F
)Ú__name__Ú
__module__Ú__qualname__Ú__doc__ÚstrÚ__annotations__r   r   r    r!   r"   r#   r$   r%   r&   r'   r(   r9   © ó    r8   r   r   +   s¬   … ñ!ðF ƒOØƒOØ $€J�˜‘Ó$Ø'+Ð�x ‘}Ó+Ø'+Ð�x ‘}Ó+Ø'+Ð�x ‘}Ó+Ø"&€L�(˜3‘-Ó&Ø%)€O�X˜c‘]Ó)Ø+/Ð˜8 C™=Ó/Ø04Ð ¨¡Ó4Ø-1Ð˜X c™]Ó1Ø%)€O�X˜c‘]Ó)óBrA   r   z[CONFIG_CLASS]z[TOKENIZER_CLASS]z[IMAGE_PROCESSOR_CLASS]z[IMAGE_PROCESSOR_FAST_CLASS]z[FEATURE_EXTRACTOR_CLASS]z[PROCESSOR_CLASS]z[CHECKPOINT]z[MODEL_TYPE]z[MODEL_UPPER_CASED]z[MODEL_CAMELCASED]z[MODEL_LOWER_CASED]z[MODEL_NAME])r#   r$   r%   r&   r'   r(   r   r   r"   r!   r    r   ÚlineÚreturnc                 óB   — t        | «      dk(  xs | j                  «       S )z4
    Determines whether a line is empty or not.
    r   )ÚlenÚisspace)rB   s    r8   Úis_empty_linerG      s   € ô ˆt‹9˜‰>Ò+˜TŸ\™\›^Ð+rA   c                 ól   — t        j                  d| «      }|€yt        |j                  «       d   «      S )z@
    Returns the number of spaces that start a line indent.
    z^(\s*)(?:\S|$)r   )ÚreÚsearchrE   Úgroups)rB   rJ   s     r8   Úfind_indentrL   †   s3   € ô �Y‰YÐ(¨$Ó/€FØ€~ØÜˆv�}‰}‹˜qÑ!Ó"Ð"rA   Úcontentc                 ó  — g }g }| j                  d«      }g d¢}|D ]½  }t        |«      dkD  }|r#t        |«      dk(  r|d   j                  d«       }t        |«      slt	        |«      dk(  r^|r\||v r4|j                  |«       |j                  dj                  |«      «       g }Œ‰|j                  dj                  |«      «       |g}Œ­|j                  |«       Œ¿ t        |«      dkD  r |j                  dj                  |«      «       |S )zÑ
    Parse the content of a module in the list of objects it defines.

    Args:
        content (`str`): The content to parse

    Returns:
        `List[str]`: The list of objects defined in the module.
    ú
©ú)ú]ú}z"""r   r   z# Copied from)r1   rE   Ú
startswithrG   rL   Úappendr4   )rM   ÚobjectsÚcurrent_objectÚlinesÚend_markersrB   Úis_valid_objects          r8   Úparse_module_contentr[   �   sÿ   € ð €GØ€NØ�M‰M˜$Ó€Eâ(€Kàò (ˆä˜nÓ-°Ñ1ˆÙœs >Ó2°aÒ7Ø"0°Ñ"3×">Ñ">¸Ó"OÐOˆOÜ˜TÔ"¤{°4Ó'8¸AÒ'=Á/à�{Ñ"Ø×%Ñ% dÔ+Ø—‘˜tŸy™y¨Ó8Ô9Ø!#‘à—‘˜tŸy™y¨Ó8Ô9Ø"& ‘à×!Ñ! $Õ'ð(ô$ ˆ>Ó˜QÒØ�‰�t—y‘y Ó0Ô1à€NrA   Úindent_levelc           	      óJ  — g }| j                  d«      }g d¢}t        |«      D ]ß  \  }}|dk(  r9|dkD  r4t        |«      s)t        |«      |k7  rt	        d|› dt        |«      › d�«      ‚t        |«      |k  rt        |«      s nƒt        |«      dkD  }t        |«      sW|j                  d«      sFt        |«      |k(  r8|r6|j                  «       |v r|j                  |«       dj                  |«      c S |j                  |«       Œá t        |«      dkD  rdj                  |«      S y)	aE  Return the first block in `content` with the indent level `indent_level`.

    The first line in `content` should be indented at `indent_level` level, otherwise an error will be thrown.

    This method will immediately stop the search when a (non-empty) line with indent level less than `indent_level` is
    encountered.

    Args:
        content (`str`): The content to parse
        indent_level (`int`, *optional*, default to 0): The indent level of the blocks to search for

    Returns:
        `str`: The first block in `content` with the indent level `indent_level`.
    rO   rP   r   zNWhen `indent_level > 0`, the first line in `content` should have indent level z. Got z	 instead.ú:N)
r1   Ú	enumeraterG   rL   Ú
ValueErrorrE   ÚendswithÚlstriprU   r4   )rM   r\   rW   rX   rY   ÚidxrB   rZ   s           r8   Úextract_blockrd   ¸   s.  € ð €NØ�M‰M˜$Ó€Eâ(€Kä˜uÓ%ò (‰	ˆˆTØ�!Š8˜ qÒ(´¸tÔ1DÌÐUYÓIZÐ^jÒIjÜØ`ÐamÐ`nÐntÜ˜tÓ$Ð% Yð0óð ô
 �tÓ˜|Ò+´MÀ$Ô4GÙô ˜nÓ-°Ñ1ˆä˜dÔ#Ø—M‘M #Ô&Ü˜DÓ! \Ò1Ùð �{‰{‹} Ñ+Ø×%Ñ% dÔ+Ø—9‘9˜^Ó,Ò,à×!Ñ! $Õ'ð/(ô4 ˆ>Ó˜QÒØ�y‰y˜Ó(Ð(ð rA   ÚtextÚ	add_afterÚ
add_beforeÚexact_matchc                 óP  ‡‡— |€|€t        d«      ‚|�|�t        d«      ‚|€|n|Šˆˆfd„}g }| j                  d«      D ]T  } ||«      r9|�|j                  |«       |j                  |«       |€Œ2|j                  |«       ŒD|j                  |«       ŒV dj                  |«      S )aÛ  
    A utility to add some content inside a given text.

    Args:
       text (`str`): The text in which we want to insert some content.
       content (`str`): The content to add.
       add_after (`str` or `Pattern`):
           The pattern to test on a line of `text`, the new content is added after the first instance matching it.
       add_before (`str` or `Pattern`):
           The pattern to test on a line of `text`, the new content is added before the first instance matching it.
       exact_match (`bool`, *optional*, defaults to `False`):
           A line is considered a match with `add_after` or `add_before` if it matches exactly when `exact_match=True`,
           otherwise, if `add_after`/`add_before` is present in the line.

    <Tip warning={true}>

    The arguments `add_after` and `add_before` are mutually exclusive, and one exactly needs to be provided.

    </Tip>

    Returns:
        `str`: The text with the new content added if a match was found.
    z3You need to pass either `add_after` or `add_before`z/You can't pass both `add_after` or `add_before`c                 ó`   •— t        ‰t        «      r‰j                  | «      d uS ‰r‰| k(  S ‰| v S ©N)Ú
isinstancer   rJ   )rB   rh   Úpatterns    €€r8   Úthis_is_the_linez-add_content_to_text.<locals>.this_is_the_line  s8   ø€ Ü�gœwÔ'Ø—>‘> $Ó'¨tÐ3Ð3ÙØ˜d‘?Ð"à˜d�?Ð"rA   rO   )r`   r1   rU   r4   )	re   rM   rf   rg   rh   rn   Ú	new_linesrB   rm   s	       `   @r8   Úadd_content_to_textrp   ê   sÀ   ù€ ð< Ð˜ZÐ/ÜÐNÓOÐOØÐ Ð!7ÜÐJÓKÐKØ%Ð-‰i°:€Gõ#ð €IØ—
‘
˜4Ó ò #ˆÙ˜DÔ!ØÐ%Ø× Ñ  Ô)Ø×Ñ˜TÔ"ØÑ$Ø× Ñ  Õ)à×Ñ˜TÕ"ð#ð �9‰9�YÓÐrA   Ú	file_namec                 óò   — t        | dd¬«      5 }|j                  «       }ddd«       t        ||||¬«      }t        | dd¬«      5 }|j                  |«       ddd«       y# 1 sw Y   ŒBxY w# 1 sw Y   yxY w)a¦  
    A utility to add some content inside a given file.

    Args:
       file_name (`str` or `os.PathLike`): The name of the file in which we want to insert some content.
       content (`str`): The content to add.
       add_after (`str` or `Pattern`):
           The pattern to test on a line of `text`, the new content is added after the first instance matching it.
       add_before (`str` or `Pattern`):
           The pattern to test on a line of `text`, the new content is added before the first instance matching it.
       exact_match (`bool`, *optional*, defaults to `False`):
           A line is considered a match with `add_after` or `add_before` if it matches exactly when `exact_match=True`,
           otherwise, if `add_after`/`add_before` is present in the line.

    <Tip warning={true}>

    The arguments `add_after` and `add_before` are mutually exclusive, and one exactly needs to be provided.

    </Tip>
    Úrúutf-8©ÚencodingN)rf   rg   rh   r7   )ÚopenÚreadrp   Úwrite)rq   rM   rf   rg   rh   ÚfÚold_contentÚnew_contents           r8   Úadd_content_to_filer}   $  s€   € ô6 
ˆi˜ wÔ	/ð °1Ø—f‘f“hˆ÷ô &Ø�W¨	¸jÐVaô€Kô 
ˆi˜ wÔ	/ð °1Ø	�‰�Ô÷ð ÷ð ú÷ð ús   �A!ÁA-Á!A*Á-A6Úold_model_patternsÚnew_model_patternsc                 ó†  — dg}dD ]-  }t        ||«      €Œt        ||«      €Œ|j                  |«       Œ/ |j                  |j                  |j                  fvr|j                  d«       |j                  |j                  k7  r|j                  d«       n%t        j                  d|j                  › d�d| «      } |j                  |j                  k(  r@|j                  }t        j                  |› d�| «      �,t        j                  |› d	�d
| «      } n|j                  d«       |j                  g d¢«       |D ]%  }| j                  t        ||«      t        |   «      } Œ' g }t        j                  «       D ]M  \  }}|| v sŒ|j                  t        ||«      t        ||«      f«       | j                  |t        ||«      «      } ŒO |D ��	cg c]  \  }}	|‘Œ	 }
}}	t        t        |
«      «      t        |
«      k7  r| dfS t!        |«      }|D ��	cg c]  \  }}	|› d|	› �‘Œ }}}	| dj#                  |«      fS c c}	}w c c}	}w )az  
    Replace all patterns present in a given text.

    Args:
        text (`str`): The text to treat.
        old_model_patterns (`ModelPatterns`): The patterns for the old model.
        new_model_patterns (`ModelPatterns`): The patterns for the new model.

    Returns:
        `Tuple(str, str)`: A tuple of with the treated text and the replacement actually done in it.
    r#   )r$   r%   r&   r'   r(   r   r   z(\s*)model_type = "ú"z\1model_type = "[MODEL_TYPE]"z_[A-Z_]*[^A-Z_]z([A-Z_]*)([^a-zA-Z_])z[MODEL_UPPER_CASED]\1\2r"   )r!   r    r   r-   z->ú,)ÚgetattrrU   r   r   r    rI   Úsubr"   r!   rJ   Úextendr0   ÚATTRIBUTE_TO_PLACEHOLDERÚitemsrE   ÚsetÚsimplify_replacementsr4   )re   r~   r   Úattributes_to_checkÚattrÚold_model_valueÚreplacementsÚplaceholderÚoldÚnewÚold_replacement_valuess              r8   Úreplace_model_patternsr’   J  s_  € ð  *Ð*Ððò -ˆô Ð% tÓ,Ñ8¼WÐEWÐY]Ó=^Ñ=jØ×&Ñ& tÕ,ð-ð ×$Ñ$Ð-?×-JÑ-JÐL^×LpÑLpÐ,qÑqØ×"Ñ" <Ô0Ø×$Ñ$Ð(:×(LÑ(LÒLØ×"Ñ" <Õ0ä�v‰vØ"Ð#5×#@Ñ#@Ð"AÀÐCØ,Øó
ˆð ×+Ñ+Ð/A×/SÑ/SÒSØ,×>Ñ>ˆÜ�9‰9˜Ð)¨Ð9¸4Ó@ÐLÜ—6‘6˜oÐ.Ð.CÐDÐF`ÐbfÓg‰Dà×"Ñ"Ð#6Ô7à×ÑÒWÔXð $ò _ˆØ�|‰|œGÐ$6¸Ó=Ô?WÐX\Ñ?]Ó^‰ð_ð €LÜ5×;Ñ;Ó=ò PÑˆˆkØ˜$ÒØ×Ñ¤Ð);¸TÓ!BÄGÐL^Ð`dÓDeÐ fÔgØ—<‘< ¬WÐ5GÈÓ-NÓO‰DðPð 3?×?¡h c¨3šcÐ?ÐÑ?Ü
Œ3Ð%Ó&Ó'¬3Ð/EÓ+FÒFØ�Rˆxˆä(¨Ó6€LØ4@×A©¨¨S�s�e˜2˜c˜U’OÐA€LÑAØ�—‘˜,Ó'Ð'Ð'ùó @ùó
 Bs   ÇH7ÈH=c                 óP  — t        | «      dk  r| S | j                  d„ ¬«       d}|t        | «      k  rr| |   \  }}|dz   }|t        | «      k  rC| |   \  }}|j                  ||«      |k(  r| j                  |«       n|dz  }|t        | «      k  rŒC|dz  }|t        | «      k  rŒr| S )a¼  
    Simplify a list of replacement patterns to make sure there are no needless ones.

    For instance in the sequence "Bert->BertNew, BertConfig->BertNewConfig, bert->bert_new", the replacement
    "BertConfig->BertNewConfig" is implied by "Bert->BertNew" so not needed.

    Args:
        replacements (`List[Tuple[str, str]]`): List of patterns (old, new)

    Returns:
        `List[Tuple[str, str]]`: The list of patterns simplified.
    r   c                 ó   — t        | d   «      S )Nr   )rE   ©Úxs    r8   ú<lambda>z'simplify_replacements.<locals>.<lambda>¤  s   € ¤C¨¨!©£I€ rA   ©Úkeyr   )rE   Úsortr0   Úpop)r�   rc   r�   r�   ÚjÚold_2Únew_2s          r8   r‰   r‰   ’  sÉ   € ô ˆ<Ó˜AÒàÐð ×ÑÑ-ÐÔ.à
€CØ
”�LÓ!Ò
!Ø Ñ$‰ˆˆSà�!‰GˆØ”#�lÓ#Ò#Ø'¨™?‰LˆE�5à�}‰}˜S #Ó&¨%Ò/Ø× Ñ  Õ#à�Q‘�ð ”#�lÓ#Ó#ð 	ˆq‰ˆð ”�LÓ!Ó
!ð ÐrA   Úmodule_filec                 ó  — t        | «      j                  «       }|j                  d«      j                  }t	        |«      dz
  }|dk\  r||   dk7  r|dz  }|dk\  r	||   dk7  rŒ|dk  rt        | › d�«      ‚dj                  ||d «      S )zA
    Returns the module name corresponding to a module file.
    r-   r   r   Útransformersz is not a transformers module.ú.N)r   ÚabsoluteÚwith_suffixÚpartsrE   r`   r4   )rŸ   Úfull_module_pathÚmodule_partsrc   s       r8   Úget_module_from_filer¨   ·  s    € ô ˜KÓ(×1Ñ1Ó3ÐØ#×/Ñ/°Ó3×9Ñ9€Lô ˆlÓ
˜aÑ
€CØ
�Š(�| CÑ(¨NÒ:Øˆq‰ˆð �Š(�| CÑ(¨NÓ:à
ˆQ‚wÜ˜K˜=Ð(FÐGÓHÐHà�8‰8�L  Ð&Ó'Ð'rA   r   r#   r$   r%   r'   r(   )z_CHECKPOINT_FOR_DOC =z_CONFIG_FOR_DOC =z_TOKENIZER_FOR_DOC =z_IMAGE_PROCESSOR_FOR_DOC =z_FEAT_EXTRACTOR_FOR_DOC =z_PROCESSOR_FOR_DOC =z(^(?:class|def)\s+([^\s:\(]+)\s*(?:\(|\:)©Úflagsc                 ó  — | j                  t        j                  «      }d}t        |«      D ]R  \  }}|j	                  «       j                  |› d�«      r|} n)|j	                  «       j                  d|› d�«      sŒP|} n |€| S ||   }t        |«      }t        dj                  ||d «      |«      }t        |j                  d«      «      }t        |«      D ]
  }d|||z   <   Œ t        |dz
  dd«      D ]Z  }||   }|j	                  «       j                  d«      s|j	                  «       j                  d	«      rt        |«      |k(  rd||<   ŒZ n t        j                  j                  |D �	cg c]  }	|	€Œ|	‘Œ	 c}	«      }
|
S c c}	w )
zRemove `target_attr` in `obj`.Nz = zdef ú(rO   r   éÿÿÿÿú#ú@)r1   ÚosÚlinesepr_   rb   rT   rL   rd   r4   rE   Úrange)ÚobjÚtarget_attrrX   Ú
target_idxrc   rB   r\   ÚparsedÚ	num_linesr–   Únew_objs              r8   Úremove_attributesr¹   Õ  sƒ  € à�I‰I”b—j‘jÓ!€Eà€JÜ˜uÓ%ò ‰	ˆˆTà�;‰;‹=×#Ñ# { m°3Ð$7Ô8ØˆJÙà�[‰[‹]×%Ñ%¨¨[¨M¸Ð&;Õ<ØˆJÙðð ÐØˆ
à�Ñ€DÜ˜tÓ$€Lä˜4Ÿ9™9 U¨:¨;Ð%7Ó8¸,ÓG€FÜ�F—L‘L Ó&Ó'€IÜ�YÓò 'ˆØ"&ˆˆj˜3ÑÒð'ô �Z !‘^ R¨Ó,ò ˆØ�S‰zˆØ�K‰K‹M×$Ñ$ SÔ)¨T¯[©[«]×-EÑ-EÀcÔ-JÔP[Ð\`ÓPaÐeqÒPqØˆE�#ŠJáðô �j‰j�o‰o¨%ÖA Q°1±=šqÒAÓB€Gà€Nùò Bs   Å0FÅ8FÚ	dest_fileÚadd_copied_fromÚattrs_to_removec           
      óX  — |€/t        | «      j                  |j                  |j                  «      }t        | dd¬«      5 }|j	                  «       }ddd«       t        j                  ddt        › d�«      }t        |«      }g }	|D �]8  }
d}t        j                  «       D ]D  \  }}||
v sŒ|
j                  t        ||«      t        ||«      «      }
|	j                  |
«       d	} n |rŒd|
}t        |
||«      \  }
}t        j                  d
|
t
        j                  ¬«      du}|rq|sot         j                  |
«      �Zt#        |«      dkD  rLt%        | «      }t         j                  |«      j'                  «       d   }t)        |
d|› d|› d|› �t         ¬«      }
t        j                  dd|
«      }
|	j                  |
«       �Œ; dj+                  |	«      }|�|D ]  }t-        ||¬«      }Œ t        |dd¬«      5 }|j/                  |«       ddd«       y# 1 sw Y   �ŒÄxY w# 1 sw Y   yxY w)a\  
    Create a new module from an existing one and adapting all function and classes names from old patterns to new ones.

    Args:
        module_file (`str` or `os.PathLike`): Path to the module to duplicate.
        old_model_patterns (`ModelPatterns`): The patterns for the old model.
        new_model_patterns (`ModelPatterns`): The patterns for the new model.
        dest_file (`str` or `os.PathLike`, *optional*): Path to the new module.
        add_copied_from (`bool`, *optional*, defaults to `True`):
            Whether or not to add `# Copied from` statements in the duplicated module.
    Nrs   rt   ru   z# Copyright (\d+)\sz# Copyright r*   FTz^#\s+Copied fromr©   r   z# Copied from r¢   z with )rg   z
[ ]+# Copied from [^
]*
rO   )r´   r7   )r>   r0   r    rw   rx   rI   r„   ÚCURRENT_YEARr[   ÚSPECIAL_PATTERNSr‡   rƒ   rU   r’   rJ   Ú	MULTILINEÚ_re_class_funcrE   r¨   rK   rp   r4   r¹   ry   )rŸ   r~   r   rº   r»   r¼   rz   rM   rV   Únew_objectsr³   Úspecial_patternrm   r‹   Úold_objÚreplacementÚhas_copied_fromÚmodule_nameÚold_object_names                      r8   Úduplicate_modulerÉ   ý  sA  € ð& ÐÜ˜Ó$×,Ñ,Ø×0Ñ0Ð2D×2VÑ2Vó
ˆ	ô 
ˆk˜3¨Ô	1ð °QØ—&‘&“(ˆ÷ô �f‰fÐ+¨|¼L¸>ÈÐ-KÈWÓU€GÜ" 7Ó+€Gð €KØó  ˆØˆÜ-×3Ñ3Ó5ò 	‰MˆG�TØ˜#Š~Ø—k‘k¤'Ð*<¸dÓ"CÄWÐM_ÐaeÓEfÓg�Ø×"Ñ" 3Ô'Ø"&�Ùð	ñ Øð ˆÜ1°#Ð7IÐK]Ó^Ñˆˆ[ÜŸ)™)Ð$7¸ÄBÇLÁLÔQÐY]Ð]ˆÙ¡?´~×7LÑ7LÈSÓ7QÐ7]ÔbeÐfqÓbrÐuvÒbvô /¨{Ó;ˆKÜ,×3Ñ3°GÓ<×CÑCÓEÀaÑHˆOÜ%Ø�~ k ]°!°OÐ3DÀFÈ;È-ÐXÔesôˆCô �f‰fÐ3°T¸3Ó?ˆà×Ñ˜3Öð7 ð: �i‰i˜Ó$€GàÐ"Ø#ò 	CˆDÜ'¨¸TÔB‰Gð	Cô 
ˆi˜ wÔ	/ð °1Ø	�‰�Ô÷ð ÷Wñ ú÷Vð ús   Á HÇ8H ÈHÈ H)ÚfilesÚ
frameworksc                 ó  — |€
t        «       }i }g }| D ]U  }t        |«      j                  j                  d«      }d|vr|j	                  |«       Œ=d|v r||d<   ŒGd|v r||d<   ŒQ||d<   ŒW |D �cg c]  }||v sŒ||   ‘Œ c}|z   S c c}w )aW  
    Filter a list of files to only keep the ones corresponding to a list of frameworks.

    Args:
        files (`List[Union[str, os.PathLike]]`): The list of files to filter.
        frameworks (`List[str]`, *optional*): The list of allowed frameworks.

    Returns:
        `List[Union[str, os.PathLike]]`: The list of filtered files.
    r,   ÚmodelingÚtfÚflaxÚpt)Úget_default_frameworksr   Únamer1   rU   )rÊ   rË   Úframework_to_fileÚothersrz   r¥   s         r8   Úfilter_framework_filesrÕ   D  s´   € ð ÐÜ+Ó-ˆ
àÐØ€FØò 
(ˆÜ�Q“—‘×"Ñ" 3Ó'ˆØ˜UÑ"Ø�M‰M˜!ÔØØ�5‰=Ø&'Ð˜dÒ#Ø�u‰_Ø()Ð˜fÒ%à&'Ð˜dÒ#ð
(ð +5ÖO Q¸Ð=NÒ8NÐ˜aÓ ÒOÐRXÑXÐXùÒOs   Á/	BÁ9Br   c           	      ó²  — t        | «      }t        dz  |z  }t        |j                  d«      «      }t	        ||¬«      }t
        dz  dz  dz  dz  | › d�z  }d	|› d
�d|› d
�d|› d
�d|› d
�d|› d
�d|› d
�d|› d
�g}t	        ||¬«      }|D �cg c]  }t
        dz  dz  |z  |z  ‘Œ }}|D �cg c]  }|j                  «       sŒ|‘Œ }}||||dœS c c}w c c}w )a2  
    Retrieves all the files associated to a model.

    Args:
        model_type (`str`): A valid model type (like "bert" or "gpt2")
        frameworks (`List[str]`, *optional*):
            If passed, will only keep the model files corresponding to the passed frameworks.

    Returns:
        `Dict[str, Union[Path, List[Path]]]`: A dictionary with the following keys:
        - **doc_file** -- The documentation file for the model.
        - **model_files** -- All the files in the model module.
        - **test_files** -- The test files for the model.
    Úmodelsz*.py©rË   ÚdocsÚsourceÚenÚ	model_docú.mdÚtest_modeling_z.pyÚtest_modeling_tf_Útest_modeling_flax_Útest_tokenization_Útest_image_processing_Útest_feature_extraction_Útest_processor_Útests)Údoc_fileÚmodel_filesrÇ   Ú
test_files)r   ÚTRANSFORMERS_PATHr2   ÚglobrÕ   Ú	REPO_PATHÚexists)r   rË   rÇ   Úmodel_modulerç   ræ   rè   rz   s           r8   Úget_model_filesrî   e  s*  € ô ,¨JÓ7€Kä$ xÑ/°+Ñ=€LÜ�|×(Ñ(¨Ó0Ó1€KÜ(¨ÀÔL€Kä˜6Ñ! HÑ,¨tÑ3°kÑAÀzÀlÐRUÐDVÑV€Hð ˜˜ SÐ)Ø
˜K˜=¨Ð,Ø
˜k˜]¨#Ð.Ø
˜[˜M¨Ð-Ø
   ¨SÐ1Ø
" ; -¨sÐ3Ø
˜+˜ cÐ*ð€Jô (¨
¸zÔJ€JàLVÖWÀq”)˜gÑ%¨Ñ0°;Ñ>ÀÓBÐW€JÐWà'Ö6˜¨1¯8©8­:’!Ð6€JÐ6à °È[ÐhrÑsÐsùò	 Xùâ6s   ÂCÂ,CÃCz$^_CHECKPOINT_FOR_DOC\s+=\s+(\S*)\s*$rç   c                 ó’  — |€t        | «      }|d   }|D ]¤  }dt        |«      vrŒt        |dd¬«      5 }|j                  «       }t        j                  |«      �Wt        j                  |«      j                  «       d   }|j                  dd	«      }|j                  d
d	«      }|cddd«       c S 	 ddd«       Œ¦ y	# 1 sw Y   Œ±xY w)a�  
    Finds the model checkpoint used in the docstrings for a given model.

    Args:
        model_type (`str`): A valid model type (like "bert" or "gpt2")
        model_files (`Dict[str, Union[Path, List[Path]]`, *optional*):
            The files associated to `model_type`. Can be passed to speed up the function, otherwise will be computed.

    Returns:
        `str`: The checkpoint used.
    Nrç   rÍ   rs   rt   ru   r   r�   r-   ú')rî   r>   rw   rx   Ú_re_checkpoint_for_docrJ   rK   r0   )r   rç   Úmodule_filesÚfnamerz   rM   r   s          r8   Úfind_base_model_checkpointrô   ’  s×   € ð ÐÜ% jÓ1ˆØ˜}Ñ-€LØò "ˆØœS ›ZÑ'Øä�%˜ wÔ/ð 	"°1Ø—f‘f“hˆGÜ%×,Ñ,¨WÓ5ÐAÜ3×:Ñ:¸7ÓC×JÑJÓLÈQÑO�
à'×/Ñ/°°RÓ8�
Ø'×/Ñ/°°RÓ8�
Ø!÷	"ó 	"àA÷	"ð 	"ð	"ð ÷	"ð 	"ús   ´A1B=Â=C	c                  ó¬   — g } t        «       r| j                  d«       t        «       r| j                  d«       t        «       r| j                  d«       | S )zk
    Returns the list of frameworks (PyTorch, TensorFlow, Flax) that are installed in the environment.
    rÐ   rÎ   rÏ   )r   rU   r   r   rØ   s    r8   rÑ   rÑ   ´  sK   € ð €JÜÔØ×Ñ˜$ÔÜÔØ×Ñ˜$ÔÜÔØ×Ñ˜&Ô!ØÐrA   zMODEL_([A-Z_]*)MAPPING_NAMESc                 ó  — |€
t        «       }t        «       rt        j                  ndt	        «       rt        j
                  ndt        «       rt        j                  nddœ}i }|D ]ž  }g }||   €t        d|› d�«      ‚t        ||   «      D �cg c]  }t        j                  |«      €Œ|‘Œ }}|D ]*  }t        ||   |«      }	| |	v sŒ|j                  |	|    «       Œ, t        |«      dkD  sŒˆt        t!        |«      «      ||<   Œ  |S c c}w )aü  
    Retrieve the model classes associated to a given model.

    Args:
        model_type (`str`): A valid model type (like "bert" or "gpt2")
        frameworks (`List[str]`, *optional*):
            The frameworks to look for. Will default to `["pt", "tf", "flax"]`, passing a smaller list will restrict
            the classes returned.

    Returns:
        `Dict[str, List[str]]`: A dictionary with one key per framework and the list of model classes associated to
        that framework as values.
    N©rÐ   rÎ   rÏ   zYou selected z, in the frameworks, but it is not installed.r   )rÑ   r   Úauto_moduleÚmodeling_autor   Úmodeling_tf_autor   Úmodeling_flax_autor`   ÚdirÚ_re_model_mappingrJ   rƒ   rU   rE   r2   rˆ   )
r   rË   ÚmodulesÚmodel_classesÚ	frameworkÚnew_model_classesr‹   Úmodel_mappingsÚmodel_mapping_nameÚmodel_mappings
             r8   Úretrieve_model_classesr  Å  s*  € ð ÐÜ+Ó-ˆ
ô ,>Ô+?Œk×'Ò'ÀTÜ.=Ô.?Œk×*Ò*ÀTÜ2CÔ2E”×.Ò.È4ñ€Gð €MØò Dˆ	ØÐØ�9ÑÐ%Ü˜}¨Y¨KÐ7cÐdÓeÐeÜ+.¨w°yÑ/AÓ+BÖq 4ÔFW×F^ÑF^Ð_cÓFdÑFpš$ÐqˆÐqØ"0ò 	DÐÜ# G¨IÑ$6Ð8JÓKˆMØ˜]Ò*Ø!×(Ñ(¨°zÑ)BÕCð	Dô
 Ð Ó! AÓ%ä'+¬CÐ0AÓ,BÓ'CˆM˜)Ò$ðDð Ðùò rs   ÂDÂ)Dc                 óT  — | t         j                  vrt        | › d�«      ‚t         j                  |    }t         j                  j                  |    }| t         j
                  j                  v r-t         j
                  j                  |    }|d   �|d   n|d   }nd}t         j                  j                  j                  | d«      }t        |t        «      r|\  }}n|}d}t         j                  j                  j                  | d«      }	t         j                  j                  j                  | d«      }
t!        | |¬«      }|j#                  dd«      }g }|d   D ]_  }d	t%        |«      v r|j'                  d
«       Œ"dt%        |«      v r|j'                  d«       ŒAdt%        |«      v sŒO|j'                  d«       Œa |€
t)        «       }|D �cg c]	  }||v sŒ|‘Œ }}t+        | |¬«      }|j-                  «       }t/        |t1        | |¬«      | ||d   ||||||	|
¬«      }||||dœS c c}w )aß  
    Retrieves all the information from a given model_type.

    Args:
        model_type (`str`): A valid model type (like "bert" or "gpt2")
        frameworks (`List[str]`, *optional*):
            If passed, will only keep the info corresponding to the passed frameworks.

    Returns:
        `Dict`: A dictionary with the following keys:
        - **frameworks** (`List[str]`): The list of frameworks that back this model type.
        - **model_classes** (`Dict[str, List[str]]`): The model classes implemented for that model type.
        - **model_files** (`Dict[str, Union[Path, List[Path]]]`): The files associated with that model type.
        - **model_patterns** (`ModelPatterns`): The various patterns for the model.
    ú is not a valid model type.r   Nr   rØ   r.   r-   rç   Úmodeling_tfrÎ   Úmodeling_flaxrÏ   rÍ   rÐ   )rç   rÇ   )r   r   r!   r    r"   r#   r$   r%   r&   r'   r(   )rË   rÿ   rç   Úmodel_patterns)rø   ÚMODEL_NAMES_MAPPINGr`   Úconfiguration_autoÚCONFIG_MAPPING_NAMESÚtokenization_autoÚTOKENIZER_MAPPING_NAMESÚimage_processing_autoÚIMAGE_PROCESSOR_MAPPING_NAMESÚgetrl   ÚtupleÚfeature_extraction_autoÚFEATURE_EXTRACTOR_MAPPING_NAMESÚprocessing_autoÚPROCESSOR_MAPPING_NAMESrî   r0   r>   rU   rÑ   r  r3   r   rô   )r   rË   r   r#   Útokenizer_classesr$   Úimage_processor_classesr%   r&   r'   r(   rç   r!   Úavailable_frameworksró   rz   rÿ   r"   r
  s                      r8   Úretrieve_info_for_modelr  î  sC  € ð  œ×8Ñ8Ñ8Ü˜J˜<Ð'BÐCÓDÐDä×0Ñ0°Ñ<€JÜ×1Ñ1×FÑFÀzÑR€LØ”[×2Ñ2×JÑJÑJÜ'×9Ñ9×QÑQÐR\Ñ]ÐØ2CÀAÑ2FÐ2RÐ+¨AÒ.ÐXiÐjkÑXl‰àˆÜ)×?Ñ?×]Ñ]×aÑaÐblÐnrÓsÐÜÐ)¬5Ô1Ø<SÑ9ÐÑ9à 7ÐØ%)Ð"Ü)×AÑA×aÑa×eÑeÐfpÐrvÓwÐÜ!×1Ñ1×IÑI×MÑMÈjÐZ^Ó_€Oä! *¸ÔD€KØ$×,Ñ,¨X°rÓ:ÐàÐØ˜]Ñ+ò .ˆØœC ›JÑ&Ø ×'Ñ'¨Õ-Ø¤ E£
Ñ*Ø ×'Ñ'¨Õ/Øœ3˜u›:Ò%Ø ×'Ñ'¨Õ-ð.ð ÐÜ+Ó-ˆ
à'ÖE˜¨1Ð0DÒ+D’!ÐE€JÐEä*¨:À*ÔM€Mà)×/Ñ/Ó1ÐÜ"ØÜ-¨jÀkÔRØØ+Ø% mÑ4Ø+Ø!Ø'Ø3Ø#=Ø 7Ø'ô€Nð  !Ø&Ø"Ø(ñ	ð ùò) Fs   Ç	H%ÇH%Ú	init_fileÚkeep_processingc                 ó¶  — |€
t        «       }ddi}dD �cg c]  }||vsŒ|j                  ||«      ‘Œ }}|s|j                  g d¢«       t        |«      dk(  rydj	                  |«      }t        j                  d|› d	�«      }t        j                  d
«      }t        j                  d«      }	t        j                  d|› d�«      }
t        | dd¬«      5 }|j                  «       }ddd«       j                  d«      }g }d}|t        |«      k  �rô|j                  ||   «      �Ñ|j                  ||dz
     «      �º|j                  «        |dz  }t        ||   «      s|	j                  ||   «      €)|dz  }t        ||   «      rŒ|	j                  ||   «      €Œ)|dz  }t        ||   «      }t        ||   «      |k\  st        ||   «      �r&|dz  }t        ||   «      |k\  rŒt        ||   «      rŒ&nÿ|
j                  ||   «      �—||   }|D ]Z  }|j                  d|› d�d«      }|j                  d|› d�d«      }|j                  d|› d�d«      }|j                  d|› d�d«      }Œ\ t        |j                  «       «      dkD  r|j!                  |«       |dz  }nT|s3t        j                  d||   «      €4t        j                  d||   «      	 €|j!                  ||   «       |dz  }n|dz  }|t        |«      k  r�Œôt        | dd¬«      5 }|j#                  dj	                  |«      «       ddd«       yc c}w # 1 sw Y   �Œ_xY w# 1 sw Y   yxY w)ag  
    Removes all the import lines that don't belong to a given list of frameworks or concern tokenizers/feature
    extractors/image processors/processors in an init.

    Args:
        init_file (`str` or `os.PathLike`): The path to the init to treat.
        frameworks (`List[str]`, *optional*):
           If passed, this will remove all imports that are subject to a framework not in frameworks
        keep_processing (`bool`, *optional*, defaults to `True`):
            Whether or not to keep the preprocessing (tokenizer, feature extractor, image processor, processor) imports
            in the init.
    NrÐ   Útorchr÷   )ÚsentencepieceÚ
tokenizersÚvisionr   ú|z^\s*if not is_(z)_available\(\):\s*$z\s*try:z\s*else:zis_(z)_availablers   rt   ru   rO   r   z, is_Ú
_availabler-   Úis_z_available, z_available,zB^\s*"(tokenization|processing|feature_extraction|image_processing)zG^\s*from .(tokenization|processing|feature_extraction|image_processing)r7   )rÑ   r  r…   rE   r4   rI   Úcompilerw   rx   r1   rJ   r›   rG   rL   r0   ÚstriprU   ry   )r  rË   r  Únamesrz   Ú	to_removeÚremove_patternÚre_conditional_importsÚre_tryÚre_elseÚre_is_xxx_availablerM   rX   ro   rc   ÚindentrB   r   s                     r8   Úclean_frameworks_in_initr0  <  sl  € ð ÐÜ+Ó-ˆ
à�7ˆO€EØ*>ÖV QÀ!È:ÒBU�—‘˜1˜a•ÐV€IÐVÙØ×ÑÒBÔCä
ˆ9ƒ~˜Òàà—X‘X˜iÓ(€NÜŸZ™Z¨?¸>Ð:JÐJ^Ð(_Ó`ÐÜ�Z‰Z˜
Ó#€FÜ�j‰j˜Ó%€GÜŸ*™*¨¨^Ð,<¸KÐ%HÓIÐä	ˆi˜ wÔ	/ð °1Ø—&‘&“(ˆ÷ð �M‰M˜$Ó€EØ€IØ
€CØ
”�E“
Ó
à"×)Ñ)¨%°©*Ó5ÐAÈÏÉÐV[Ð\_ÐbcÑ\cÑVdÓHeÐHqà�M‰MŒOØ�1‰HˆCä  c¡
Ô+¨w¯~©~¸eÀC¹jÓ/IÐ/QØ�q‘�ô    c¡
Õ+¨w¯~©~¸eÀC¹jÓ/IÑ/Qà�1‰HˆCÜ   s¡Ó,ˆFÜ˜e C™jÓ)¨VÒ3´}ÀUÈ3ÁZÕ7PØ�q‘�ô ˜e C™jÓ)¨VÓ3´}ÀUÈ3ÁZÖ7Pð !×'Ñ'¨¨c©
Ó3Ð?Ø˜‘:ˆDØ&ò E�	Ø—|‘| e¨I¨;°jÐ$AÀ2ÓF�Ø—|‘| c¨)¨°LÐ$AÀ2ÓF�Ø—|‘| c¨)¨°KÐ$@À"ÓE�Ø—|‘| c¨)¨°JÐ$?ÀÓD‘ð	Eô �4—:‘:“<Ó  1Ò$Ø× Ñ  Ô&Ø�1‰H‰CáÜ�I‰IÐ[Ð]bÐcfÑ]gÓhÐpÜ—	‘	ÐdÐfkÐloÑfpÓqØðð ×Ñ˜U 3™ZÔ(Ø�1‰H‰Cà�1‰HˆCðE ”�E“
Ô
ôH 
ˆi˜ wÔ	/ð &°1Ø	�‰�—	‘	˜)Ó$Ô%÷&ð &ùòq W÷ñ ú÷T&ð &ús"   •	L=ŸL=ÃMÌ!MÍMÍMÚwith_processingc                 ó  — t        t        dz  dd¬«      5 }|j                  «       }ddd«       j                  d«      }d}g }d}	|t	        |«      k  �rÏd}
t        ||   «      st        ||   «      dk(  rd}	nt||   j                  «       j                  d	«      rd
}	d}
nM||   j                  «       j                  d«      rd}	d}
n&||   j                  «       j                  d«      rd}	d}
|
rF||   j                  «       dk7  r0|j                  ||   «       |dz  }||   j                  «       dk7  rŒ0|	�!|�|	|vr|j                  ||   «       |dz  }�n¼t        j                  d| j                  › d�||   «      ��{||   g}t        ||   «      }|dz  }t        ||   «      |kD  r+|j                  ||   «       |dz  }t        ||   «      |kD  rŒ+||   j                  «       dv r|j                  ||   «       |dz  }dj                  |«      }|j                  |«       d}|s°| j                  | j                   | j"                  | j$                  | j&                  g}|D �cg c]  }|€Œ|‘Œ	 }}|D ]`  }|j)                  d|› d�d«      }|j)                  d|› d�d«      }|j)                  d|› d�d«      }|j)                  d|› �d«      }||v sŒ_d}Œb |r9|j                  t+        || |«      d   «       n|j                  ||   «       |dz  }|t	        |«      k  r�ŒÏt        t        dz  dd¬«      5 }|j-                  dj                  |«      «       ddd«       y# 1 sw Y   �Œ>xY wc c}w # 1 sw Y   yxY w)a  
    Add a model to the main init of Transformers.

    Args:
        old_model_patterns (`ModelPatterns`): The patterns for the old model.
        new_model_patterns (`ModelPatterns`): The patterns for the new model.
        frameworks (`List[str]`, *optional*):
            If specified, only the models implemented in those frameworks will be added.
        with_processing (`bool`, *optional*, defaults to `True`):
            Whether the tokenizer/feature extractor/processor of the model should also be added to the init or not.
    ú__init__.pyrs   rt   ru   NrO   r   Fzif not is_torch_availablerÐ   Tzif not is_tf_availablerÎ   zif not is_flax_availablerÏ   zelse:r   zmodels.z( |"))rQ   rR   z],z "z",r-   z, "r�   r*   r‚   ú, r7   )rw   ré   rx   r1   rE   rG   rL   rb   rT   r'  rU   rI   rJ   r    r4   r$   r%   r&   r'   r(   r0   r’   ry   )r~   r   rË   r1  rz   rM   rX   rc   ro   r   Únew_frameworkÚblockr/  Ú	add_blockÚprocessing_classesÚcÚprocessing_classs                    r8   Úadd_model_to_main_initr;  ‹  sÃ  € ô" 
Ô -Ñ/°¸wÔ	Gð È1Ø—&‘&“(ˆ÷ð �M‰M˜$Ó€EØ
€CØ€IØ€IØ
”�E“
Ó
ØˆÜ˜U 3™ZÔ(¬[¸¸s¹Ó-DÈÒ-IØ‰IØ�3‰Z×ÑÓ ×+Ñ+Ð,GÔHØˆIØ ‰MØ�3‰Z×ÑÓ ×+Ñ+Ð,DÔEØˆIØ ‰MØ�3‰Z×ÑÓ ×+Ñ+Ð,FÔGØˆIØ ˆMáà˜‘*×"Ñ"Ó$¨Ò/Ø× Ñ   s¡Ô,Ø�q‘�ð ˜‘*×"Ñ"Ó$¨Ó/ð
 Ð  ZÐ%;À	ÐQ[Ñ@[Ø×Ñ˜U 3™ZÔ(Ø�1‰HŠCÜ�Y‰Y˜'Ð"4×"FÑ"FÐ!GÀuÐMÈuÐUXÉzÓZÑfØ˜3‘Z�LˆEÜ   s¡Ó,ˆFØ�1‰HˆCÜ˜e C™jÓ)¨FÒ2Ø—‘˜U 3™ZÔ(Ø�q‘�ô ˜e C™jÓ)¨FÓ2ð �S‰z×ÑÓ!Ð%5Ñ5Ø—‘˜U 3™ZÔ(Ø�q‘�Ø—I‘I˜eÓ$ˆEØ×Ñ˜UÔ#àˆIÙ"à&×6Ñ6Ø&×<Ñ<Ø&×AÑAØ&×>Ñ>Ø&×6Ñ6ð&Ð"ð 2DÖ%U¨AÀqÁ}¢aÐ%UÐ"Ð%UØ(:ò *Ð$Ø!ŸM™M¨BÐ/?Ð.@ÀÐ*CÀRÓH�EØ!ŸM™M¨CÐ0@Ð/AÀÐ*CÀRÓH�EØ!ŸM™M¨AÐ.>Ð-?¸qÐ*AÀ2ÓF�EØ!ŸM™M¨BÐ/?Ð.@Ð*AÀ2ÓF�Eà'¨5Ò0Ø$)™	ð*ñ Ø× Ñ Ô!7¸Ð?QÐSeÓ!fÐghÑ!iÕjà×Ñ˜U 3™ZÔ(Ø�1‰HˆCðy ”�E“
Ô
ô| 
Ô -Ñ/°¸wÔ	Gð &È1Ø	�‰�—	‘	˜)Ó$Ô%÷&ð &÷Kñ üòl &V÷&ð &ús#   –M#É&M0É.M0Ì9!M5Í#M-Í5M>c                 ó  — | j                   �|j                   €yt        t        dz  dz  dz  dd¬«      5 }|j                  «       }ddd«       j	                  d«      }d	}||   j                  d
«      s|dz  }||   j                  d
«      sŒ|dz  }||   j                  d«      sª||   j                  d«      r||   }nUg }||   j                  d«      s.|j                  ||   «       |dz  }||   j                  d«      sŒ.dj                  |«      }|dz  }d| j                  › d�|v r| j                   |v rn||   j                  d«      sŒªj                  | j                  |j                  «      }|j                  | j                   |j                   «      }|d| |gz   ||d z   }t        t        dz  dz  dz  dd¬«      5 }|j                  dj                  |«      «       ddd«       y# 1 sw Y   �Œ®xY w# 1 sw Y   yxY w)zí
    Add a tokenizer to the relevant mappings in the auto module.

    Args:
        old_model_patterns (`ModelPatterns`): The patterns for the old model.
        new_model_patterns (`ModelPatterns`): The patterns for the new model.
    Nr×   r   ztokenization_auto.pyrs   rt   ru   rO   r   z*    TOKENIZER_MAPPING_NAMES = OrderedDict(r   z$TOKENIZER_MAPPING = _LazyAutoMappingr‚   z            ),r�   r7   )r$   rw   ré   rx   r1   rT   ra   rU   r4   r   r0   ry   )	r~   r   rz   rM   rX   rc   r6  Ú	new_blockro   s	            r8   Úinsert_tokenizer_in_auto_moduler>  å  s1  € ð ×)Ñ)Ð1Ð5G×5WÑ5WÐ5_Øä	Ô (Ñ*¨VÑ3Ð6LÑLÈcÐ\cÔ	dð ÐhiØ—&‘&“(ˆ÷ð �M‰M˜$Ó€EØ
€Cà�C‰j×#Ñ#Ð$PÔQØˆq‰ˆð �C‰j×#Ñ#Ð$PÕQàˆ1�H€Cð �C‰j×#Ñ#Ð$JÔKà�‰:×Ñ˜sÔ#Ø˜#‘J‰Eð ˆEØ˜C‘j×+Ñ+Ð,<Ô=Ø—‘˜U 3™ZÔ(Ø�q‘�ð ˜C‘j×+Ñ+Ð,<Õ=ð —I‘I˜eÓ$ˆEØˆq‰ˆð Ð!×,Ñ,Ð-¨QÐ/°5Ñ8Ð=O×=_Ñ=_ÐchÑ=hØð �C‰j×#Ñ#Ð$JÕKð" —‘Ð0×;Ñ;Ð=O×=ZÑ=ZÓ[€IØ×!Ñ!Ð"4×"DÑ"DÐFX×FhÑFhÓi€Ià�d�s�˜y˜kÑ)¨E°#°$¨KÑ7€IÜ	Ô (Ñ*¨VÑ3Ð6LÑLÈcÐ\cÔ	dð &ÐhiØ	�‰�—	‘	˜)Ó$Ô%÷&ð &÷Añ ú÷@&ð &ús   µG2Ç!G?Ç2G<Ç?H)z)        ("{model_type}", "{model_name}"),z+        ("{model_type}", "{config_class}"),z5        ("{model_type}", "{pretrained_archive_map}"),z6        ("{model_type}", "{feature_extractor_class}"),z6        ("{model_type}", "{image_processor_classes}"),z+        ("{model_type}", "{any_pt_class}"),z+        ("{model_type}", "{any_tf_class}"),z-        ("{model_type}", "{any_flax_class}"),z.        ("{model_type}", "{processor_class}"),)zconfiguration_auto.pyzfeature_extraction_auto.pyzimage_processing_auto.pyzmodeling_auto.pyzmodeling_tf_auto.pyzmodeling_flax_auto.pyzprocessing_auto.pyrÿ   c                 ó”  — t         D �]­  }g }t         |   D �]  }t        j                  d|«      �it        j                  d|«      j                  «       d   }||v sŒF|j	                  ||   D �cg c]  }|j                  dd|› d�z   dz   |«      ‘Œ  c}«       Œƒd|v r,|j                  |j                  d| j                  «      «       Œ³d	|v r¡| j                  €ŒÄ|j                  €ŒÑ| j                  �J|j                  �>|j                  |j                  d
d| j                  › d| j                  › d�«      «       �Œ'|j                  |j                  d
d| j                  › d�«      «       �ŒXd|v rI| j                  €�Œj|j                  €�Œx|j                  |j                  d| j                  «      «       �Œ¥d|v rI| j                  €�Œ·|j                  €�ŒÅ|j                  |j                  d| j                  «      «       �Œò|j                  |«       �Œ |D ]”  }t        dz  dz  |z  }|}	|}
dD ]F  }|	j                  d|z   dz   t        | |«      «      }	|
j                  d|z   dz   t        ||«      «      }
ŒH |
j                  | j                  |j                  «      }
t        ||
|	¬«       Œ– �Œ° t!        | |«       yc c}w )aV  
    Add a model to the relevant mappings in the auto module.

    Args:
        old_model_patterns (`ModelPatterns`): The patterns for the old model.
        new_model_patterns (`ModelPatterns`): The patterns for the new model.
        model_classes (`Dict[str, List[str]]`): A dictionary framework to list of model classes implemented.
    zany_([a-z]*)_classNr   ú{Úany_Ú_classrS   z{config_class}z{image_processor_classes}z"{image_processor_classes}"z("z", "z")z",)z{feature_extractor_class}z{processor_class}r×   r   )r   r   )rf   )ÚAUTO_CLASSES_PATTERNSrI   rJ   rK   r…   r0   rU   r#   r%   r&   r'   r(   ré   rƒ   r!   r}   r>  )r~   r   rÿ   ÚfilenameÚnew_patternsrm   r   ÚclsÚ	full_nameÚold_model_lineÚnew_model_liner‹   s               r8   Úadd_model_to_auto_classesrJ  #  s  € ô *ó =UˆàˆÜ,¨XÑ6ó -	-ˆGÜ�y‰yÐ-¨wÓ7ÐCÜŸI™IÐ&:¸GÓD×KÑKÓMÈaÑP�	Ø Ò-Ø ×'Ñ'ð (5°YÑ'?öà #ð $ŸO™O¨C°D¸¸À6Ð2JÑ,JÈSÑ,PÐRUÕVòõð " WÑ,Ø×#Ñ# G§O¡OÐ4DÐFX×FeÑFeÓ$fÕgØ,°Ñ7à&×<Ñ<ÑHØ*×@Ñ@ÑLð +×EÑEÐQØ.×IÑIÐUà$×+Ñ+Ø#ŸO™OØ =Ø"$Ð%7×%MÑ%MÐ$NÈdÐSe÷  TAñ  TAð  SBð  BDð  !Eóöð %×+Ñ+Ø#ŸO™OØ =ÀÐDV×DlÑDlÐCmÐmpÐ?qóöð
 -°Ñ7à&×>Ñ>ÒJØ*×BÑBÒNà ×'Ñ'ØŸ™Ð(CÐEW×EoÑEoÓpöð %¨Ñ/Ø%×5Ñ5ÒAÐFX×FhÑFhÒFtØ ×'Ñ'¨¯©Ð8KÐM_×MoÑMoÓ(pÖqà×#Ñ# GÖ,ð[-	-ð` $ò 
	UˆGÜ)¨HÑ4°vÑ=ÀÑHˆIØ$ˆNØ$ˆNØ4ò m�Ø!/×!7Ñ!7¸¸d¹
ÀSÑ8HÌ'ÐRdÐfjÓJkÓ!l�Ø!/×!7Ñ!7¸¸d¹
ÀSÑ8HÌ'ÐRdÐfjÓJkÓ!l‘ðmð ,×3Ñ3Ø"×4Ñ4Ð6H×6ZÑ6ZóˆNô   	¨>À^ÖTò
	Uðg=Uô@ $Ð$6Ð8JÕKùòqs   Á.#KaÖ  ## Overview

The {model_name} model was proposed in [<INSERT PAPER NAME HERE>](<INSERT PAPER LINK HERE>) by <INSERT AUTHORS HERE>.
<INSERT SHORT SUMMARY HERE>

The abstract from the paper is the following:

*<INSERT PAPER ABSTRACT HERE>*

Tips:

<INSERT TIPS ABOUT MODEL HERE>

This model was contributed by [INSERT YOUR HF USERNAME HERE](https://huggingface.co/<INSERT YOUR HF USERNAME HERE>).
The original code can be found [here](<INSERT LINK TO GITHUB REPO HERE>).

ræ   c                 ó  — t        | dd¬«      5 }|j                  «       }ddd«       t        j                  ddt        › d�«      }|€
t        «       }|€%t        | «      j                  |j                  › d�z  }|j                  d	«      }g }g }	|D ]H  }
|
j                  d
«      r$|j                  d	j                  |	«      «       |
g}	Œ8|	j                  |
«       ŒJ |j                  d	j                  |	«      «       g }d}|D �]¶  }|j                  d
«      s|j                  |«       Œ't        j                  d|«      � |j                  d|j                  › d	�«       Œ]|ss|j                  |j                  d	«      d   v rSd}|j                  t         j#                  |j                  ¬«      «       t%        |||«      \  }}|j                  |«       ŒÒ|sŒÕd}|j                  d	«      d   }t        j                  d|«      j'                  «       d   }t%        |||«      \  }}d|v r.|j(                  |j(                  k7  s�ŒA|j                  |«       �ŒTd|v r.|j*                  |j*                  k7  s�Œs|j                  |«       �Œ†d|v r.|j,                  |j,                  k7  s�Œ¥|j                  |«       �Œ¸d|v r.|j.                  |j.                  k7  s�Œ×|j                  |«       �Œêd|v r.|j0                  |j0                  k7  s�Œ	|j                  |«       �Œ|j                  d«      rd|v s�Œ3|j                  |«       �ŒF|j                  d«      rd|v s�Œ]|j                  |«       �Œpt3        |j                  d«      «      dk(  rd|v s�Œ“|j                  |«       �Œ¦|j                  |«       �Œ¹ t        |dd¬«      5 }|j5                  d	j                  |«      «       ddd«       y# 1 sw Y   �ŒÖxY w# 1 sw Y   yxY w)a¬  
    Duplicate a documentation file and adapts it for a new model.

    Args:
        module_file (`str` or `os.PathLike`): Path to the doc file to duplicate.
        old_model_patterns (`ModelPatterns`): The patterns for the old model.
        new_model_patterns (`ModelPatterns`): The patterns for the new model.
        dest_file (`str` or `os.PathLike`, *optional*): Path to the new doc file.
            Will default to the a file named `{new_model_patterns.model_type}.md` in the same folder as `module_file`.
        frameworks (`List[str]`, *optional*):
            If passed, will only keep the model classes corresponding to this list of frameworks in the new doc file.
    rs   rt   ru   Nz<!--\s*Copyright (\d+)\sz<!--Copyright r*   rÝ   rO   r®   Fz^#\s+\S+z# r   T©r   z^#+\s+(\S.*)$Ú	TokenizerÚImageProcessorÚImageProcessorFastÚFeatureExtractorÚ	ProcessorÚFlaxrÏ   ÚTFrÎ   r   rÐ   r7   )rw   rx   rI   r„   r¾   rÑ   r   Úparentr   r1   rT   rU   r4   rJ   r   r#   ÚDOC_OVERVIEW_TEMPLATEÚformatr’   rK   r$   r%   r&   r'   r(   rE   ry   )ræ   r~   r   rº   rË   rz   rM   rX   ÚblocksÚcurrent_blockrB   Ú
new_blocksÚ
in_classesr6  r=  r,   Úblock_titleÚblock_classs                     r8   Úduplicate_doc_filer]  „  s»  € ô& 
ˆh˜ gÔ	.ð °!Ø—&‘&“(ˆ÷ô �f‰fÐ0°NÄ<À.ÐPQÐ2RÐT[Ó\€GØÐÜ+Ó-ˆ
ØÐÜ˜“N×)Ñ)Ð/A×/LÑ/LÐ.MÈSÐ,QÑQˆ	ð �M‰M˜$Ó€EØ€FØ€Màò 'ˆØ�?‰?˜3ÔØ�M‰M˜$Ÿ)™) MÓ2Ô3Ø!˜F‰Mà× Ñ  Õ&ð'ð ‡M�M�$—)‘)˜MÓ*Ô+à€JØ€JØó 5-ˆà×Ñ Ô$Ø×Ñ˜eÕ$ä�Y‰Y�{ EÓ*Ð6Ø×Ñ Ð#5×#@Ñ#@Ð"AÀÐDÕEáÐ 2× ?Ñ ?À5Ç;Á;ÈtÓCTÐUVÑCWÑ WØˆJØ×ÑÔ3×:Ñ:ÐFX×FcÑFcÐ:ÓdÔeÜ1°%Ð9KÐM_Ó`‰LˆI�qØ×Ñ˜iÕ(âØˆJØŸ+™+ dÓ+¨AÑ.ˆKÜŸ)™)Ð$4°kÓB×IÑIÓKÈAÑNˆKÜ1°%Ð9KÐM_Ó`‰LˆI�qà˜kÑ)à%×5Ñ5Ð9K×9[Ñ9[Ô[Ø×%Ñ% iÖ0Ø! [Ñ0à%×;Ñ;Ð?Q×?gÑ?gÔgØ×%Ñ% iÖ0Ø%¨Ñ4à%×@Ñ@ÐDV×DqÑDqÔqØ×%Ñ% iÖ0Ø# {Ñ2à%×=Ñ=ÐAS×AkÑAkÔkØ×%Ñ% iÖ0Ø Ñ+à%×5Ñ5Ð9K×9[Ñ9[Ô[Ø×%Ñ% iÖ0Ø×'Ñ'¨Ô/à˜ZÓ'Ø×%Ñ% iÖ0Ø×'Ñ'¨Ô-à˜:Ó%Ø×%Ñ% iÖ0Ü�[×&Ñ& sÓ+Ó,°Ò1à˜:Ó%Ø×%Ñ% iÖ0à×!Ñ! )Ö,ðk5-ôn 
ˆi˜ wÔ	/ð '°1Ø	�‰�—	‘	˜*Ó%Ô&÷'ð '÷_ñ ú÷^'ð 'ús   �O4Ï
!PÏ4O>ÐP
c                 óš  — t         dz  dz  dz  dz  }t        |dd¬«      5 }t        j                  |«      }ddd«       d	}|   d
   dk7  r|dz  }||   d
   dk7  rŒ||   d   }d	}||   d
   dk7  r|dz  }||   d
   dk7  rŒ||   d   }| j                  }	d	}
|
t        |«      k  r4||
   d   D �cg c]  }|d   ‘Œ	 }}d|	› �|v rn|
dz  }
|
t        |«      k  rŒ4|
t        |«      k(  r+| j                  }|j                  }t        d|› d|› d�«       yd|j                  › �|j                  dœ}||
   d   j                  |«       t        ||
   d   d„ ¬«      ||
   d<   |||   d<   |||   d<   t        |dd¬«      5 }|j                  t        j                  |d¬«      «       ddd«       y# 1 sw Y   �ŒzxY wc c}w # 1 sw Y   yxY w)zû
    Insert the new model in the doc TOC, in the same section as the old model.

    Args:
        old_model_patterns (`ModelPatterns`): The patterns for the old model.
        new_model_patterns (`ModelPatterns`): The patterns for the new model.
    rÙ   rÚ   rÛ   z_toctree.ymlrs   Úutf8ru   Nr   ÚtitleÚAPIr   ÚsectionsÚModelsÚlocalz
model_doc/zDid not find z2 in the table of content, so you will need to add z
 manually.)rd  r`  c                 ó(   — | d   j                  «       S )Nr`  )r/   )Úss    r8   r—   z)insert_model_in_doc_toc.<locals>.<lambda>  s   € ÐbcÐdkÑbl×brÑbrÓbt€ rA   r˜   r7   rt   T)Úallow_unicode)rë   rw   ÚyamlÚ	safe_loadr   rE   r   ÚprintrU   Úsortedry   Údump)r~   r   Útoc_filerz   rM   Úapi_idxÚapi_docÚ	model_idxrÜ   Úold_model_typeÚsection_idxÚentryrb  Ú	old_modelÚ	new_modelÚ	toc_entrys                   r8   Úinsert_model_in_doc_tocrw  ê  s@  € ô ˜6Ñ! HÑ,¨tÑ3°nÑD€HÜ	ˆh˜ fÔ	-ð $°Ü—.‘. Ó#ˆ÷$ð €GØ
�'Ñ
˜7Ñ
# uÒ
,Ø�1‰ˆð �'Ñ
˜7Ñ
# uÓ
,à�gÑ˜zÑ*€Gà€IØ
�)Ñ
˜WÑ
%¨Ò
1Ø�Q‰ˆ	ð �)Ñ
˜WÑ
%¨Ó
1à˜	Ñ" :Ñ.€Ið (×2Ñ2€NØ€KØ
œ˜I›Ò
&Ø09¸+Ñ0FÀzÑ0RÖS u�E˜'“NÐSˆÐSØ˜Ð'Ð(¨HÑ4Øà�qÑˆð œ˜I›Ó
&ð ”c˜)“nÒ$Ø&×1Ñ1ˆ	Ø&×1Ñ1ˆ	Ü�˜i˜[Ð(ZÐ[dÐZeÐeoÐpÔqØð 'Ð'9×'DÑ'DÐ&EÐFÐQc×QnÑQnÑo€IØˆkÑ˜:Ñ&×-Ñ-¨iÔ8Ü)/°	¸+Ñ0FÀzÑ0RÑXtÔ)u€IˆkÑ˜:Ñ&Ø%.€GˆIÑ�zÑ"Ø#*€GˆGÑ�ZÑ ä	ˆh˜ gÔ	.ð 8°!Ø	�‰”—	‘	˜'°Ô6Ô7÷8ð 8÷K$ñ $üò$ T÷&8ð 8ús   ¡F/Â1F<Å?'GÆ/F9ÇG
Úold_checkpointÚcreate_fast_image_processorc           
      óÀ  — t        | |¬«      }|d   }|d   }|�||_        t        |j                  «      dk(  rt        d«      ‚d}	dD ]  }
t	        ||
«      t	        ||
«      k7  sŒd	}	Œ  |d
   }|d   }t
        dz  |j                  z  }t        j                  |d¬«       |d   }|	rC|D �cg c]8  }dt        |«      vr)dt        |«      vrdt        |«      vrdt        |«      vr|‘Œ: }}t        j                  |d¬«       |D ]M  }|j                  j                  |j                  |j                  «      }||z  }t        |||||xr d|v ¬«       ŒO t        |dz  ||	 ¬«       t        t
        dz  dz  d|j                  › d�d|› d�d¬«       t        ||||	 ¬«       |d   }|	rC|D �cg c]8  }dt        |«      vr)dt        |«      vrdt        |«      vrdt        |«      vr|‘Œ: }}dt         dt"        fd„}d	}t$        dz  dz  |j                  z  }t        j                  |d¬«       t'        |dz  d «      5  	 ddd«       |D ]v  }|j                  j                  |j                  |j                  «      }|j(                  j(                  |j                  z  |z  }t        ||||d	d!d"g¬#«       | ||«      z  }Œx |rt+        d$«       t-        |||«       t$        d%z  d&z  d'z  d(z  |j.                  › d)�z  }t1        ||||¬«       t3        ||«       |rt5        |j                  ¬*«       |j.                  |j                  k(  r4t+        d+|j.                  › d,|j.                  › d-|j.                  › d.�«       nL|j                  |j                  k(  r3t+        d+|j                  › d,|j                  › d-|j                  › d.�«       |j.                  |j                  k(  rL|j.                  |j                  k7  r3t+        d/|j                  › d0|j                  › d-|j                  › d1�«       |	s|j6                  �t+        d2«       yyyc c}w c c}w # 1 sw Y   �ŒxY w)3a³  
    Creates a new model module like a given model of the Transformers library.

    Args:
        model_type (`str`): The model type to duplicate (like "bert" or "gpt2")
        new_model_patterns (`ModelPatterns`): The patterns for the new model.
        add_copied_from (`bool`, *optional*, defaults to `True`):
            Whether or not to add "Copied from" statements to all classes in the new model modeling files.
        frameworks (`List[str]`, *optional*):
            If passed, will limit the duplicate to the frameworks specified.
        old_checkpoint (`str`, *optional*):
            The name of the base checkpoint for the old model. Should be passed along when it can't be automatically
            recovered from the `model_type`.
        create_fast_image_processor (`bool`, *optional*, defaults to `False`):
            Whether or not to add a fast image processor to the new model, if the old model had only a slow one.
    rØ   rç   r
  Nr   zuThe old model checkpoint could not be recovered from the model type. Please pass it to the `old_checkpoint` argument.T)r%   r&   r'   r(   r$   Frÿ   rÇ   r×   )Úexist_okÚtokenizationÚ
processingÚfeature_extractionÚimage_processingrÍ   )rº   r»   r3  )rË   r  z    r‚   )rf   rh   )rË   r1  rè   Ú	processorrD  rC   c                 ó  — t        | «      5 }|j                  «       }d d d «       t        j                  dd«      }t        | d«      5 }|j	                  |«       d d d «       ||k7  S # 1 sw Y   ŒKxY w# 1 sw Y   ||k7  S xY w)Nzfx_compatible\s*=\s*Truezfx_compatible = Falser7   )rw   rx   rI   r„   ry   )rD  ÚfprM   r|   s       r8   Údisable_fx_testz.create_new_model_like.<locals>.disable_fx_test†  s~   € Ü�(‹^ð 	 ˜rØ—g‘g“iˆG÷	 ä—f‘fÐ8Ð:QÐSZÓ[ˆÜ�(˜CÓ ð 	" BØ�H‰H�[Ô!÷	"à˜+Ñ%Ð%÷	 ð 	 ú÷	"à˜+Ñ%Ð%ús   ŒA'ÁA3Á'A0Á3B rå   r7   Úpipeline_model_mappingÚis_pipeline_test_to_skip)rº   r»   r¼   z}The tests for symbolic tracing with torch.fx were disabled, you can add those once symbolic tracing works for your new model.rÙ   rÚ   rÛ   rÜ   rÝ   rL  zSThe model you picked has the same name for the model type and the checkpoint name (zW). As a result, it's possible some places where the new checkpoint should be, you have z1 instead. You should search for all instances of zB in the new files and check they're not badly used as checkpoints.zYThe model you picked has the same name for the model type and the lowercased model name (zW). As a result, it's possible some places where the new model type should be, you have zE in the new files and check they're not badly used as the model type.zûThe constants at the start of the new tokenizer file created needs to be manually fixed. If your new model has a tokenizer fast, you will also need to manually add the converter in the `SLOW_TO_FAST_CONVERTERS` constant of `convert_slow_tokenizer.py`.)r  r   rE   r`   rƒ   ré   r    r°   Úmakedirsr>   rÒ   r0   rÉ   r0  r}   r;  r   Úboolrë   rw   rT  rj  rJ  r   r]  rw  r   r$   )r   r   r»   rË   rx  ry  Ú
model_inforç   r~   Úkeep_old_processingÚprocessing_attrrÿ   Úold_module_nameÚmodule_folderÚfiles_to_adaptrz   rŸ   Únew_module_namerº   rƒ  Údisabled_fx_testÚtests_folderÚ	test_fileÚnew_test_file_nameræ   s                            r8   Úcreate_new_model_liker“    sI  € ô2 )¨À
ÔK€JØ˜]Ñ+€KØ#Ð$4Ñ5ÐØÐ!Ø(6ÐÔ%Ü
Ð×(Ñ(Ó)¨QÒ.Üð)ó
ð 	
ð
 Ððò (ˆô Ð% Ó7¼7ÐCUÐWfÓ;gÓgØ"'Ñð(ð ˜Ñ/€Mð " -Ñ0€OÜ%¨Ñ0Ð3E×3WÑ3WÑW€MÜ‡K�K�¨Õ-à  Ñ/€NÙð $ö
àØ¤S¨£VÑ+Ø¤C¨£FÑ*Ø$¬C°«FÑ2Ø"¬#¨a«&Ñ0ò ð
ˆð 
ô ‡K�K�¨Õ-Ø%ò 
ˆØ%×*Ñ*×2Ñ2Ø×0Ñ0Ð2D×2VÑ2Vó
ˆð " OÑ3ˆ	ÜØØØØØ+ÒM°
¸oÐ0Mö	
ð
ô Ø˜Ñ%°*ÐReÐNeõô
 Ü˜HÑ$ }Ñ4Ø
Ð!×3Ñ3Ð4°AÐ6Ø˜Ð)¨Ð+Øõ	ô ØÐ.¸:Ð[nÐWnõð
 ! Ñ.€NÙð $ö
àØ¤S¨£VÑ+Ø¤3 q£6Ñ)Ø$¬C°«FÑ2Ø"¬#¨a«&Ñ0ò ð
ˆð 
ð&¤$ð &¬4ó &ð Ðä˜wÑ&¨Ñ1Ð4F×4XÑ4XÑX€LÜ‡K�K� tÕ,Ü	ˆl˜]Ñ*¨CÓ	0ñ Ø÷ð $ò Iˆ	Ø&Ÿ^™^×3Ñ3Ø×0Ñ0Ð2D×2VÑ2Vó
Ðð ×$Ñ$×+Ñ+Ð.@×.RÑ.RÑRÐUgÑgˆ	ÜØØØØØ!Ø5Ð7QÐRõ	
ð ,©o¸iÓ.HÑHÑðIñ Üð#ô	
ô Ð0Ð2DÀmÔTô ˜6Ñ! HÑ,¨tÑ3°kÑAÐGY×GdÑGdÐFeÐehÐDiÑi€HÜ�xÐ!3Ð5GÐT^Õ_ÜÐ.Ð0BÔCñ #Ü Ð,>×,PÑ,PÕQð ×$Ñ$Ð(:×(EÑ(EÒEÜðØ"×-Ñ-Ð.ð /#Ø#5×#@Ñ#@Ð"AÐArØ!×,Ñ,Ð-Ð-oðqõ	
ð 
×	-Ñ	-Ð1C×1NÑ1NÒ	NÜðØ"×4Ñ4Ð5ð 6.Ø.@×.RÑ.RÐ-Sð T Ø 2× DÑ DÐEð F#ð#ô	
ð 	×%Ñ%Ð);×)MÑ)MÒMØ×)Ñ)Ð-?×-QÑ-QÒQäðØ"×4Ñ4Ð5ð 6.Ø.@×.RÑ.RÐ-Sð T Ø 2× DÑ DÐEð F&ð&ô	
ñ Ð#5×#EÑ#EÐ#QÜðQõ	
ð $RÐùò}
ùòR
÷*ñ ús   Â+=Q	Æ+=QÈ;QÑQÚargsc                 óD   — t        | j                  | j                  ¬«      S )N)Úconfig_fileÚpath_to_repo)ÚAddNewModelLikeCommandr–  r—  )r”  s    r8   Ú"add_new_model_like_command_factoryr™  Ú  s   € Ü!¨d×.>Ñ.>ÈT×M^ÑM^Ô_Ð_rA   c                   ó0   — e Zd Zedefd„«       Zdd„Zd„ Zy)r˜  Úparserc                 ó²   — | j                  d«      }|j                  dt        d¬«       |j                  dt        d¬«       |j                  t        ¬«       y )Nzadd-new-model-likez--config_filez8A file with all the information for this model creation.)ÚtypeÚhelpz--path_to_repozFWhen not using an editable install, the path to the Transformers repo.)Úfunc)Ú
add_parserÚadd_argumentr>   Úset_defaultsr™  )r›  Úadd_new_model_like_parsers     r8   Úregister_subcommandz*AddNewModelLikeCommand.register_subcommandß  s`   € à$*×$5Ñ$5Ð6JÓ$KÐ!Ø!×.Ñ.Ø¤#Ð,fð 	/ô 	
ð 	"×.Ñ.Ø¤3Ð-uð 	/ô 	
ð 	"×.Ñ.Ô4VÐ.ÕWrA   Nc                 óÈ  — |�žt        |dd¬«      5 }t        j                  |«      }d d d «       d   | _        t	        d
i |d   ¤Ž| _        |j                  dd«      | _        |j                  dt        «       «      | _	        |j                  d	d «      | _
        || _        y t        «       \  | _        | _        | _        | _	        | _
        | _        || _        y # 1 sw Y   Œ²xY w)Nrs   rt   ru   rq  r   r»   TrË   rx  r@   )rw   ÚjsonÚloadrq  r   r
  r  r»   rÑ   rË   rx  Úget_user_inputry  r—  )r5   r–  r—  r”  rz   Úconfigs         r8   Ú__init__zAddNewModelLikeCommand.__init__ê  sÞ   € ØÐ"Ü�k 3°Ô9ð &¸QÜŸ™ 1›�÷&à"(Ð)9Ñ":ˆDÔÜ"/Ñ"O°&Ð9MÑ2NÑ"OˆDÔØ#)§:¡:Ð.?ÀÓ#FˆDÔ Ø$Ÿj™j¨Ô7MÓ7OÓPˆDŒOØ"(§*¡*Ð-=¸tÓ"DˆDÔð )ˆÕô Ó ñØÔ#ØÔ#ØÔ$Ø”ØÔ#ØÔ0ð )ˆÕ÷#&ð &ús   ‘CÃC!c                 óø   — | j                   �!t        | j                   «      at        dz  dz  at	        | j
                  | j                  | j                  | j                  | j                  | j                  ¬«       y )NÚsrcr¡   )r   r   r»   rË   rx  ry  )r—  r   rë   ré   r“  rq  r
  r»   rË   rx  ry  )r5   s    r8   ÚrunzAddNewModelLikeCommand.runÿ  sl   € Ø×ÑÐ(ô
 ˜T×.Ñ.Ó/ˆIÜ )¨EÑ 1°NÑ BÐäØ×*Ñ*Ø#×2Ñ2Ø ×0Ñ0Ø—‘Ø×.Ñ.Ø(,×(HÑ(Hö	
rA   ©NN)r:   r;   r<   Ústaticmethodr   r¤  rª  r­  r@   rA   r8   r˜  r˜  Þ  s(   „ ØðX Nò Xó ðXó)ó*
rA   r˜  ÚquestionÚdefault_valueÚis_valid_answerÚ
convert_toÚfallback_messagec                 ó   — | j                  d«      s| dz   } |�| › d|› d�} d}|sHt        | «      }|�t        |«      dk(  r|}|�	 ||«      }n|�	  ||«      }d}nd}|st	        |«       |sŒHS # t        $ r d}Y Œw xY w)a¸  
    A utility function that asks a question to the user to get an answer, potentially looping until it gets a valid
    answer.

    Args:
        question (`str`): The question to ask the user.
        default_value (`str`, *optional*): A potential default value that will be used when the answer is empty.
        is_valid_answer (`Callable`, *optional*):
            If set, the question will be asked until this function returns `True` on the provided answer.
        convert_to (`Callable`, *optional*):
            If set, the answer will be passed to this function. If this function raises an error on the provided
            answer, the question will be asked again.
        fallback_message (`str`, *optional*):
            A message that will be displayed each time the question is asked again to the user.

    Returns:
        `Any`: The answer provided by the user (or the default), passed through the potential conversion function.
    r*   z [z] Fr   T)ra   ÚinputrE   Ú	Exceptionrj  )r°  r±  r²  r³  r´  Úvalid_answerÚanswers          r8   Úget_user_fieldrº    s¹   € ð2 ×Ñ˜SÔ!Ø˜c‘>ˆØÐ Ø�Z˜r - °Ð3ˆà€LÙÜ�x“ˆØÐ$¬¨V«¸Ò)9Ø"ˆFØÐ&Ù*¨6Ó2‰LØÐ#ð%Ù# FÓ+�Ø#‘ð  ˆLáÜÐ"Ô#ò! ð$ €Møô ò %Ø$’ð%ús   Á
A/ Á/A=Á<A=r–   c                 ój   — | j                  «       dv ry| j                  «       dv ryt        | › d�«      ‚)z&
    Converts a string to a bool.
    )Ú1ÚyÚyesÚtrueT)Ú0ÚnÚnoÚfalseFz0 is not a value that can be converted to a bool.)r/   r`   r•   s    r8   Úconvert_to_boolrÄ  F  s;   € ð 	‡w�wƒyÐ-Ñ-ØØ‡w�wƒyÐ-Ñ-ØÜ
˜�sÐJÐKÓ
LÐLrA   c                  óT  — t        t        j                  j                  j	                  «       «      } d}|sut        d«      }|| v rd}n`t        |› d�«       t        j                  || «      }t        |«      dk\  r.t        |«      dkD  rdj                  |«      }t        d|› d�«       |sŒut        «      }|d	   j                  }|d	   j                  }|d	   j                  }|d	   j                  }|d	   j                   }	|d
   }
d}t        |d	   j"                  «      dk(  rt%        d«      }t%        d«      }t'        ||«      }t%        d|j(                  ¬«      }t%        d|j*                  ¬«      }t%        d|j,                  ¬«      }t%        d|j.                  ¬«      }t%        d|› d�¬«      }t%        d«      }|||||	fD �cg c]  }|�t1        |t2        «      s|n|d   ‘Œ }}dj                  |«      }t%        d|› d|› d�t4        d¬«      }|r|}|}|}|	}|}d}nƒ|�t%        d|› d�¬«      }nd}|�t%        d|› d �¬«      }nd}|�t%        d!|› d"�¬«      }nd}|�t%        d#|› d$�¬«      }nd}|	�t%        d%|› d&�¬«      }nd}|�|€t%        d't4        d(d)¬*«      }nd}t'        ||||||||||||¬+«      }t%        d,t4        d(d)¬*«      }t%        d-|› d|
› d�t4        d(d)¬*«      }|rd} n1t%        d.d/„ ¬0«      } t        t7        | j9                  d1«      «      «      } |||| ||fS c c}w )2zE
    Ask the user for the necessary inputs to add the new model.
    FziWhat is the model you would like to duplicate? Please provide the lowercase `model_type` (e.g. roberta): Tr  r   z or zDid you mean ú?r
  rË   Nr   zVWe couldn't find the name of the base checkpoint for that model, please enter it here.zZWhat is the name (with no special casing) for your new model in the paper (e.g. RoBERTa)? zJWhat identifier would you like to use for the `model_type` of this model? )r±  zQWhat lowercase name would you like to use for the module (folder) of this model? zdWhat prefix (camel-cased) would you like to use for the model classes of this model (e.g. Roberta)? zZWhat prefix (upper-cased) would you like to use for the constants relative to this model? z:What will be the name of the config class for this model? r.   zsPlease give a checkpoint identifier (on the model Hub) for this new model (e.g. facebook/FacebookAI/roberta-base): r4  z5Will your new model use the same processing class as z (z) (yes/no)? z.Please answer yes/no, y/n, true/false or 1/0. )r³  r´  z=What will be the name of the tokenizer class for this model? rM  zCWhat will be the name of the image processor class for this model? rN  zHWhat will be the name of the fast image processor class for this model? rO  zEWhat will be the name of the feature extractor class for this model? rP  z=What will be the name of the processor class for this model? rQ  z­A fast image processor can be created from the slow one, but modifications might be needed. Should we add a fast image processor class for this model (recommended, yes/no)? r¾  z-Please answer yes/no, y/n, true/false or 1/0.)r³  r±  r´  )
r   r    r!   r"   r#   r$   r%   r&   r'   r(   zUShould we add # Copied from statements when creating the new modeling file (yes/no)? zOShould we add a version of your new model in all the frameworks implemented by zOPlease enter the list of frameworks you want (pt, tf, flax) separated by spacesc                 óD   — t        d„ | j                  d«      D «       «      S )Nc              3   ó$   K  — | ]  }|d v –— Œ
 y­w)r÷   Nr@   )Ú.0Úps     r8   ú	<genexpr>z3get_user_input.<locals>.<lambda>.<locals>.<genexpr>û  s   è ø€ Ò)ZÈ¨!Ð/CÔ*CÑ)Zùs   ‚r*   )Úallr1   r•   s    r8   r—   z get_user_input.<locals>.<lambda>û  s   € ¤cÑ)ZÈQÏWÉWÐUXË\Ô)ZÓ&Z€ rA   )r²  r*   )r2   rø   r  r  Úkeysr¶  rj  ÚdifflibÚget_close_matchesrE   r4   r  r$   r%   r&   r'   r(   r   rº  r   r   r    r!   r"   rl   r  rÄ  rˆ   r1   )!Úmodel_typesÚvalid_model_typerq  Únear_choicesÚold_model_infoÚold_tokenizer_classÚold_image_processor_classÚold_image_processor_fast_classÚold_feature_extractor_classÚold_processor_classÚold_frameworksrx  r   Údefault_patternsr   r    r!   r"   r#   r   r9  Úold_processing_classesr  r%   r&   r'   r(   r$   ry  r
  r»   Úall_frameworksrË   s!                                    r8   r¨  r¨  Q  sL  € ô ”{×5Ñ5×IÑI×NÑNÓPÓQ€Kð ÐÙÜØwó
ˆð ˜[Ñ(Ø#Ñä�^Ð$Ð$?Ð@ÔAÜ"×4Ñ4°^À[ÓQˆLÜ�<Ó  AÒ%Ü�|Ó$ qÒ(Ø#)§;¡;¨|Ó#<�LÜ˜ l ^°1Ð5Ô6ò ô -¨^Ó<€NØ(Ð)9Ñ:×JÑJÐØ .Ð/?Ñ @× VÑ VÐØ%3Ð4DÑ%E×%`Ñ%`Ð"Ø"0Ð1AÑ"B×"ZÑ"ZÐØ(Ð)9Ñ:×JÑJÐØ# LÑ1€Nà€NÜ
ˆ>Ð*Ñ+×6Ñ6Ó7¸1Ò<Ü'Ødó
ˆô  Ødó€Jô % Z°Ó<ÐäØTØ&×1Ñ1ô€Jô 'Ø[Ø&×8Ñ8ôÐô 'ØnØ&×8Ñ8ôÐô 'ØdØ&×8Ñ8ôÐô "ØDÐWhÐViÐioÐTpô€Lô  Ø}ó€Jð &Ø*Ø'ØØð
ö
àð ˆ=ô ˜AœuÔ%‰¨1¨Q©4Ñ/ð
Ðð 
ð "ŸY™YÐ'=Ó>ÐÜ$Ø
?ÀÐ?OÈrÐRhÐQiÐiuÐvÜ"ØIô€Oñ
 Ø 9ÐØ%CÐ"Ø"=ÐØ-ˆØ-ˆØ&+Ñ#àÐ*Ü,ØOØ!2Ð 3°9Ð=ô‰Oð
 #ˆOØ$Ð0Ü$2ØUØ!2Ð 3°>ÐBô%Ñ!ð
 %)Ð!Ø)Ð5Ü)7ØZØ!2Ð 3Ð3EÐFô*Ñ&ð
 *.Ð&Ø&Ð2Ü&4ØWØ!2Ð 3Ð3CÐDô'Ñ#ð
 '+Ð#ØÐ*Ü,ØOØ!2Ð 3°9Ð=ô‰Oð
 #ˆOØ$Ð0Ð5SÐ5[Ü*8ðdä*Ø#Ø!Pô+Ñ'ð +0Ð'ä"ØØØØ+Ø+Ø+Ø!Ø'Ø3Ø#=Ø 7Ø'ô€Nô %Ø_Ü"ØØHô	€Oô $ð	ØÐ˜2˜nÐ-¨\ð	;ä"ØØHô€Nñ Ø‰
ä#Ø]ÙZô
ˆ
ô œ#˜j×.Ñ.¨sÓ3Ó4Ó5ˆ
à˜N¨O¸ZÈÐYtÐuÐuùò]
s   Ç L%)r   )NNF)NTNrk   )NTr®  )TNNF)NNNN)[rÎ  r¦  r°   rI   Úargparser   r   Údataclassesr   Údatetimer   Ú	itertoolsr   Úpathlibr   Útypingr	   r
   r   r   r   r   r   r   rh  r×   r   rø   Úmodels.auto.configuration_autor   Úutilsr   r   r   r   r-   r   r   Ú
get_loggerr:   ÚloggerÚtodayÚyearr¾   Ú__file__rT  ré   rë   r   r†   r>   r‡  rG   ÚintrL   r[   rd   rp   ÚPathLiker}   r’   r‰   r¨   r¿   r&  rÀ   rÁ   r¹   rÉ   rÕ   rî   rñ   rô   rÑ   rý   r  r  r0  r;  r>  rC  rJ  rU  r]  rw  r“  r™  r˜  rº  rÄ  r¨  r@   rA   r8   ú<module>rì     sØ  ðó Û Û 	Û 	ß .Ý !Ý Ý Ý ß M× MÓ Mã å (Ý Fß SÓ SÝ (Ý >ð 
ˆ×	Ñ	˜HÓ	%€ð ˆt�z‰z‹|× Ñ €Ù˜“N×)Ñ)×0Ñ0Ð Ü×$Ñ$×+Ñ+€	ð ÷@Bð @Bó ð@BðH %Ø*Ø6Ø"@Ø:Ø*Ø Ø Ø.Ø-Ø.Ø ñÐ ð ,˜ð , ó ,ð#�cð #˜có #ð% #ð %¨$¨s©)ó %ñP/)˜3ð /)¨cð /)¸#ó /)ðj 04Ø04Øñ7 Ø
ð7 àð7 ð ˜˜c 7˜lÑ+Ñ,ð7 ð ˜˜s G˜|Ñ,Ñ-ð	7 ð
 ð7 ð 	ó7 ðz 04Ø04Øñ#Ø�S˜"Ÿ+™+Ð%Ñ&ð#àð#ð ˜˜c 7˜lÑ+Ñ,ð#ð ˜˜s G˜|Ñ,Ñ-ð	#ð
 ó#ðLE(Ø
ðE(Ø#0ðE(ØFSðE(à
ˆ3�ˆ8�_óE(òP"ðJ( e¨C°·±Ð,<Ñ&=ð (À#ó (ð$ *Ø'Ø-Ø"9Ø!:Ø-ñÐ ð �—‘ÐGÈrÏ|É|Ô\€ò%ðX  $Ø Ø!%ñDØ�s˜BŸK™KÐ'Ñ(ðDà%ðDð &ðDð ˜‰}ð	Dð
 ðDð ˜#‘YóDðP MQñYØ��c˜2Ÿ;™;Ð&Ñ'Ñ(ðYØ6>¸tÀC¹yÑ6IðYà	ˆ%��R—[‘[Ð Ñ
!Ñ"óYñB't ð 't°¸$¸s¹)Ñ1Dð 'tÐPTÐUXÐZ_Ð`dÐfjÐkoÑfpÐ`pÑZqÐUqÑPró 'tðT $˜Ÿ™Ð$KÐSU×S_ÑS_Ô`Ð ð RVñØðØ"*¨4°°U¸4ÀÀdÁÐ;KÑ5LÐ0LÑ+MÑ"NðàóòDð �B—J‘JÐ=Ó>Ð ñ& sð &¸ÀÀcÁÑ8Kð &ÐW[Ð\_ÐaeÐfiÑajÐ\jÑWkó &ñRK°H¸TÀ#¹YÑ4Gó Kð^ imñL&Ø�S˜"Ÿ+™+Ð%Ñ&ðL&Ø4<¸TÀ#¹YÑ4GðL&ØaeóL&ðd '+Ø ñ	W&Ø%ðW&à%ðW&ð ˜˜c™Ñ#ðW&ð ó	W&ðt,&¸ð ,&Ð[hó ,&ò`ð
 $\Ð"\Ø!YÐ ZØFÐGØIÐJØMÐNØKÐLñÐ ðKLØ%ðKLØ;HðKLØY]Ð^aÐcgÐhkÑclÐ^lÑYmóKLð\Ð ð. 48Ø&*ñc'Ø�C˜Ÿ™Ð$Ñ%ðc'à%ðc'ð &ðc'ð ˜˜c 2§;¡;Ð.Ñ/Ñ0ð	c'ð
 ˜˜c™Ñ#óc'òL/8ðj !Ø&*Ø$(Ø(-ñ{
Øð{
à%ð{
ð ð{
ð ˜˜c™Ñ#ð	{
ð
 ˜S‘Mð{
ð "&ó{
ð|`¨Yó `ô1
Ð7ô 1
ðl $(Ø*.Ø%)Ø&*ñ1Øð1à˜C‘=ð1ð ˜hÑ'ð1ð ˜Ñ"ð	1ð
 ˜s‘mð1ð 	ó1ðhM�sð M˜tó MónvrA   