Ë
    Q^(h@<  ã            
       ó†  — d dl Z d dlZd dlZd dlZd dlZd dlZd dlmZ d dlm	Z	m
Z
 d dlmZ d dlmZmZmZmZmZmZ ddlmZmZmZ  ej0                  e«      Zh d£Zh d	£Ze	 G d
„ d«      «       Zdeej<                  ef   deeef   fd„Z deeej<                  f   deeeeeee!f   f      ddfd„Z"deeej<                  f   deeej<                  f   ddfd„Z#dejH                  dedeeej<                  e!f   ddfd„Z%deeee!f   de!fd„Z&dedefd„Z'dedee   ddfd„Z(dejH                  dejR                  de*fd„Z+y) é    N)Úcontextmanager)Ú	dataclassÚfield)ÚPath)ÚAnyÚDictÚ	GeneratorÚIterableÚTupleÚUnioné   )ÚDDUFCorruptedFileErrorÚDDUFExportErrorÚDDUFInvalidEntryNameError>   ú.txtú.jsonú.modelú.safetensors>   úconfig.jsonúscheduler_config.jsonútokenizer_config.jsonúpreprocessor_config.jsonc                   ó€   — e Zd ZU dZeed<   eed<   eed<    ed¬«      Ze	ed<   e
deed	d	f   fd
„«       Zddedefd„Zy	)Ú	DDUFEntrya¿  Object representing a file entry in a DDUF file.

    See [`read_dduf_file`] for how to read a DDUF file.

    Attributes:
        filename (str):
            The name of the file in the DDUF archive.
        offset (int):
            The offset of the file in the DDUF archive.
        length (int):
            The length of the file in the DDUF archive.
        dduf_path (str):
            The path to the DDUF archive (for internal use).
    ÚfilenameÚlengthÚoffsetF)ÚreprÚ	dduf_pathÚreturnNc              #   óN  K  — | j                   j                  d«      5 }t        j                  |j                  «       dt        j                  ¬«      5 }|| j
                  | j
                  | j                  z    –— ddd«       ddd«       y# 1 sw Y   ŒxY w# 1 sw Y   yxY w­w)a-  Open the file as a memory-mapped file.

        Useful to load safetensors directly from the file.

        Example:
            ```py
            >>> import safetensors.torch
            >>> with entry.as_mmap() as mm:
            ...     tensors = safetensors.torch.load(mm)
            ```
        Úrbr   )r   ÚaccessN)r   ÚopenÚmmapÚfilenoÚACCESS_READr   r   )ÚselfÚfÚmms      úa/var/www/skyplay_api_hub/venv/lib/python3.12/site-packages/huggingface_hub/serialization/_dduf.pyÚas_mmapzDDUFEntry.as_mmap9   s�   è ø€ ð �^‰^× Ñ  Ó&ð 	B¨!Ü—‘˜1Ÿ8™8›:¨a¼×8HÑ8HÔIð BÈRØ˜Ÿ™ t§{¡{°T·[±[Ñ'@ÐAÒA÷B÷	Bð 	B÷Bð Bú÷	Bð 	Büs4   ‚B%ž5BÁ)BÁ<BÂ	B%ÂB	ÂBÂB"ÂB%Úencodingc                 óð   — | j                   j                  d«      5 }|j                  | j                  «       |j	                  | j
                  «      j                  |¬«      cddd«       S # 1 sw Y   yxY w)zÕRead the file as text.

        Useful for '.txt' and '.json' entries.

        Example:
            ```py
            >>> import json
            >>> index = json.loads(entry.read_text())
            ```
        r"   )r-   N)r   r$   Úseekr   Úreadr   Údecode)r(   r-   r)   s      r+   Ú	read_textzDDUFEntry.read_textJ   s_   € ð �^‰^× Ñ  Ó&ð 	A¨!Ø�F‰F�4—;‘;ÔØ—6‘6˜$Ÿ+™+Ó&×-Ñ-°xÐ-Ó@÷	A÷ 	Aò 	Aús   œAA,Á,A5)zutf-8)Ú__name__Ú
__module__Ú__qualname__Ú__doc__ÚstrÚ__annotations__Úintr   r   r   r   r	   Úbytesr,   r2   © ó    r+   r   r   "   se   … ñð ƒMØƒKØƒKá Ô'€IˆtÓ'àðB˜ 5¨$°Ð#4Ñ5ò Bó ðBñ A #ð A°Cô Ar<   r   r   r    c                 óX  — i }t        | «      } t        j                  d| › �«       t        j                  t        | «      d«      5 }|j                  «       D ]ž  }t        j                  d|j                  › �«       |j                  t        j                  k7  rt        d«      ‚	 t        |j                  «       t        ||«      }t        |j                  ||j                   | ¬«      ||j                  <   Œ  	 ddd«       d|vrt        d	«      ‚t#        j$                  |d   j'                  «       «      }t)        ||j+                  «       «       t        j                  d
| › dt-        |«      › d�«       |S # t        $ r}t        d|j                  › �«      |‚d}~ww xY w# 1 sw Y   Œ©xY w)a  
    Read a DDUF file and return a dictionary of entries.

    Only the metadata is read, the data is not loaded in memory.

    Args:
        dduf_path (`str` or `os.PathLike`):
            The path to the DDUF file to read.

    Returns:
        `Dict[str, DDUFEntry]`:
            A dictionary of [`DDUFEntry`] indexed by filename.

    Raises:
        - [`DDUFCorruptedFileError`]: If the DDUF file is corrupted (i.e. doesn't follow the DDUF format).

    Example:
        ```python
        >>> import json
        >>> import safetensors.torch
        >>> from huggingface_hub import read_dduf_file

        # Read DDUF metadata
        >>> dduf_entries = read_dduf_file("FLUX.1-dev.dduf")

        # Returns a mapping filename <> DDUFEntry
        >>> dduf_entries["model_index.json"]
        DDUFEntry(filename='model_index.json', offset=66, length=587)

        # Load model index as JSON
        >>> json.loads(dduf_entries["model_index.json"].read_text())
        {'_class_name': 'FluxPipeline', '_diffusers_version': '0.32.0.dev0', '_name_or_path': 'black-forest-labs/FLUX.1-dev', ...

        # Load VAE weights using safetensors
        >>> with dduf_entries["vae/diffusion_pytorch_model.safetensors"].as_mmap() as mm:
        ...     state_dict = safetensors.torch.load(mm)
        ```
    zReading DDUF file ÚrzReading entry z)Data must not be compressed in DDUF file.z!Invalid entry name in DDUF file: N)r   r   r   r   úmodel_index.jsonú7Missing required 'model_index.json' entry in DDUF file.zDone reading DDUF file z. Found z entries)r   ÚloggerÚinfoÚzipfileÚZipFiler7   ÚinfolistÚdebugr   Úcompress_typeÚ
ZIP_STOREDr   Ú_validate_dduf_entry_namer   Ú_get_data_offsetr   Ú	file_sizeÚjsonÚloadsr2   Ú_validate_dduf_structureÚkeysÚlen)r   ÚentriesÚzfrB   Úer   Úindexs          r+   Úread_dduf_filerU   Z   s{  € ðN €GÜ�Y“€IÜ
‡K�KÐ$ Y KÐ0Ô1Ü	�‰œ˜Y›¨Ó	-ð °Ø—K‘K“Mò 	ˆDÜ�L‰L˜>¨$¯-©-¨Ð9Ô:Ø×!Ñ!¤W×%7Ñ%7Ò7Ü,Ð-XÓYÐYðiÜ)¨$¯-©-Ô8ô & b¨$Ó/ˆFä%.ØŸ™¨v¸d¿n¹nÐXaô&ˆG�D—M‘MÒ"ñ	÷ð$  Ñ(Ü$Ð%^Ó_Ð_Ü�J‰J�wÐ1Ñ2×<Ñ<Ó>Ó?€EÜ˜U G§L¡L£NÔ3ä
‡K�KÐ)¨)¨°H¼SÀ»\¸NÈ(ÐSÔTØ€Nøô! -ò iÜ,Ð/PÐQU×Q^ÑQ^ÐP_Ð-`ÓaÐghÐhûðiú÷ð ús1   ÁAF Â$E6Â9>F Å6	FÅ?FÆFÆF Æ F)rQ   c                 ó$  — t         j                  d| › d�«       t        «       }d}t        j                  t        | «      dt        j                  «      5 }|D ]Œ  \  }}||v rt        d|› �«      ‚|j                  |«       |dk(  r-	 t        j                  t        |«      j                  «       «      }	 t        |«      }t         j!                  d	|› d
�«       t#        |||«       ŒŽ 	 ddd«       |€t        d«      ‚	 t%        ||«       t         j                  d| › �«       y# t        j                  $ r}t        d«      |‚d}~ww xY w# t        $ r}t        d|› �«      |‚d}~ww xY w# 1 sw Y   ŒƒxY w# t&        $ r}t        d«      |‚d}~ww xY w)a¯  Write a DDUF file from an iterable of entries.

    This is a lower-level helper than [`export_folder_as_dduf`] that allows more flexibility when serializing data.
    In particular, you don't need to save the data on disk before exporting it in the DDUF file.

    Args:
        dduf_path (`str` or `os.PathLike`):
            The path to the DDUF file to write.
        entries (`Iterable[Tuple[str, Union[str, Path, bytes]]]`):
            An iterable of entries to write in the DDUF file. Each entry is a tuple with the filename and the content.
            The filename should be the path to the file in the DDUF archive.
            The content can be a string or a pathlib.Path representing a path to a file on the local disk or directly the content as bytes.

    Raises:
        - [`DDUFExportError`]: If anything goes wrong during the export (e.g. invalid entry name, missing 'model_index.json', etc.).

    Example:
        ```python
        # Export specific files from the local disk.
        >>> from huggingface_hub import export_entries_as_dduf
        >>> export_entries_as_dduf(
        ...     dduf_path="stable-diffusion-v1-4-FP16.dduf",
        ...     entries=[ # List entries to add to the DDUF file (here, only FP16 weights)
        ...         ("model_index.json", "path/to/model_index.json"),
        ...         ("vae/config.json", "path/to/vae/config.json"),
        ...         ("vae/diffusion_pytorch_model.fp16.safetensors", "path/to/vae/diffusion_pytorch_model.fp16.safetensors"),
        ...         ("text_encoder/config.json", "path/to/text_encoder/config.json"),
        ...         ("text_encoder/model.fp16.safetensors", "path/to/text_encoder/model.fp16.safetensors"),
        ...         # ... add more entries here
        ...     ]
        ... )
        ```

        ```python
        # Export state_dicts one by one from a loaded pipeline
        >>> from diffusers import DiffusionPipeline
        >>> from typing import Generator, Tuple
        >>> import safetensors.torch
        >>> from huggingface_hub import export_entries_as_dduf
        >>> pipe = DiffusionPipeline.from_pretrained("CompVis/stable-diffusion-v1-4")
        ... # ... do some work with the pipeline

        >>> def as_entries(pipe: DiffusionPipeline) -> Generator[Tuple[str, bytes], None, None]:
        ...     # Build an generator that yields the entries to add to the DDUF file.
        ...     # The first element of the tuple is the filename in the DDUF archive (must use UNIX separator!). The second element is the content of the file.
        ...     # Entries will be evaluated lazily when the DDUF file is created (only 1 entry is loaded in memory at a time)
        ...     yield "vae/config.json", pipe.vae.to_json_string().encode()
        ...     yield "vae/diffusion_pytorch_model.safetensors", safetensors.torch.save(pipe.vae.state_dict())
        ...     yield "text_encoder/config.json", pipe.text_encoder.config.to_json_string().encode()
        ...     yield "text_encoder/model.safetensors", safetensors.torch.save(pipe.text_encoder.state_dict())
        ...     # ... add more entries here

        >>> export_entries_as_dduf(dduf_path="stable-diffusion-v1-4.dduf", entries=as_entries(pipe))
        ```
    zExporting DDUF file 'ú'NÚwzCan't add duplicate entry: r?   z#Failed to parse 'model_index.json'.zInvalid entry name: zAdding entry 'z' to DDUF filer@   zInvalid DDUF file structure.zDone writing DDUF file )rA   rB   ÚsetrC   rD   r7   rH   r   ÚaddrL   rM   Ú_load_contentr1   ÚJSONDecodeErrorrI   r   rF   Ú_dump_content_in_archiverN   r   )r   rQ   Ú	filenamesrT   Úarchiver   ÚcontentrS   s           r+   Úexport_entries_as_ddufra   Ÿ   s¢  € ôt ‡K�KÐ'¨	 {°!Ð4Ô5Ü“€IØ€EÜ	�‰œ˜Y›¨¬g×.@Ñ.@Ó	Að AÀWØ!(ò 	AÑˆH�gØ˜9Ñ$Ü%Ð(CÀHÀ:Ð&NÓOÐOØ�M‰M˜(Ô#àÐ-Ò-ðXÜ ŸJ™J¤}°WÓ'=×'DÑ'DÓ'FÓG�EðPÜ4°XÓ>�ô �L‰L˜>¨(¨°>ÐBÔCÜ$ W¨h¸Õ@ñ!	A÷Að( €}ÜÐWÓXÐXðEÜ  ¨	Ô2ô ‡K�KÐ)¨)¨Ð5Õ6øô% ×+Ñ+ò XÜ)Ð*OÓPÐVWÐWûðXûô
 -ò PÜ%Ð(<¸X¸JÐ&GÓHÈaÐOûðPú÷Að Aûô0 "ò EÜÐ<Ó=À1ÐDûðEúsf   Á1E)Â,D"Â3E	Â>(E)Ã=E5 Ä"EÄ5EÅEÅE)Å		E&ÅE!Å!E&Å&E)Å)E2Å5	FÅ>F
Æ
FÚfolder_pathc                 ó~   ‡— t        ‰«      Šdt        t        t        t         f      fˆfd„}t	        |  |«       «       y)a  
    Export a folder as a DDUF file.

    AUses [`export_entries_as_dduf`] under the hood.

    Args:
        dduf_path (`str` or `os.PathLike`):
            The path to the DDUF file to write.
        folder_path (`str` or `os.PathLike`):
            The path to the folder containing the diffusion model.

    Example:
        ```python
        >>> from huggingface_hub import export_folder_as_dduf
        >>> export_folder_as_dduf(dduf_path="FLUX.1-dev.dduf", folder_path="path/to/FLUX.1-dev")
        ```
    r    c               3   ót  •K  — t        ‰«      j                  d«      D ]–  } | j                  «       sŒ| j                  t        vrt
        j                  d| › d�«       Œ@| j                  ‰«      }t        |j                  «      dk\  rt
        j                  d| › d�«       Œƒ|j                  «       | f–— Œ˜ y ­w)Nz**/*zSkipping file 'z' (file type not allowed)é   z"' (nested directories not allowed))r   ÚglobÚis_fileÚsuffixÚDDUF_ALLOWED_ENTRIESrA   rF   Úrelative_torP   ÚpartsÚas_posix)ÚpathÚpath_in_archiverb   s     €r+   Ú_iterate_over_folderz3export_folder_as_dduf.<locals>._iterate_over_folder  s¨   øè ø€ Ü˜Ó%×*Ñ*¨6Ó2ò 
	3ˆDØ—<‘<”>ØØ�{‰{Ô"6Ñ6Ü—‘˜¨t¨fÐ4MÐNÔOØØ"×.Ñ.¨{Ó;ˆOÜ�?×(Ñ(Ó)¨QÒ.Ü—‘˜¨t¨fÐ4VÐWÔXØØ!×*Ñ*Ó,¨dÐ2Ó2ñ
	3ùs   ƒB5B8N)r   r
   r   r7   ra   )r   rb   ro   s    ` r+   Úexport_folder_as_ddufrp   ú   s9   ø€ ô$ �{Ó#€Kð3¤(¬5´´d°Ñ+;Ñ"<õ 3ô ˜9Ñ&:Ó&<Õ=r<   r_   r   r`   c                 óv  — | j                  |dd¬«      5 }t        |t        t        f«      r=t        |«      }|j                  d«      5 }t	        j
                  ||d«       d d d «       n1t        |t        «      r|j                  |«       nt        d|› d�«      ‚d d d «       y # 1 sw Y   ŒxY w# 1 sw Y   y xY w)NrX   T)Úforce_zip64r"   i  € zInvalid content type for z. Must be str, Path or bytes.)	r$   Ú
isinstancer7   r   ÚshutilÚcopyfileobjr:   Úwriter   )r_   r   r`   Ú
archive_fhÚcontent_pathÚ
content_fhs         r+   r]   r]     s»   € Ø	�‰�h °ˆÓ	6ð g¸*Ü�g¤¤T˜{Ô+Ü ›=ˆLØ×"Ñ" 4Ó(ð L¨JÜ×"Ñ" :¨z¸?ÔK÷Lð Lä˜¤Ô'Ø×Ñ˜WÕ%ä!Ð$=¸h¸ZÐGdÐ"eÓfÐf÷gð g÷Lð Lú÷gð gús#   •3B/ÁB#Á :B/Â#B,	Â(B/Â/B8c                 ó´   — t        | t        t        f«      rt        | «      j                  «       S t        | t        «      r| S t        dt        | «      › d�«      ‚)zoLoad the content of an entry as bytes.

    Used only for small checks (not to dump content into archive).
    z6Invalid content type. Must be str, Path or bytes. Got ú.)rs   r7   r   Ú
read_bytesr:   r   Útype)r`   s    r+   r[   r[   *  sR   € ô
 �'œC¤˜;Ô'Ü�G‹}×'Ñ'Ó)Ð)Ü	�GœUÔ	#ØˆäÐ VÔW[Ð\cÓWdÐVeÐefÐgÓhÐhr<   Ú
entry_namec                 óê   — d| j                  d«      d   z   t        vrt        d| › �«      ‚d| v rt        d| › d�«      ‚| j                  d«      } | j	                  d«      dkD  rt        d| › d�«      ‚| S )	Nr{   éÿÿÿÿzFile type not allowed: ú\z0Entry names must use UNIX separators ('/'). Got ú/é   z-DDUF only supports 1 level of directory. Got )Úsplitri   r   ÚstripÚcount)r~   s    r+   rI   rI   7  s”   € Ø
ˆZ×Ñ˜cÓ" 2Ñ&Ñ&Ô.BÑBÜ'Ð*AÀ*ÀÐ(NÓOÐOØˆzÑÜ'Ð*ZÐ[eÐZfÐfgÐ(hÓiÐiØ×!Ñ! #Ó&€JØ×Ñ˜Ó˜qÒ Ü'Ð*WÐXbÐWcÐcdÐ(eÓfÐfØÐr<   rT   Úentry_namesc                 ó@  ‡‡— t        | t        «      st        dt        | «      › d�«      ‚‰D �ch c]  }d|v sŒ|j	                  d«      d   ’Œ }}|D ]D  Š‰| vrt        d‰› d�«      ‚t        ˆˆfd„t        D «       «      rŒ0t        d‰› d	t        › d�«      ‚ y
c c}w )a�  
    Consistency checks on the DDUF file structure.

    Rules:
    - The 'model_index.json' entry is required and must contain a dictionary.
    - Each folder name must correspond to an entry in 'model_index.json'.
    - Each folder must contain at least a config file ('config.json', 'tokenizer_config.json', 'preprocessor_config.json', 'scheduler_config.json').

    Args:
        index (Any):
            The content of the 'model_index.json' entry.
        entry_names (Iterable[str]):
            The list of entry names in the DDUF file.

    Raises:
        - [`DDUFCorruptedFileError`]: If the DDUF file is corrupted (i.e. doesn't follow the DDUF format).
    z>Invalid 'model_index.json' content. Must be a dictionary. Got r{   r‚   r   zMissing required entry 'z' in 'model_index.json'.c              3   ó0   •K  — | ]  }‰› d |› �‰v –— Œ y­w)r‚   Nr;   )Ú.0Úrequired_entryr‡   Úfolders     €€r+   ú	<genexpr>z+_validate_dduf_structure.<locals>.<genexpr>[  s"   øè ø€ ÒrÀ>�f�X˜Q˜~Ð.Ð/°;Ô>Ñrùs   ƒz!Missing required file in folder 'z!'. Must contains at least one of N)rs   Údictr   r}   r„   ÚanyÚDDUF_FOLDER_REQUIRED_ENTRIES)rT   r‡   ÚentryÚdduf_foldersrŒ   s    `  @r+   rN   rN   B  s¹   ù€ ô$ �eœTÔ"Ü$Ð'eÔfjÐkpÓfqÐerÐrsÐ%tÓuÐuà5@ÖQ¨EÀCÈ5ÂL�E—K‘K Ó$ QÓ'ÐQ€LÐQØò ˆØ˜ÑÜ(Ð+CÀFÀ8ÐKcÐ)dÓeÐeÜÔrÔUqÔrÕrÜ(Ø3°F°8Ð;\Ô]yÐ\zÐz{Ð|óð ñ	ùò Rs
   ¯	B¹BrR   rB   c                 ód  — | j                   €t        d«      ‚|j                  }| j                   j                  |«       | j                   j	                  d«      }t        |«      dk  rt        d«      ‚t        j                  |dd d«      }t        j                  |dd d«      }|dz   |z   |z   }|S )a1  
    Calculate the data offset for a file in a ZIP archive.

    Args:
        zf (`zipfile.ZipFile`):
            The opened ZIP file. Must be opened in read mode.
        info (`zipfile.ZipInfo`):
            The file info.

    Returns:
        int: The offset of the file data in the ZIP archive.
    z+ZipFile object must be opened in read mode.é   zIncomplete local file header.é   é   Úlittle)Úfpr   Úheader_offsetr/   r0   rP   r9   Ú
from_bytes)rR   rB   r™   Úlocal_file_headerÚfilename_lenÚextra_field_lenÚdata_offsets          r+   rJ   rJ   a  s²   € ð 
‡u�u€}Ü$Ð%RÓSÐSð ×&Ñ&€Mð ‡E�E‡J�Jˆ}ÔØŸ™Ÿ
™
 2›Ðä
ÐÓ Ò"Ü$Ð%DÓEÐEô —>‘>Ð"3°B°rÐ":¸HÓE€LÜ—n‘nÐ%6°r¸"Ð%=¸xÓH€Oð   "Ñ$ |Ñ3°oÑE€KàÐr<   ),rL   Úloggingr%   Úosrt   rC   Ú
contextlibr   Údataclassesr   r   Úpathlibr   Útypingr   r   r	   r
   r   r   Úerrorsr   r   r   Ú	getLoggerr3   rA   ri   r�   r   ÚPathLiker7   rU   r:   ra   rp   rD   r]   r[   rI   rN   ÚZipInfor9   rJ   r;   r<   r+   ú<module>r©      sÙ  ðÛ Û Û Û 	Û Û Ý %ß (Ý ß ?× ?ç WÑ Wð 
ˆ×	Ñ	˜8Ó	$€òÐ ò Ð ð ÷4Að 4Aó ð4AðnB˜e B§K¡K°Ð$4Ñ5ð B¸$¸sÀI¸~Ñ:Nó BðJX7Ø�S˜"Ÿ+™+Ð%Ñ&ðX7Ø19¸%ÀÀUÈ3ÐPTÐV[ÐK[ÑE\Ð@\Ñ:]Ñ1^ðX7à	óX7ðv!> U¨3°·±Ð+;Ñ%<ð !>È5ÐQTÐVX×VaÑVaÐQaÑKbð !>Ðgkó !>ðH	g g§o¡oð 	gÀð 	gÈuÐUXÐZ\×ZeÑZeÐglÐUlÑOmð 	gÐrvó 	gð
i˜5  d¨EÐ!1Ñ2ð 
i°uó 
ið¨#ð °#ó ð Cð °h¸s±mð Èó ð>"˜Ÿ™ð "°·±ð "ÀCô "r<   