g^(hUdZddlZddlZddlZddlZddlZddlmZddlm Z m Z m Z m Z m Z ddlZddlZddlmZddlmZmZmZddlmZdd lmZdd lmZdd lmZmZmZm Z m!Z!dd l"m#Z#m$Z$m%Z% dd lm&Z&da(ejRZ*ejVZ,ga-e.e/e gdfe.e0fe1d<e2ejfddZ4e ee0e5dfZ6dZ7dZ8 ddlm9Z: e:jvsddlm?Z?GddZ@e@5ddlAZAddddZ7[: eaDeEejfdrejfjZFnedZFeEejfdrejfjZHnde5de5fdZHeEejfdrejfjZJnde5de5fdZJdZKeLe1d<ejfjZNeLe1d<d ZOe/ejfje1d!<deLfd"ZQdeLfd#ZRdeLfd$ZSdd%eLfd&ZTed'(de6fd)ZUdeLfd*ZVd+ZWd,e0fd-ZXd.ZYd/ZZd0Z[d1Z\e\eYe\eZGd2d3e]Z^ejfjZ_d4Z`d5Zad6ZbGd7d8ZcGd9d:edZed;e5ddfd<ZfGd=d>ZgGd?dZGd@dAeZhde6ddfdBZidde e6de0fdCZjdde e6de/e5e5ffdDZkdde e6deFfdEZlde6dFe6deLfdGZmGdHdIZndJe dKdenfdLZodMZpdJe%fdNZqde e.e5e.e0ffdOZrde5fdPZsde5fdQZtde e.e0fdRZude e.e0fdSZvdTe.e0dUe.e0de.e5fdVZwde5fdWZxde5fdXZyde e e5efde5fdYZzda{e e5e1dZ<de5fd[Z|de.e0fd\Z}de0fd]Z~de5fd^Zdde6ddfd_Zd`Zdde e6de%fdaZdde e6de%fdbZ ddce5de e6de%fddZdeZdfe e5e0fddfdgZde5fdhZdde e ee5ffdiZdde e ee5ffdjZde e e5efde5fdkZdde e ee5fde5fdlZdde e ee5fde5fdmZdde e ee5fde5fdnZdde e ee5fde5fdoZdde e ee5fde5fdpZdde e ee5fde5fdqZdde e ee5fde5fdrZdde e ee5fde5fdsZdde e ee5fde5fdtZdde e ee5fde5fduZdde e ee5fde5fdvZdde e ee5fde5fdwZde e5e0ej fdej fdxZdej dejfjfdyZ ddze5de e5e0ej fddfd{Zdde e5e0ej fde5fd|Zdd}ldd}led~ZGddZddlmZmZGddeZGddeZGddeZGddeZGddeZGddeZGddeZGddeZGddeZGddeZGddeZGddeZGddeZ[[ej`jceej`jceej`jceej`jceej`jceej`jceej`jceej`jceej`jceej`jceej`jceej`jceGddZdZe\eddlmZmZmZmZmZmZgdZy#e'$rdZ&Y"wxYw#1swYxYw#eB$rYwxYw#[:wxYw#e'$r ZCeCZ8YdZC[CdZC[CwwxYw)aM This package adds support for CUDA tensor types. It implements the same function as CPU tensors, but they utilize GPUs for computation. It is lazily initialized, so you can always import it, and use :func:`is_available()` to determine if your system supports CUDA. :ref:`cuda-semantics` has more details about working with CUDA. N) lru_cache)AnyCallablecastOptionalUniondevice) _dummy_type_LazySeedTracker classproperty)Device)gds)_get_device_index) CUDAGraphgraphgraph_pool_handleis_current_stream_capturingmake_graphed_callables)EventExternalStreamStream)_cudartF _queued_calls_cuda_isInBadForkcyNFrQ/var/www/skyplay_api_hub/venv/lib/python3.12/site-packages/torch/cuda/__init__.pyr"3sr )version)PathcleZdZd dZdeeedfdededejfdZ d dZ d ed ed eddfd Z y)_amdsmi_cdll_hookreturnNctj|_dg}tjdtjdx}r$tj j |dg|z}||_y)N libamd_smi.so ROCM_HOME ROCM_PATHzlib/libamd_smi.so)ctypesCDLL original_CDLLosgetenvpathjoinpaths)selfr3 rocm_homes r!__init__z_amdsmi_cdll_hook.__init__SsZ)/D&,-E$&IIk299[;Q$RRyR!#i9L!M NQV V,1DJr nameargskwargsc|rAt|jdk(r)|jD]} |j|g|i|cS|j|g|i|S#t$rY>wxYw)Nr))r$r7r3r.OSError)r4r7r8r9r1s r! hooked_CDLLz_amdsmi_cdll_hook.hooked_CDLLZs~T ? B$(JJ%D%'9t'9'9$'P'P'P P% .4--dDTDVDD$+% $%sA A&%A&c.|jt_yN)r<r,r-r4s r! __enter__z_amdsmi_cdll_hook.__enter__es"&"2"2FKr typevalue tracebackc.|jt_yr>)r.r,r-r4rArBrCs r!__exit__z_amdsmi_cdll_hook.__exit__hs"&"4"4FKr )r'N) __name__ __module__ __qualname__r6rstrr$rr,r-r<r@rFrr r!r&r&Rsf2 E %c4o 6 E?B ENQ E[[ E355S5S5T5r r&T_CudaDeviceProperties_cuda_exchangeDevicer r'c$|dkrytdNrz)PyTorch was compiled without CUDA support RuntimeErrorr s r!_exchange_devicerR A:FGGr _cuda_maybeExchangeDevicec$|dkrytdrNrPr s r!_maybe_exchange_devicerVrSr has_half has_magmardefault_generatorsc6ttjdS)z)Return true if compile with CUDA support._cuda_getDeviceCount)hasattrtorch_Crr r! _is_compiledr_s 5883 44r c2tjddk(S)NPYTORCH_NVML_BASED_CUDA_CHECK1)r/r0rr r!_nvml_based_availrcs 994 5 <~!! xx,,.22r including_emulationc8tjjrytsytjj }tjj}|-tjj |jdk\ry|syt|S)zQReturn a bool indicating if the current CUDA/ROCm device supports dtype bfloat16.TF) r]r#hiprfcudacurrent_deviceget_device_propertiesmajor_check_bf16_tensor_supported)rgr cuda_versions r!is_bf16_supportedrqsz }} > ZZ & & (F==%%LEJJ$D$DV$L$R$RVW$W  ( //r )maxsizecr tjdgtj|y#t$rYywxYw)Ng?)dtyper TF)r]tensorbfloat16 Exceptionr s r!roros2 cU%..@ s '* 66cPtjjrytdS)zMReturn a bool indicating if the current CUDA/ROCm device supports dtype tf32.F)rg)r]r#rjrqrr r!is_tf32_supportedrzs! }}  77r cBtjj|yr>)r]r^ _cuda_sleep)cycless r!_sleepr~s HH r arch_stringcb|jdd}|jd}t|S)z4Extracts the architecture string from a CUDA version_ra)split removesuffixint)rbases r!_extract_arch_versionrs1   S !! $D   S !D t9r c d}d}tjjtjj }t t D]}t|}|d}|d}t|}|dz|z}tdtjjDd} || ksatj|||||| dz| dzfzyy) Nz Found GPU%d %s which requires CUDA_VERSION >= %d to work properly, but your PyTorch was compiled with CUDA_VERSION %d. Please install the correct PyTorch binary using instructions from https://pytorch.org z Found GPU%d %s which is of cuda capability %d.%d. PyTorch no longer supports this GPU because it is too old. The minimum cuda capability supported by this library is %d.%d. rr c32K|]}t|ywr>)r).0archs r! z$_check_capability..sT&t,Ts#)default) r]r#rkr^_cuda_getCompiledVersionrangereget_device_capabilityget_device_namemin get_arch_listwarningswarn) incorrect_binary_warn old_gpu_warn CUDA_VERSIONd capabilityrnminorr7 current_archmin_archs r!_check_capabilityrsL  }}%xx88: |~& A.q1JqMEqME"1%D 2:-LT9Q9Q9STHh&  $uh"nhmLM &r c  d}tjjyt}t |dk(ry|Dcgc]}d|vst |}}t tD]p}t|\ }t fd|D}|r(t|} dz|z}tj|j||dj||rycc}w)Na  {} with CUDA capability sm_{} is not compatible with the current PyTorch installation. The current PyTorch install supports CUDA capabilities {}. If you want to use the {} GPU with PyTorch, please check the instructions at https://pytorch.org/get-started/locally/ rsm_c3.K|] }|dzk(yw)rNr)rsm cap_majors r!rz _check_cubins..sE"bI-Esr )r]r#rkrlenrrreranyrrrformatr2) incompatible_device_warn arch_listr supported_smidx cap_minor supported device_namerrs @r! _check_cubinsr s   }}!I 9~r)rcallss r!rz_lazy_init..zsXuRWUXsz6CUDA call failed lazily at initialization with error: z( CUDA call was originally invoked at: )rr\_tlsrrrQr]r^AssertionErrorrr/environ _cuda_initrrextendr get_callsrxrJr2rdelattrr) queued_callorig_tracebackemsgs r!rrVs74):; .   ..  I uxx!78 !GH H ? Y  ! 206BJJ, -  $X0B0L0L0NXX -/< <+ ^<M < D+ , ]..L!<PQTUVQWPXYCCE77>CZB[]04!; < D+ ,[..sM FCF? E. D1E.F1 E+:,E&&E++E..FFF c"ttS)a Retrieves the CUDA runtime API module. This function initializes the CUDA runtime environment if it is not already initialized and returns the CUDA runtime API module (_cudart). The CUDA runtime API module provides access to various CUDA runtime functions. Args: ``None`` Returns: module: The CUDA runtime API module (_cudart). Raises: RuntimeError: If CUDA cannot be re-initialized in a forked subprocess. AssertionError: If PyTorch is not compiled with CUDA support or if libcudart functions are unavailable. Example of CUDA operations with profiling: >>> import torch >>> from torch.cuda import cudart, check_error >>> import os >>> >>> os.environ['CUDA_PROFILE'] = '1' >>> >>> def perform_cuda_operations_with_streams(): >>> stream = torch.cuda.Stream() >>> with torch.cuda.stream(stream): >>> x = torch.randn(100, 100, device='cuda') >>> y = torch.randn(100, 100, device='cuda') >>> z = torch.mul(x, y) >>> return z >>> >>> torch.cuda.synchronize() >>> print("====== Start nsys profiling ======") >>> check_error(cudart().cudaProfilerStart()) >>> with torch.autograd.profiler.emit_nvtx(): >>> result = perform_cuda_operations_with_streams() >>> print("CUDA operations completed.") >>> check_error(torch.cuda.cudart().cudaProfilerStop()) >>> print("====== End nsys profiling ======") To run this example and save the profiling information, execute: >>> $ nvprof --profile-from-start off --csv --print-summary -o trace_name.prof -f -- python cudart_test.py This command profiles the CUDA operations in the provided script and saves the profiling information to a file named `trace_name.prof`. The `--profile-from-start off` option ensures that profiling starts only after the `cudaProfilerStart` call in the script. The `--csv` and `--print-summary` options format the profiling output as a CSV file and print a summary, respectively. The `-o` option specifies the output file name, and the `-f` option forces the overwrite of the output file if it already exists. )rrrr r!cudartrslL Nr c*eZdZUdZeed<dZeed<y) cudaStatusrSUCCESS"ERROR_NOT_READYN)rGrHrIrr__annotations__rrr r!rrsGSOSr rc(eZdZdeddffd ZxZS) CudaErrorcoder'Nctjtj|}t||d|dy)Nz ())rcudaGetErrorString cudaErrorsuperr6)r4rr __class__s r!r6zCudaError.__init__s8(():):4)@A C54&*+r )rGrHrIrr6 __classcell__rs@r!rrs,S,T,,r rrescT|tjjk7r t|yr>)rrsuccessr)rs r! check_errorrs% g'''n(r c2eZdZdefdZdZdededefdZy) _DeviceGuardindexc ||_d|_yNrO)rprev_idx)r4rs r!r6z_DeviceGuard.__init__s r c`tjj|j|_yr>r]rkrRrrr?s r!r@z_DeviceGuard.__enter__ 33DHH= r rArBrCc`tjj|j|_yrr]rkrVrrrEs r!rFz_DeviceGuard.__exit__::44T]]Cr N)rGrHrIrr6r@rrFrr r!rrs-c>Sr rc6eZdZdZdefdZdZdededefdZy) r zContext-manager that changes the selected device. Args: device (torch.device or int): device index to select. It's a no-op if this argument is a negative integer or ``None``. c6t|d|_d|_y)NToptionalrO)rrr)r4r s r!r6zdevice.__init__s$Vd; r c`tjj|j|_yr>rr?s r!r@zdevice.__enter__rr rArBrCc`tjj|j|_yrrrEs r!rFzdevice.__exit__rr N)rGrHrI__doc__rr6r@rFrr r!r r s2s>Sr c"eZdZdZfdZxZS) device_ofaContext-manager that changes the current device to that of given object. You can use both tensors and storages as arguments. If a given object is not allocated on a GPU, this is a no-op. Args: obj (Tensor or Storage): object allocated on the selected device. c`|jr|jnd}t| |yr)is_cuda get_devicerr6)r4objrrs r!r6zdevice_of.__init__s$"%++cnn2 r )rGrHrIrr6rrs@r!rrsr rcdt|}|dk\r tjj|yy)a=Set the current device. Usage of this function is discouraged in favor of :any:`device`. In most cases it's better to use ``CUDA_VISIBLE_DEVICES`` environmental variable. Args: device (torch.device or int): selected device. This function is a no-op if this argument is negative. rN)rr]r^_cuda_setDevicer s r! set_devicer s,v &F {   (r c,t|jS)aGet the name of a device. Args: device (torch.device or int or str, optional): device for which to return the name. This function is a no-op if this argument is a negative integer. It uses the current device, given by :func:`~torch.cuda.current_device`, if :attr:`device` is ``None`` (default). Returns: str: the name of the device )rmr7r s r!rrs ! ( - --r cHt|}|j|jfS)aGet the cuda capability of a device. Args: device (torch.device or int or str, optional): device for which to return the device capability. This function is a no-op if this argument is a negative integer. It uses the current device, given by :func:`~torch.cuda.current_device`, if :attr:`device` is ``None`` (default). Returns: tuple(int, int): the major and minor cuda capability of the device )rmrnr)r props r!rr#s! ! (D ::tzz !!r ctt|d}|dks |tk\r tdt |S)aGet the properties of a device. Args: device (torch.device or int or str, optional): device for which to return the properties of the device. It uses the current device, given by :func:`~torch.cuda.current_device`, if :attr:`device` is ``None`` (default). Returns: _CudaDeviceProperties: the properties of the device TrrInvalid device id)rrrer_get_device_propertiesr s r!rmrm4s<L v 5F zV|~-011 !& ))r peer_devicectt|d}t|}|dks |tk\r td|dks |tk\r tdtj j ||S)z5Check if peer access between two devices is possible.TrrrzInvalid peer device id)rrrerr]r^_cuda_canDeviceAccessPeer)r rs r!can_device_access_peerrGslL v 5F#K0K zV|~-011Q+7566 88 - -fk BBr cNeZdZUdZeded<dedfdZdZdeded efd Z y ) StreamContextaContext-manager that selects a given stream. All CUDA kernels queued within its context will be enqueued on a selected stream. Args: Stream (Stream): selected stream. This manager is a no-op if it's ``None``. .. note:: Streams are per-device. torch.cuda.Stream cur_streamstreamc||_tdd|_tjj s|jd|_tjj sdntj jd|_tjj sd|_ ytj jd|_ y)NTrO) rrrr]jit is_scriptingrkdefault_streamsrc_prev_streamdst_prev_stream)r4rs r!r6zStreamContext.__init__as $T40yy%%'xx ..0Dejj6O6OPT6U  ..0D 6;jj6O6OPT6U r c|j}||jdk(rytjj d|_|j j |j k7rLt |j 5tjj |j |_dddtjj|y#1swY)xYwr) rrr]rkcurrent_streamrr r  set_stream)r4rs r!r@zStreamContext.__enter__os[[  R $zz88>    & &**;*; ; ))* T',zz'@'@ARAR'S$ T j) T Ts ;/CCrArBrCc*|j}||jdk(ry|jj|jk7r)tj j |jtj j |jyr)rrrr r]rkr#r )r4rArBrCrs r!rFzStreamContext.__exit__~sj[[  R     & &**;*; ; JJ ! !$"6"6 7 d223r N) rGrHrIrrrr6r@rrFrr r!rrSsF ,--  x(;<   * 4S 4 4 4r rrrct|S)aZWrap around the Context-manager StreamContext that selects a given stream. Arguments: stream (Stream): selected stream. This manager is a no-op if it's ``None``. .. note:: In eager mode stream is of type Stream class while in JIT it is an object of the custom class ``torch.classes.cuda.Stream``. )rrs r!rrs   r cHtjj|||y)zset stream specified by the stream id, device index and device type Args: stream_id (int): stream id in stream pool device_index (int): device index in topo device_type (int): enum device type  stream_id device_index device_typeN)r]r^_cuda_setStreamr(s r!_set_stream_by_idr-s$ HH!r cb|yt|j|j|jy)aSet the current stream.This is a wrapper API to set the stream. Usage of this function is discouraged in favor of the ``stream`` context manager. Args: stream (Stream): selected stream. This function is a no-op if this argument is ``None``. Nr()r-r)r*r+r&s r!r#r#s/~""((&&r c0tjd}tjjrtjd}tjd}|[t |j d}|+t |j d|kDr td|}ntt|S||}|ttdSdtdtfd }d td tdttfd }|jd r ||d S|jdr ||dSg}|j dD]N}||j}||vrtttgcS|dkr|S|j|P|S)z0Parse CUDA_VISIBLE_DEVICES environment variable.CUDA_VISIBLE_DEVICESHIP_VISIBLE_DEVICESROCR_VISIBLE_DEVICES,zCHIP_VISIBLE_DEVICES contains more devices than ROCR_VISIBLE_DEVICES@sr'c|syt|D]7\}}|js |dk(r|dvsn|dzt|k(s3|dz }9dkDrt|d|SdS)z:Return -1 or positive integer sequence string starts with.rOrz+-rN) enumerateisdigitrr)r5rcs r!_strtoulz(_parse_visible_devices.._strtoulsml FCIIKC1HdQw#a& q   #Qws1Tc7|.B.r lstprefixcg}|jdD]D}||vrtttgcS|j |s|S|j |F|S)Nr3)rrlistrJ startswithr)r;r<rcselems r!parse_list_with_prefixz6_parse_visible_devices..parse_list_with_prefixs_IIcN Ds{DIr**??6*  JJt   r GPU-MIG-r)r/r0r]r#rjrrrQr>rrJrr?striprr) var hip_devices rocr_devices rocr_countr:rBrcrAxs r!_parse_visible_devicesrLs ))* +C }}ii 56 yy!78  #\//45J&{((-.;&]"E*-..  $C {E"I /C /C / C  c  ~~f%c622 ~~f%c622B # TZZ\ " 7S 2& & q5  I !  Ir ctsy tjtj}t|S#tj$r,}t j d|j Yd}~yd}~wwxYw)NrOz&Can't initialize amdsmi - Error code: ) _HAS_PYNVMLamdsmi amdsmi_initAmdSmiExceptionrrerr_codeamdsmi_get_processor_handlesr)rsocket_handless r!_raw_device_count_amdsmirUsg 88:N ~   ! ! >qzzlKLs<A;"A66A;c ddlm}m}m}|d}|j }|dk7rt j dy|d}|j||}|dk7rt j dy~|jS)zgReturn number of devices as reported by NVML or negative value if NVML discovery/initialization failed.r)byrefc_intr-libnvidia-ml.so.1Can't initialize NVMLrOCan't get nvml device count) r,rWrXr-nvmlInitrrnvmlDeviceGetCount_v2rB)rWrXr-nvml_hrJ dev_counts r!_raw_device_count_nvmlr`sx)) % &F  B Qw -.b I  % %eI&6 7B Qw 34 ??r cddlm}m}m}m}m}t sy tj tj}t|}g}t|D]^} tj|} tj| ddd} |j!t#| j%`|S#tj$rtjdYywxYw#tj$rtjdYywxYw#tj$rtjdYywxYw#tj$rtjdYywxYw) NrrWrXc_void_pr-create_string_bufferzCan't initialize amdsmizCan't get amdsmi device countzCannot get amd device handler asic_serialzCannot get uuid for amd device)r,rWrXrcr-rdrNrOrPrQrrrSrramdsmi_get_gpu_asic_inforrJlower) rWrXrcr-rdrTr_uuidsrhandleruuids r!_raw_device_uuid_amdsmirl sOII <<>' EY  99;C@G 227;MJD  IOO   L5  ! ! /0  ! ! 56%%  MM9 : %%  MM: ; sFB;C)D4E;(C&%C&)(DD(EE(E21E2cddlm}m}m}m}m}|d}|j }|dk7rtjdy|d}|j||}|dk7rtjdyg}t|jD]} |} |j| || }|dk7rtjdyd } || } |j| | | }|dk7rtjd y|j| jj!d j#d ~|S) z^Return list of device UUID as reported by NVML or None if NVM discovery/initialization failed.rrbrYrZNrOr[zCan't get device handle`zCan't get device UUIDascii)r,rWrXrcr-rdr\rrr]rrBnvmlDeviceGetHandleByIndex_v2nvmlDeviceGetUUIDrrawdecoderE) rWrXrcr-rdr^rJr_rirdev_idbuf_lenbufs r!_raw_device_uuid_nvmlrxDs$II % &F  B Qw -.b I  % %eI&6 7B Qw 34EY__% :  1 1#uV} E 7 MM3 4"7+  % %fc7 ; 7 MM1 2 SWW^^G,22489 :  Lr candidatesric*dtdttdtfd}g}|D]n}tjj r|j ddd}|||}|dkr|S||vrtttgcS|j|p|S) zqGiven the set of partial uuids and list of known uuids builds a set of ordinals excluding ambiguous partials IDs. candidaterir'cfd}t|D] \}}|j|s|dk7ry|}"|Sr)r7r?)r{ri best_matchrrks r!uuid_to_ordinalz4_transform_uuid_to_ordinals..uuid_to_ordinalgsF "5) IC??9-RJ  r rCrrr) rJr>rr]r#rjreplacerr)ryrir~rJr{rs r!_transform_uuid_to_ordinalsrds 3 tCy S B  ==  !))AIi/ 7  I "9S 2& & #  Ir cft}|sy t|dtur1t}|yt t t|}t ||}n;t}|dkr|St|D]\}}t t||k\s|cSt|S#t$rYyt$rYywxYw)NrrO) rLrArJrlrr>rrUr7rr;AttributeErrorr)visible_devicesrivisible_device_strraw_cntrvals r!_device_count_amdsmirs,.O  " #s *+-E}!%d3i!A 9:LeTO.0G!|%o6 SS>W,J     s. B4B%$B BB B0%B0/B0ct}|sy t|dturD|djdryt }|yt t tt||}n;t}|dkr|St|D]\}}t t||k\s|cSt|S#t$rYyt$rYywxYw)zReturn number of devices as reported by NVML taking CUDA_VISIBLE_DEVICES into account. Negative value is returned if NVML discovery or initialization has failed. rrDrO)rLrArJr?rxrrr>r`r7rr;rr)rrirrrs r!_device_count_nvmlrs -.O  " #s *q!,,V4)+E}9T#Y0%O-.G!|%o6 SS>W,J     s4(B- B-2B-8$B-B-!B-- C8CCcJt|d}t}t|dtur8t }| t dt ttt||}ttt|}|dks|t|k\rt d|d|d||S)zNReturn the NVML index of the device, taking CUDA_VISIBLE_DEVICES into account.TrrCan't get device UUIDsdevice z& is not visible (CUDA_VISIBLE_DEVICES=r) rrLrArJrxrQrrr>rr)r rrris r!_get_nvml_device_indexrs FT 2C,.O OA 3&%' =78 85 cO ,e 49o6O Qw#_--cU@@QQR S   3 r _cached_device_countctsyttStjjr t n t }|dkrtjjn|}tr|a|S)z$Return the number of GPUs available.r) r_rr]r#rjrrr^r[r) nvml_countrs r!reres] >'##+0==+<+<%'BTBVJ+5>%%'zA  Hr c~tsgStjj}|gS|j S)z=Return list CUDA architectures this library was compiled for.)rfr]r^_cuda_getArchFlagsr) arch_flagss r!rrs8 > ,,.J    r c t}t|dk(ry|Dcgc]}|jd}}dj|Dcgc]\}}d|d|d|c}}Scc}wcc}}w)z9Return NVCC gencode flags this library was compiled with.rrrrz-gencode compute=compute_z,code=)rrrr2)rr arch_list_kinds r!get_gencode_flagsrs}I 9~.78d$**S/8J8 88!+ t(vVD64& A  9 s A%A* cRttjjS)z0Return the index of a currently selected device.)rr]r^_cuda_getDevicerr r!rlrlsL 88 # # %%r cttjj|5tjj cdddS#1swYyxYw)a,Wait for all kernels in all streams on a CUDA device to complete. Args: device (torch.device or int, optional): device for which to synchronize. It uses the current device, given by :func:`~torch.cuda.current_device`, if :attr:`device` is ``None`` (default). N)rr]rkr r^_cuda_synchronizer s r! synchronizers@L   6 ",xx))+,,,s AAcRttjjS)axForce collects GPU memory after it has been released by CUDA IPC. .. note:: Checks if any sent CUDA tensors could be cleaned from the memory. Force closes shared memory file used for reference counting if there is no active counters. Useful when the producer process stopped actively sending tensors and want to release unused memory. )rr]r^_cuda_ipc_collectrr r! ipc_collectrsL 88 % % ''r cttjjt |d}t |d|d|dS)aSReturn the currently selected :class:`Stream` for a given device. Args: device (torch.device or int, optional): selected device. Returns the currently selected :class:`Stream` for the current device, given by :func:`~torch.cuda.current_device`, if :attr:`device` is ``None`` (default). Trrrrfr()rr]r^_cuda_getCurrentStreamrrr streamdatas r!r"r" JL00&40J Q-jmTU r cttjjt |d}t |d|d|dS)a=Return the default :class:`Stream` for a given device. Args: device (torch.device or int, optional): selected device. Returns the default :class:`Stream` for the current device, given by :func:`~torch.cuda.current_device`, if :attr:`device` is ``None`` (default). Trrrrfr()rr]r^_cuda_getDefaultStreamrrrs r!rr2rr data_ptrcttjj|t |d}t |d|d|dS)aReturn a :class:`Stream` from an externally allocated CUDA stream. This function is used to wrap streams allocated in other libraries in order to facilitate data exchange and multi-library interactions. .. note:: This function doesn't manage the stream life-cycle, it is the user responsibility to keep the referenced stream alive while this returned stream is being used. Args: data_ptr(int): Integer representation of the `cudaStream_t` value that is allocated externally. device(torch.device or int, optional): the device where the stream was originally allocated. If device is specified incorrectly, subsequent launches using this stream may fail. Trrrrfr()rr]r^_cuda_getStreamFromExternalrr)rr rs r!get_stream_from_externalrDsM&L55#FT:J Q-jmTU r cRttjjS)z6Return cublasHandle_t pointer to current cuBLAS handle)rr]r^_cuda_getCurrentBlasHandlerr r!current_blas_handler`sL 88 . . 00r debug_modectt|tr#|dk(rd}n|dk(rd}n|dk(rd}n tdtj j |y) aSet the debug mode for cuda synchronizing operations. Args: debug_mode(str or int): if "default" or 0, don't error or warn on synchronizing operations, if "warn" or 1, warn on synchronizing operations, if "error" or 2, error out synchronizing operations. Warning: This is an experimental feature, and not all synchronizing operations will trigger warning or error. In particular, operations in torch.distributed and torch.sparse namespaces are not covered yet. rrrrerrorrfzGinvalid value of debug_mode, expected one of `default`, `warn`, `error`N)r isinstancerJrQr]r^_cuda_set_sync_debug_mode)rs r!set_sync_debug_moderfs^L*c"  "J 6 !J 7 "JY  HH&&z2r cRttjjS)zEReturn current value of debug mode for cuda synchronizing operations.)rr]r^_cuda_get_sync_debug_moderr r!get_sync_debug_modersL 88 - - //r ctstdtddlm} tj t|}tj|}|S#|$r}t d|d}~wwxYw)Nz=pynvml does not seem to be installed or it can't be imported.r)NVMLError_DriverNotLoadedz-cuda driver can't be loaded, is cuda enabled?) rNModuleNotFoundError _PYNVML_ERRpynvmlrr\rQrnvmlDeviceGetHandleByIndex)r rrhandles r!_get_pynvml_handlerrsq ! K  1S$F +F  . .v 6F M %SJKQRRSsAA* A%%A*ctstdt tjt|}tj|}|S#tj $r}t d|d}~wwxYw)Nz=amdsmi does not seem to be installed or it can't be imported.z>amdsmi driver can't be loaded, requires >=ROCm5.6 installation) rNrrrOrPrQrQ_get_amdsmi_device_indexrS)r rrs r!_get_amdsmi_handlerrsx ! K   &f -F  0 0 26 :F M  ! ! L  sAA4# A//A4cTt|d}t}t|dtur:t }| t dt tt|}t||}ttt tt|}||vrt d|d|d||S)zKReturn the amdsmi index of the device, taking visible_devices into account.Trrrrz% is not visible (HIP_VISIBLE_DEVICES=r) rrLrArJrlrQrr>rdictr7r)r rrrivisible_devices_stridx_maps r!rrs FT 2C,.O OA 3&') =78 8" I 66I5Q9T$s)_=>?G 'cU??PPQ R   3<r cpt}t|}tj|d}|dzdz}|S)N vram_usedi)rrrOamdsmi_get_gpu_vram_usage)r rmem_mega_bytes mem_bytess r!_get_amdsmi_device_memory_usedrs> "F %f -F55f=kJN%,I r ct}t|}tj|}tj|dS)N umc_activityrrrOrSamdsmi_get_gpu_activityr rs r!_get_amdsmi_memory_usager= "F %f -F  0 0 26 :F  ) )& 1. AAr ct}t|}tj|}tj|dS)N gfx_activityrrs r!_get_amdsmi_utilizationrrr ct|}tj|tjjtj j Sr>)rrOamdsmi_get_temp_metricAmdSmiTemperatureTypeJUNCTIONAmdSmiTemperatureMetricCURRENTrs r!_get_amdsmi_temperaturers@  (F  ( ($$--&&.. r ct|}tj|d}|dk7r|Stj|dS)Naverage_socket_powerzN/Acurrent_socket_power)rrOamdsmi_get_power_info)r r socket_powers r!_get_amdsmi_power_drawrsG  (F//78NOLu++F34JKKr ct|}tj|tjj}d|vr|dS|dS)Ncur_clkclk)rrOamdsmi_get_clock_info AmdSmiClkTypeGFX)r r clock_infos r!_get_amdsmi_clock_ratersH  (F--ff6J6J6N6NOJJ)$$%  r ctjjsIt}t |}t j |}t j|jSt|S)a<Return used global (device) memory in bytes as given by `nvidia-smi` or `amd-smi`. Args: device (torch.device or int, optional): selected device. Returns statistic for the current device, given by :func:`~torch.cuda.current_device`, if :attr:`device` is ``None`` (default). ) r]r#rjrrrrnvmlDeviceGetMemoryInfousedrrs r!device_memory_usedrsU ==  $&'/226:--f5:::-f55r ctjjsIt}t |}t j |}t j|jSt|S)aReturn the percent of time over the past sample period during which global (device) memory was being read or written as given by `nvidia-smi`. Args: device (torch.device or int, optional): selected device. Returns statistic for the current device, given by :func:`~torch.cuda.current_device`, if :attr:`device` is ``None`` (default). Warning: Each sample period may be between 1 second and 1/6 second, depending on the product being queried. ) r]r#rjrrrrnvmlDeviceGetUtilizationRatesmemoryrrs r! memory_usagersU ==  $&'/226:33F;BBB'//r ctjjsJt|}t |}t j |}t j|jSt|S)aReturn the percent of time over the past sample period during which one or more kernels was executing on the GPU as given by `nvidia-smi`. Args: device (torch.device or int, optional): selected device. Returns statistic for the current device, given by :func:`~torch.cuda.current_device`, if :attr:`device` is ``None`` (default). Warning: Each sample period may be between 1 second and 1/6 second, depending on the product being queried. ) r]r#rjrrrrrgpurrs r! utilizationrsW ==  $V,'/226:33F;???&v..r ctjjs!t|}t j |dSt |S)a Return the average temperature of the GPU sensor in Degrees C (Centigrades). The average temperature is computed based on past sample period as given by `nvidia-smi`. Args: device (torch.device or int, optional): selected device. Returns statistic for the current device, given by :func:`~torch.cuda.current_device`, if :attr:`device` is ``None`` (default). Warning: Each sample period may be between 1 second and 1/6 second, depending on the product being queried. r)r]r#rjrrnvmlDeviceGetTemperaturerrs r! temperaturer*s9 ==  $V,..vq99&v..r ctjjs t|}t j |St |S)a Return the average power draw of the GPU sensor in mW (MilliWatts) over the past sample period as given by `nvidia-smi` for Fermi or newer fully supported devices. Args: device (torch.device or int, optional): selected device. Returns statistic for the current device, given by :func:`~torch.cuda.current_device`, if :attr:`device` is ``None`` (default). Warning: Each sample period may be between 1 second and 1/6 second, depending on the product being queried. )r]r#rjrrnvmlDeviceGetPowerUsagerrs r! power_drawr?s7 ==  $V,--f55%f--r ctjjs!t|}t j |dSt |S)aReturn the clock speed of the GPU SM in MHz (megahertz) over the past sample period as given by `nvidia-smi`. Args: device (torch.device or int, optional): selected device. Returns statistic for the current device, given by :func:`~torch.cuda.current_device`, if :attr:`device` is ``None`` (default). Warning: Each sample period may be between 1 second and 1/6 second, depending on the product being queried. r)r]r#rjrrnvmlDeviceGetClockInforrs r! clock_raterRs9 ==  $V,,,VQ77%f--r ct|trtj|}|St|trtjd|}|S)zReturn the torch.device type object from the passed in device. Args: device (torch.device or int): selected device. rk)rrJr]r rr s r! _get_devicerdsD &#f% M FC ff- Mr cl|j}| t}tjj|S)zvReturn the CUDA Generator object for the given device. Args: device (torch.device): selected device. )rrlr]rkrY)r rs r!_get_generatorrqs/ ,,C { :: ( ( --r offsetc@t|fd}t|y)a'Set the random number generator state offset of the specified GPU. Args: offset (int): The desired offset device (torch.device or int, optional): The device to set the RNG state. Default: ``'cuda'`` (i.e., ``torch.device('cuda')``, the current CUDA device). c>t}|jyr>)r set_offset)default_generator final_devicers r!cbz!_set_rng_state_offset..cbs*<8$$V,r N)rr)rr r r s` @r!_set_rng_state_offsetr }sv&L-rNr cbtt|}t|}|jS)aPReturn the random number generator state offset of the specified GPU. Args: device (torch.device or int, optional): The device to return the RNG state offset of. Default: ``'cuda'`` (i.e., ``torch.device('cuda')``, the current CUDA device). .. warning:: This function eagerly initializes CUDA. )rrr get_offset)r r rs r!_get_rng_state_offsetrs-Lv&L&|4  ' ' ))r )*cDttt||g|i|Sr>)rr _CudaBase__new__clsr8r9s r! _lazy_newrs%L C ( >t >v >>r c*eZdZdZdZfdZeZxZS)rTFct|j5t| |i|cdddS#1swYyxYwr>)r rrrA)r4r8r9rs r!rAz_CudaBase.types:DOO% & 17<00 1 1 1s4=) rGrHrIr is_sparserArrrrs@r!rrsGI1Gr r)_LegacyStorage_warn_typed_storage_removalcDeZdZedZedZeddddZy)_CudaLegacyStoragec,ttd)Nz+from_buffer: Not available for CUDA storage)rrQrs r! from_bufferz_CudaLegacyStorage.from_buffers#%HIIr ctd)Nz2_new_with_weak_ptr: Not available for CUDA storagerPrs r!_new_with_weak_ptrz%_CudaLegacyStorage._new_with_weak_ptrsOPPr N)r ructd)Nz4_new_shared_filename: Not available for CUDA storagerP)rmanagerrsizer rus r!_new_shared_filenamez'_CudaLegacyStorage._new_shared_filenamesQRRr )rGrHrI classmethodrr r$rr r!rrsGJJQQ@DDSSr rc,eZdZedZedZy) ByteStoragec.t|jSr>r_dtyper?s r!ruzByteStorage.dtype#%{{r c"tjSr>)r]uint8r?s r!r*zByteStorage._dtype {{r NrGrHrIr rur*rr r!r'r'(r r'c,eZdZedZedZy) DoubleStoragec.t|jSr>r)r?s r!ruzDoubleStorage.dtyper+r c"tjSr>)r]doubler?s r!r*zDoubleStorage._dtype ||r Nr/rr r!r2r2(r r2c,eZdZedZedZy) FloatStoragec.t|jSr>r)r?s r!ruzFloatStorage.dtyper+r c"tjSr>)r]floatr?s r!r*zFloatStorage._dtyper.r Nr/rr r!r9r9r0r r9c,eZdZedZedZy) HalfStoragec.t|jSr>r)r?s r!ruzHalfStorage.dtyper+r c"tjSr>)r]halfr?s r!r*zHalfStorage._dtype zzr Nr/rr r!r>r>(r r>c,eZdZedZedZy) LongStoragec.t|jSr>r)r?s r!ruzLongStorage.dtyper+r c"tjSr>)r]longr?s r!r*zLongStorage._dtyperBr Nr/rr r!rErErCr rEc,eZdZedZedZy) IntStoragec.t|jSr>r)r?s r!ruzIntStorage.dtype r+r c"tjSr>)r]rr?s r!r*zIntStorage._dtypes yyr Nr/rr r!rJrJs(r rJc,eZdZedZedZy) ShortStoragec.t|jSr>r)r?s r!ruzShortStorage.dtyper+r c"tjSr>)r]shortr?s r!r*zShortStorage._dtyper.r Nr/rr r!rNrNr0r rNc,eZdZedZedZy) CharStoragec.t|jSr>r)r?s r!ruzCharStorage.dtyper+r c"tjSr>)r]int8r?s r!r*zCharStorage._dtype$rBr Nr/rr r!rSrSrCr rSc,eZdZedZedZy) BoolStoragec.t|jSr>r)r?s r!ruzBoolStorage.dtype*r+r c"tjSr>)r]boolr?s r!r*zBoolStorage._dtype/rBr Nr/rr r!rXrX)rCr rXc,eZdZedZedZy)BFloat16Storagec.t|jSr>r)r?s r!ruzBFloat16Storage.dtype5r+r c"tjSr>)r]rwr?s r!r*zBFloat16Storage._dtype:s ~~r Nr/rr r!r]r]4s(r r]c,eZdZedZedZy)ComplexDoubleStoragec.t|jSr>r)r?s r!ruzComplexDoubleStorage.dtype@r+r c"tjSr>)r]cdoubler?s r!r*zComplexDoubleStorage._dtypeEs }}r Nr/rr r!rara?s(r rac,eZdZedZedZy)ComplexFloatStoragec.t|jSr>r)r?s r!ruzComplexFloatStorage.dtypeKr+r c"tjSr>)r]cfloatr?s r!r*zComplexFloatStorage._dtypePr6r Nr/rr r!rfrfJr7r rfceZdZdZdZdZy)_WrappedTritonKernelzBJust a simple wrapper to store some metadata for testing purposes.c ||_d|_yrkernelkernel_invoked)r4rns r!r6z_WrappedTritonKernel.__init__is #r c8|j|i|}d|_|S)NTrm)r4r8r9rs r!__call__z_WrappedTritonKernel.__call__ms$dkk4*6*" r N)rGrHrIrr6rqrr r!rkrkfsL$r rkc.tjrytd}td}tjj ddu}|rEtj jdd|dtj jdd|dyy) Nc"ddlm}||ddi|S)Nr) bsr_dense_mm skip_checksT)torch.sparse._triton_opsrt)r8r9rts r! kernel_implz-_register_triton_kernels..kernel_implws9T>t>v>>r c"ddlm}||ddi|S)Nr)bsr_dense_addmmruT)rvry)r8r9rys r!addmm_kernel_implz3_register_triton_kernels..addmm_kernel_impl}s<A$A&AAr triton_triton_bsr_dense_mm_outzS_triton_bsr_dense_mm_out(Tensor bsr, Tensor dense, *, Tensor(a!) out) -> Tensor(a!) SparseCsrCUDA_triton_bsr_dense_addmm_outz_triton_bsr_dense_addmm_out(Tensor input, Tensor bsr, Tensor dense, *, Scalar beta, Scalar alpha, Tensor(a!) out) -> Tensor(a!))r]_running_with_deployrk importlibutil find_spec_TritonLibrary registerOp)rwrz has_tritons r!_register_triton_kernelsrss !!#?? BB ))(34?J '' & a     '' )O   r )amp jiteratornvtxprofilersparsetunable){r]BFloat16TensorrX BoolTensorr' ByteTensorrS CharTensorrarfr2 DoubleTensorr9 FloatTensorr> HalfTensorrJ IntTensorrE LongTensorrN ShortTensorrrrrrrrrcaching_allocator_alloccaching_allocator_deletecaching_allocator_enablerrrrrrlr"rYrr rerr empty_cacheget_allocator_backendCUDAPluggableAllocatorchange_current_allocatorrrrrmrget_per_process_memory_fraction get_rng_stateget_rng_state_allrrrrgraphsrWrXhost_memory_stats host_memory_stats_as_nested_dictr initial_seedrrfrqrrrzrlist_gpu_processesr manual_seedmanual_seed_allmax_memory_allocatedmax_memory_cachedmax_memory_reserved mem_get_informemory_allocated memory_cachedmemory_reservedmemory_snapshot memory_statsmemory_stats_as_nested_dictmemory_summaryrMemPoolMemPoolContext use_mem_poolrrrncclrrrandom#reset_accumulated_host_memory_statsreset_accumulated_memory_statsreset_max_memory_allocatedreset_max_memory_cachedreset_peak_host_memory_statsreset_peak_memory_statsrrr set_per_process_memory_fraction set_rng_stateset_rng_state_allr#rrrstreamsrrr)Tr>)rk)rrr/ threadingrCr functoolsrtypingrrrrrr]torch._Cr _device torch._utilsr r r torch.typesrrr_utilsrrrrrrrrrrrr ImportErrorrlocalrLockrrr>tuplerJrgetattrr^rr _device_trNrr#_versionrjrr,pathlibr$r&rOrerrrr\rKrLrRrTrVrWr[ _has_magmarXrY Generatorr_rcrfrqrorzr~rrrrrrxrOutOfMemoryErrorrrrrrQrrrrr rrrmrrrr-r#rLrUr`rlrxrrrrrrerrrlrrr"rrrrrrrrrrrrrrrrrrrrrrr rrr staticmethodrr torch.storagerrrr'r2r9r>rErJrNrSrXr]rarf_storage_classesaddrkrrrrrrr__all__rr r!rsA   77 #EE%32  y%y~~'t (2t8 d3i '(%(($7G '3T) *   <)7||   $$ 5 52#$    &' 588,-!HH::'(?@ 588+,xx44HHH  58801"XX??HsHsH $((%% 4%02E%((,,-25d5 =4= 3d 3 0404 2 84 8!sB42 K$  = I 88,, 2j7t , , ST  (   )y )T ) .HY/ .3 ."(9"5"sCx""*(9"5*AV*& C9 C9 C C6464r !8/0 !] ! v$Ed3ic&: ;EP # $!$s)!4!HxS 2@DId3iDQTI> c 6 C D 8E#v+,>#? C &'+hsm* c $tCy 3 && , ,T , (8I.&$8I.&&26#I. 81 3E#s(O3360S0 vs{); <" vs{); <  XeCK.@%Ac(8E&#+4F+GSVBXeFCK.@%ABSBBHU63;-?$@BCBHU63;-?$@CL8E&#+,>#?L3L!8E&#+,>#?!3!6xfck(:;6s6$0% "4500*/vs{!34//*/vs{!34//*.xfck 23.s.&.xfck 23.s.$ c3 45 %,,  .5<< .EHH,>,> .:@ sC56 & *%S%,,(>"? *S * ??  F S S$&%$$#%$$(-,=)<(;':&<(;';';';'?+/0./  ! H #$>=} g3GB       Ksr/^ ^:,^(^ ^(^: ^^^% ^((^1-^40^11^44^77^::_ ?__