§
     ’ÞjìÓ  ã                  óâ  — d Z ddlmZ ddlZddlZddlmZ ddlmZ ddl	Z	ddl
mZmZ d„ Zdd	d
 e¦   «         rdnd› d�dfddgZddddœZdadadadadadadadadadadadadadadada  e!¦   «         Z"dZ#dZ$d„ Z%d„ Z&d„ Z'd„ Z(d„ Z)d„ Z*d„ Z+d „ Z,d!„ Z-d"„ Z.	 	 dLd$„Z/d%„ Z0dMd&„Z1dMd'„Z2d(„ Z3d)„ Z4d*„ Z5d+„ Z6dNd-„Z7d.„ Z8d/„ Z9d0„ Z:dMd1„Z;e	j<        fdfd2„Z=ddd#dddd3œd4„Z>d5„ Z?dOd6„Z@d7„ ZAd8„ ZBd9„ ZCdMd:„ZDd;„ ZEd<„ ZFd=„ ZG e	jH        ¦   «         d>„ ¦   «         ZId?„ ZJdMdd@œdA„ZKdMdB„ZLdMdC„ZM G dD„ dE¦  «        ZN eN¦   «         ZO G dF„ dG¦  «        ZP eP¦   «         ZQ G dH„ dI¦  «        ZR eR¦   «         ZSdPddJœdK„ZTdS )QzBOptional INT8 backends shared by Quanto and ConvRot linear layers.é    )ÚannotationsN©Úfnmatchcase)Ú
MethodType)Úkernel_policyÚquanto_int8_injectc                 ó€   — 	 ddl m}  n# t          $ r Y dS w xY w|                      ¦   «                              ¦   «         S )zqThe WGPKernels package with its built INT8 library, checked without the GPU (the backend probe runs the kernels).r   ©Úint8F)Ú
wgpkernelsr   ÚImportErrorÚdefault_library_pathÚis_filer
   s    úCC:\Users\johnn\OneDrive\Documents\AI\shared\kernels\int8_backend.pyÚ_wgp_installedr      s\   € ðØ#Ð#Ð#Ð#Ð#Ð#Ð#øÝð ð ð Øˆuˆuðøøøà×$Ò$Ñ&Ô&×.Ò.Ñ0Ô0Ð0s   ‚	 ‰
–)z)Disabled: PyTorch (slowest, most precise)Údisabled)zAuto (best available)ÚautozWGPKernels (recommended, Ú	installedznot installedz, second fastest, precise)r   )zTriton (fast, less precise)Útriton)z.Comfy Kitchen Kernels (fastest, least precise)ÚkitchenÚ
WGPKernelszComfy KitchenÚTriton©r   r   r   ÚpytorchFé   i   c                ó–  — | rt           t          k    rdS ddlm} t          €
 |¦   «         at
          j                             t          dt          › �t          ¦  «        }t          j	        |d¬¦  «         t          j                             ¦   «          |t
          j        d<   t          a t          dt          › d	t          › d
�¦  «         dS )zHIsolate compiled graphs by resolved backend; leave eager runs untouched.Nr   )Ú	cache_dirÚwangp_int8_vT)Úexist_okÚTORCHINDUCTOR_CACHE_DIRz[INT8] Compile Cache: z (vú).)Ú_compile_cache_backendÚ_backendÚ%torch._inductor.runtime.runtime_utilsr   Ú_compile_cache_rootÚosÚpathÚjoinÚ_COMPILE_CACHE_VERSIONÚmakedirsÚtorchÚcompilerÚresetÚenvironÚprint)Úenabledr   r'   s      r   Úprepare_compile_cacher1   4   sÈ   € ð ð Õ,µÒ8Ð8ØˆØ?Ð?Ð?Ð?Ð?Ð?åÐ"Ø'˜i™kœkÐÝŒ7�<Š<Õ+Ð-TÕ<RÐ-TÐ-TÕV^Ñ_Ô_€DÝ„K�˜tÐ$Ñ$Ô$Ð$å	„N×ÒÑÔÐØ,0…B„JÐ(Ñ)Ý%ÐÝ	Ð
J¥8Ð
JÐ
JÕ0FÐ
JÐ
JÐ
JÑKÔKÐKÐKÐKó    c                óv   — t           j                             | j        r|                      ¦   «         n| ¦  «        S ©N)r+   Ú_CÚ
_to_dlpackÚrequires_gradÚdetach)Útensors    r   Ú_export_dlpackr:   F   s.   € õ Œ8×Ò°&Ô2FÐR˜vŸ}š}™œ˜ÈFÑSÔSÐSr2   c                 ó®  — t           j                             ¦   «         sdS t           j        j        d u} | s$t           j                             ¦   «         dk     rdS 	 ddlm} ddlm}  | |d¦  «        ¦  «         |d¦  «        k     rd	S t          j
        | rd
nd¦  «        }| r|                     ¦   «         sdS t          j        ¦   «         5  t          j        dt           j                             ¦   «         ¦  «        }t          j        dt           j        |¬¦  «        }t          j        dt           j        |¬¦  «        }t           j        t           j        t           j        fD ]�}dD ]‹}t          j        |df||¬¦  «        }	dD ]m}
|                     |	||||
¬¦  «        }|j        |	j        k    s1|j        |k    s&t          j        |¦  «                             ¦   «         s   d d d ¦  «         dS ŒnŒŒŒ‘t           j                             |¦  «         d d d ¦  «         n# 1 swxY w Y   |dfS # t6          $ r}d t9          |¦  «        fcY d }~S d }~ww xY w)N)Nzrequires a CUDA or ROCm GPU©é   é   ©Nz;requires an NVIDIA GPU with compute capability 7.5 or newerr   )Úversion)ÚVersionzcomfy-kitchenz0.2.35)Nz requires comfy-kitchen >= 0.2.35zcomfy_kitchen.backends.hipzcomfy_kitchen.backends.cuda)NzOKitchen HIP INT8 requires a working HIP extension and supported RDNA3/3.5/4 GPUÚcuda)é   rC   ©ÚdtypeÚdevicerC   )é   é    ©FT)Ú	out_dtypeÚconvrot)Nz'Kitchen INT8 compatibility probe failedÚ	available)r+   rB   Úis_availabler@   ÚhipÚget_device_capabilityÚimportlib.metadataÚpackaging.versionrA   Ú	importlibÚimport_moduleÚhas_wmmaÚinference_moderF   Úcurrent_deviceÚzerosr   ÚonesÚfloat32Úfloat16Úbfloat16Úint8_linearÚshaperE   ÚisfiniteÚallÚsynchronizeÚ	ExceptionÚstr)rN   r@   rA   ÚmodulerF   ÚwÚsrE   ÚrowsÚxrK   ÚoutÚexcs                r   Ú_probe_kitchenrj   L   sò  € ÝŒ:×"Ò"Ñ$Ô$ð 3Ø2Ð2Ý
Œ-Ô
 4Ð
'€CØð S•5”:×3Ò3Ñ5Ô5¸Ò>Ð>ØRÐRðØ.Ð.Ð.Ð.Ð.Ð.Ø-Ð-Ð-Ð-Ð-Ð-àˆ7�7�7˜?Ñ+Ô+Ñ,Ô,¨w¨w°xÑ/@Ô/@Ò@Ð@Ø;Ð;ÝÔ(ÈÐ)oÐ)EÐ)EÐRoÑpÔpˆØð 	k�v—’Ñ(Ô(ð 	kØjÐjåÔ!Ñ#Ô#ð 	+ð 	+Ý”\ &­%¬*×*CÒ*CÑ*EÔ*EÑFÔFˆFÝ”˜J­e¬jÀÐHÑHÔHˆAÝ”
˜3¥e¤m¸FÐCÑCÔCˆAÝœ-­¬½¼ÐGð Sð S�Ø#ð Sð S�DÝœ T¨3 K°uÀVÐLÑLÔL�AØ#0ð Sð S˜Ø$×0Ò0°°A°qÀEÐSZÐ0Ñ[Ô[˜Øœ9¨¬Ò/Ð/°3´9ÀÒ3EÐ3EÍUÌ^Ð\_ÑM`ÔM`×MdÒMdÑMfÔMfÐ3EØ#RÐ#RÐ#Rð	+ð 	+ð 	+ñ 	+ô 	+ð 	+ð 	+ð 	+ð 4FðSðSõ ŒJ×"Ò" 6Ñ*Ô*Ð*ð	+ð 	+ð 	+ñ 	+ô 	+ð 	+ð 	+ð 	+ð 	+ð 	+ð 	+øøøð 	+ð 	+ð 	+ð 	+ð �{Ð"Ð"øÝð ð ð Ø•S˜‘X”Xˆ~ÐÐÐÐÐÐøøøøðøøøs[   Á-H/ Â
.H/ Â:H/ ÃDH Ç%H/ Ç2"H ÈH/ È H$È$H/ È'H$È(H/ È/
IÈ9IÉ	IÉIc                 ó‚  — t           j                             ¦   «         sdS t           j        j        �9t           j                             d¦  «        j                             d¦  «        sdS t           j        j        €$t           j                             ¦   «         dk     rdS 	 t          j
        d¦  «        } | j        €|                      ¦   «          t          j        ¦   «         5  |                      ¦   «         }d d d ¦  «         n# 1 swxY w Y   |rd |fn| dfS # t          $ r}d t!          |¦  «        fcY d }~S d }~ww xY w)	N)Nzrequires a GPUr   Úgfx1201)Nz-requires an AMD RDNA4 GPU (gfx1201) with ROCmr<   r?   zwgpkernels.int8rL   )r+   rB   rM   r@   rN   Úget_device_propertiesÚgcnArchNameÚ
startswithrO   rR   rS   Ú_libÚload_libraryrU   Úselftestra   rb   )rc   Úfailureri   s      r   Ú_probe_wgpkernelsrt   m   sx  € ÝŒ:×"Ò"Ñ$Ô$ð &Ø%Ð%Ý„}ÔÐ$­U¬Z×-MÒ-MÈaÑ-PÔ-PÔ-\×-gÒ-gÐhqÑ-rÔ-rÐ$ØDÐDå„}ÔÐ ¥U¤Z×%EÒ%EÑ%GÔ%GÈ&Ò%PÐ%PØRÐRð	ÝÔ(Ð):Ñ;Ô;ˆØŒ;ÐØ×ÒÑ!Ô!Ð!åÔ!Ñ#Ô#ð 	(ð 	(Ø—o’oÑ'Ô'ˆGð	(ð 	(ð 	(ñ 	(ô 	(ð 	(ð 	(ð 	(ð 	(ð 	(ð 	(øøøð 	(ð 	(ð 	(ð 	(à")ÐD��gˆˆ°¸Ð/DÐDøÝð ð ð Ø•S˜‘X”Xˆ~ÐÐÐÐÐÐøøøøðøøøsC   Â!AD Ã#DÃ8D ÄDÄD ÄDÄD Ä
D>Ä#D9Ä3D>Ä9D>c                ó„  — | d„ t           D ¦   «         vrt          d| ›�¦  «        ‚| dk    rdS t          t          t          j        dœ}| dk    r: ||          ¦   «         \  }}|�| |fS t          dt          |          › d	|› d
�¦  «         t          j	        dd…         D ]#}|| k    r ||         ¦   «         \  }}|�||fc S Œ$dS )zÈValidate user selection before changing the active runtime or config. The selected backend, else the Auto choice: WGPKernels,
    then Comfy Kitchen, then Triton, then PyTorch (kernel_policy.RANKING).c                ó   — h | ]\  }}|’ŒS © rw   )Ú.0Ú_Úvalues      r   ú	<setcomp>z"resolve_backend.<locals>.<setcomp>„   s   € Ð7Ð7Ð7¡8 1 e˜Ð7Ð7Ð7r2   zUnknown INT8 kernel backend: r   )r   Nr   r   Nz[INT8] z unavailable: z. Using the Auto choice.é   )
ÚCHOICESÚ
ValueErrorrt   rj   r   Ú_probe_triton_backendr/   Ú_LABELSr   ÚRANKING)Ú	selectionÚprobesrc   ÚreasonÚbackends        r   Úresolve_backendr†   �   s  € ð Ð7Ð7­wÐ7Ñ7Ô7Ð7Ð7ÝÐF¸ÐFÐFÑGÔGÐGØ�JÒÐØˆÝ-½.ÕTZÔTpÐqÐq€FØ�FÒÐØ*˜ 	Ô*Ñ,Ô,‰ˆ�ØÐØ˜fÐ$Ð$ÝÐZ� 	Ô*ÐZÐZ¸&ÐZÐZÐZÑ[Ô[Ð[Ý Ô(¨¨!¨Ô,ð 'ð 'ˆØ�iÒÐØ,˜V Gœ_Ñ.Ô.‰NˆF�FØÐ!Ø �Ð&Ð&Ð&øØˆ?r2   c                 ó   — t           dk    S )Nr   ©r#   rw   r2   r   Úkitchen_enabledr‰   –   s   € Ý�yÒ Ð r2   c                 ó   — t           dk    S )Nr   rˆ   rw   r2   r   Úwgp_enabledr‹   š   s   € Ý�|Ò#Ð#r2   c                óJ  — t          | ¦  «        t          j        u s)t          j        | ¦  «        ott          j        | ¦  «         o_| j        oX| j        t          j        t          j	        fv o9|j
        j        o-|j
        j        t          j        k    o| j        d         dz  dk    S )z�Inputs of WGPKernels: BF16/FP16 activations on the GPU (plain, or traced by torch.compile, not Quanto activations), INT8
    weights with K a multiple of 16.éÿÿÿÿé   r   )Útyper+   ÚTensorr   Ú_is_fake_tensorÚ_is_qbytes_tensorÚis_cudarE   r[   rZ   Ú_datar   r]   ©ÚinputÚweights     r   Ú_wgp_supportsr˜   ž   s¢   € õ �%‰[Œ[�EœLÐ(Ðs­VÔ-CÀEÑ-JÔ-JÐ-rÕSYÔSkÐlqÑSrÔSrÐOrð hØ”ðhØ"'¤+µ%´.Å%Ä-Ð1PÐ"Pðhà”Ô$ðhà)/¬Ô);½u¼zÒ)IðhàNSÌkÐZ\ÌoÐ`bÑNbÐfgÒNgðir2   c                óÖ   — |                       d¦  «        dk    s/|                       d¦  «        dz  s|                      ¦   «         dz  r |                      t          j        ¬¦  «        n| S )Nr�   rG   r   é   rŽ   )Úmemory_format)ÚstrideÚdata_ptrÚcloner+   Úcontiguous_format)rg   s    r   Ú_wgp_alignedr    ¦   sf   € à=>¿XºXÀb¹\¼\ÈQÒ=NÐ=NÐRS×RZÒRZÐ[\ÑR]ÔR]Ð`aÑRaÐ=NÐef×eoÒeoÑeqÔeqÐtvÑevÐ=Nˆ1�7Š7¥Ô!8ˆ7Ñ9Ô9Ð9Ð|}Ð}r2   c                óZ   — t                                t          | ¦  «        |||||¦  «        S r4   )Ú_wgpÚlinearr    ©rg   r—   ÚscaleÚbiasÚgroupÚprecises         r   Ú_wgp_runr©   «   s$   € Ý�;Š;•| A‘”¨°°t¸UÀGÑLÔLÐLr2   ç        c                ó0  — |j         j        d         }t          ||| j        ¦  «        }|                      d| j        d         ¦  «        }|�*|j        |j        k    r|                     |j        ¦  «        }t          j         	                    ¦   «         st          j        | ¦  «        rGt          j        j                             ||j         ||||¦  «        j        g | j        dd…         ¢|‘R Ž S |€7|€5|
€3 t          ||j         ||||¦  «        j        g | j        dd…         ¢|‘R Ž S |�*|j        |j        k    r|                     |j        ¦  «        }|�E|                     d|¦  «        }|	�-|	                     d¦  «                             |j        ¦  «        }	d}|
�}|
j        g | j        dd…         ¢|‘R k    s;|
j        |j        k    s+|
                     ¦   «         r|
                     ¦   «         dz  rt%          d¦  «        ‚|
                     d|¦  «        }t(                               t+          |¦  «        |j         |||||||||	|¬¦  «        }|
€ |j        g | j        dd…         ¢|‘R Ž S t          j        j                             |
¦  «         |
S )aé  INT8 linear with WGPKernels (ConvRot rotation `group`, 0 for plain INT8). The kernels choose, per GPU and shape, full
    precision activations or per-block INT8 activations; precise=True keeps full precision at every token count.
    Fusions (eager inference, see can_fuse_linear): act 'rms_norm' (act_weight, act_eps) or 'swiglu' (input rows: gate | value) applied in
    FP32 to the input, and residual + residual_scale * linear rounded once, written to out (which may be the residual).r   r�   NrŽ   zZFused INT8 output must match the output shape and dtype, be contiguous and 16-byte aligned)ÚactÚ
act_weightÚepsÚresidualÚresidual_scalerh   )r”   r]   Ú_weight_scalerF   ÚreshaperE   Útor+   r,   Úis_compilingr   r‘   ÚopsÚ
wan2gp_wgpr£   r©   Úis_contiguousr�   r~   Úviewr¢   r    ÚautogradÚgraphÚincrement_version)r–   r—   r¦   r§   r¨   r¬   r­   Úact_epsr¯   r°   rh   Únr¥   rg   Úys                  r   Ú
wgp_linearr¿   ¯   s«  € ð 	ŒÔ˜1Ô€AÝ˜& ! U¤\Ñ2Ô2€EØ�Š�b˜%œ+ bœ/Ñ*Ô*€AØÐ˜DœJ¨!¬'Ò1Ð1Ø�wŠw�q”wÑÔˆÝ„~×"Ò"Ñ$Ô$ð w­Ô(>¸uÑ(EÔ(Eð wÝŒyÔ#×*Ò*¨1¨f¬l¸EÀ4ÈÐPWÑXÔXÔ`ÐvÐbgÔbmÐnqÐoqÐnqÔbrÐvÐtuÐvÐvÐvÐvØ
€{�xÐ'¨C¨KØM�x˜˜6œ<¨°°e¸WÑEÔEÔMÐcÈuÌ{Ð[^Ð\^Ð[^ÔO_ÐcÐabÐcÐcÐcÐcØÐ *Ô"2°a´gÒ"=Ð"=Ø—]’] 1¤7Ñ+Ô+ˆ
ØÐØ×#Ò# B¨Ñ*Ô*ˆØÐ%Ø+×3Ò3°BÑ7Ô7×:Ò:¸1¼7ÑCÔCˆNØ€AØ
€ØŒ9Ð.˜%œ+ c r cÔ*Ð.¨AÐ.Ð.Ò.Ð.°#´)¸q¼wÒ2FÐ2FÈc×N_ÒN_ÑNaÔNaÐ2FÐeh×eqÒeqÑesÔesÐvxÑexÐ2FÝÐyÑzÔzÐzØ�HŠH�R˜‰OŒOˆÝ�Š•L ‘O”O V¤\°5¸$ÀÀwÐTWÐdnÐt{ð  GOØ#1°qð 	ñ 	:ô 	:€Aà
€{ØˆqŒyÐ.˜%œ+ c r cÔ*Ð.¨AÐ.Ð.Ð.Ð.Ý	„NÔ×*Ò*¨3Ñ/Ô/Ð/Ø€Jr2   c                óØ   — t           dk    o_ t          t          dd„ ¦  «        ¦   «         o@| j        o9| j        t
          j        t
          j        fv o|j        o| j        d         dz  dk    S )zíWhether WGPKernels computes an INT8 weight with one FP32 scale per output channel and 128-column block (scale (N, K / 128)):
    the selected backend, with a library taking such scales, on BF16/FP16 CUDA activations (K a multiple of 16).r   Úweight_groups_supportedc                 ó   — dS )NFrw   rw   r2   r   ú<lambda>z"wgp_group_scales.<locals>.<lambda>Ô   s   € ÐZ_€ r2   r�   rŽ   r   )	r#   Úgetattrr¢   r“   rE   r+   r[   rZ   r]   r•   s     r   Úwgp_group_scalesrÅ   Ñ   s˜   € õ ˜Ò$ð =Ð)`­µÐ7PÐR_ÐR_Ñ)`Ô)`Ñ)bÔ)bð =ÐglÔgtð =Ðy~ô  zEõ  JOô  JXõ  Z_ô  Zgð  Ihð  zhð =Ø”ð=Ø#(¤;¨r¤?°RÑ#7¸1Ò#<ð>r2   c           	     ó  — |                       d| j        d         ¦  «        }|�*|j        |j        k    r|                     |j        ¦  «        }t          j                             ¦   «         st          j        | ¦  «        rMt	          j	        j
                             ||||dd¦  «        j         g | j        dd…         ¢|j        d         ‘R Ž S  t          ||||dd¦  «        j         g | j        dd…         ¢|j        d         ‘R Ž S )z«input @ (weight (N, K) INT8 * scale (N, K / 128) per 128-column block).T + bias with WGPKernels, which choose the path per GPU and
    shape as for the other INT8 weights.r�   Nr   F)r²   r]   rE   r³   r+   r,   r´   r   r‘   rµ   r¶   r£   r©   )r–   r—   r¥   r¦   rg   s        r   Úwgp_group_linearrÇ   Ø   s  € ð 	�Š�b˜%œ+ bœ/Ñ*Ô*€AØÐ˜DœJ¨!¬'Ò1Ð1Ø�wŠw�q”wÑÔˆÝ„~×"Ò"Ñ$Ô$ð y­Ô(>¸uÑ(EÔ(Eð yÝŒyÔ#×*Ò*¨1¨f°e¸TÀ1ÀeÑLÔLÔTÐxÐV[ÔVaÐbeÐceÐbeÔVfÐxÐhnÔhtÐuvÔhwÐxÐxÐxÐxØ=�8�A�v˜u d¨A¨uÑ5Ô5Ô=Ða¸u¼{È3ÈBÈ3Ô?OÐaÐQWÔQ]Ð^_ÔQ`ÐaÐaÐaÐar2   c
                ó¨   — t           dk    r1t          st          d¦  «         dat          | |||||||||	¬¦
  «
        S t	          | ||||||||	¦	  «	        S )z]linear_with_fusion's computation by the active backend (ConvRot layers, see can_fuse_linear).r   z\[INT8] WGPKernels fused normalization/activation and residual linear kernels are being used.T©r¬   r­   r¼   r¯   r°   rh   )r#   Ú_fusion_loggedr/   r¿   Úkitchen_linear_fused)
r–   r—   r¦   r§   Ú	input_actr­   r¼   r¯   r°   rh   s
             r   Úlinear_fusedrÍ   ã   s   € õ �<ÒÐÝð 	"ÝÐpÑqÔqÐqØ!ˆNÝ˜% ¨¨u¸)ÐPZÐdkÐv~Ø)7¸SðBñ Bô Bð 	Bå  v¨t°YÀ
ÈGÐU]Ð_mÐorÑsÔsÐsr2   c                ó
  — t           dk    rk|\  }}t          | | j        j        d         |j        ¦  «        }t
                               || j        ||€dn|                     |j        ¦  «        |¬¦  «        S t          | |g|¢R Ž S )zyThe output rows of a projection whose input linear_multi / linear_grouped prepared once (operands of the active backend).r   r   N)rh   )
r#   r±   r”   r]   rF   r¢   Úlinear_preparedr³   rE   Úkitchen_linear_prequantized)r—   r¦   ÚpreparedÚprepared_inputrh   r¥   s         r   Úlinear_prequantizedrÓ   ï   s‹   € å�<ÒÐØ&Ñˆ˜Ý˜f f¤lÔ&8¸Ô&;¸S¼ZÑHÔHˆÝ×#Ò# N°F´LÀ%ÐQUÐQ]ÈÈÐcg×cjÒcjÐknÔktÑcuÔcuÐ{~Ð#ÑÔÐÝ& v¨tÐ?°hÐ?Ð?Ð?Ð?r2   c                óV  — ddl m} d}|                      ¦   «         D ]q}t          |t          j        j        ¦  «        rPt          |dd¦  «        |k    r;t          |d¦  «        s+|j	        |_
        t          t          |¦  «        |_	        |dz  }Œr|rt          d|› d|› d	�¦  «         dS dS )
a8  Keep activation outliers from erasing small values in rowwise INT8 activations: Kitchen adds a second INT8 product of the
    discarded residual, WGPKernels keep full precision activations at every token count.

    Install before MMGP profiling. Other backends and weight formats retain their own forwards.
    r   ©Úqint8Úweight_qtypeNÚ_int8_residual_forwardrG   z/[INT8] Full Precision Activations Required for z Linear Layers - ú.)Úoptimum.quantorÖ   ÚmodulesÚ
isinstancer+   ÚnnÚLinearrÄ   ÚhasattrÚforwardrØ   r   Ú_residual_module_forwardr/   )Úmodelr„   rÖ   Úcountrc   s        r   Úrequire_int8_residualrä   ø   sØ   € ð %Ð$Ð$Ð$Ð$Ð$à€EØ—-’-‘/”/ð ð ˆÝ�v�uœxœÑ/Ô/ð 	µG¸FÀNÐTXÑ4YÔ4YÐ]bÒ4bÐ4bÝ Ð(@ÑAÔAð 5cà,2¬NˆFÔ)Ý'Õ(@À&ÑIÔIˆFŒNØ�Q‰JˆEøØð cÝÐaÀÐaÐaÐX^ÐaÐaÐaÑbÔbÐbÐbÐbðcð cr2   c                ó,  — t           dk    r2t          || j        ¦  «        rt          || j        | j        d¬¦  «        S t           dk    sLt          |¦  «        t          j        us1|j        r*|j	        t          j
        t          j        t          j        fvr|                      |¦  «        S | j        }|j        j        d         }t!          |||j        ¦  «        }|                     d|j        d         ¦  «        }t          j                             ¦   «         st+          j        |¦  «        r2t          j        j                             ||j        || j        ¦  «        }nt5          ||j        || j        ¦  «        } |j        g |j        d d…         ¢|‘R Ž S )Nr   T)r¨   r   r   r�   )r#   r˜   Úqweightr¿   r¦   r�   r+   r�   r“   rE   rZ   r[   rY   rØ   r”   r]   r±   rF   r²   r,   r´   r   r‘   rµ   Úwan2gp_kitchenÚresidual_linearÚ_residual_linear_impl)rc   r–   r—   r½   r¥   rg   rh   s          r   rá   rá     sZ  € Ý�<ÒÐ¥M°%¸¼Ñ$HÔ$HÐÝ˜% ¤°´ÀdÐKÑKÔKÐKÝ�IÒÐ¥ e¡¤µE´LÐ!@Ð!@ÈÌÐ!@ØŒ{¥5¤=µ%´.Å%Ä-Ð"PÐPÐPØ×,Ò,¨UÑ3Ô3Ð3ØŒ^€FØŒÔ˜1Ô€AÝ˜& ! U¤\Ñ2Ô2€EØ�Š�b˜%œ+ bœ/Ñ*Ô*€AÝ„~×"Ò"Ñ$Ô$ð I­Ô(>¸uÑ(EÔ(Eð IÝŒiÔ&×6Ò6°q¸&¼,ÈÈvÌ{Ñ[Ô[ˆˆå# A v¤|°U¸F¼KÑHÔHˆØˆ3Œ;Ð,˜œ C R CÔ(Ð,¨!Ð,Ð,Ð,Ð,r2   c                ó  — | j         \  }}|j         d         }t          dt          d|z  d|z  z   dz   z  ¦  «        }|                      ||f¦  «        }t	          d||¦  «        D �]'}	t          |	|z   |¦  «        }
| |	|
…         }t                               |¦  «        \  }}|                     ¦   «          	                    |¦  «         
                    ¦   «                              |¦  «        }~~t                               |||t          j        ¬¦  «        }t                               |||t          j        ¬¦  «        }|                     |¦  «         |�|                     |¦  «         ||	|
…                              |¦  «         ~~~�Œ)|S )Nr   rG   é
   é   é@   ©rJ   )r]   ÚmaxÚ_SCRATCH_BYTESÚ	new_emptyÚrangeÚminÚ_kitchenÚquantize_int8_rowwiseÚfloatÚmul_Úneg_Úadd_r\   r+   rY   Úcopy_)rg   r—   r¥   r¦   ÚmÚkr½   rf   rh   ÚstartÚstopÚtileÚqÚqsr¯   ÚhighÚlows                    r   ré   ré     sh  € ð
 Œ7�D€A€qØŒ�QŒ€AÝˆq•. R¨!¡V¨b°1©f¡_°rÑ%9Ñ:Ñ;Ô;€DØ
�+Š+�q˜!�fÑ
Ô
€CÝ�q˜!˜TÑ"Ô"ð  ñ  ˆÝ�5˜4‘< Ñ#Ô#ˆØ��t�Œ}ˆÝ×.Ò.¨tÑ4Ô4‰ˆˆ2Ø—7’7‘9”9—>’> "Ñ%Ô%×*Ò*Ñ,Ô,×1Ò1°$Ñ7Ô7ˆØˆrÝ×#Ò# D¨&°%Å5Ä=Ð#ÑQÔQˆÝ×"Ò" 8¨V°UÅeÄmÐ"ÑTÔTˆØ�	Š	�#‰ŒˆØÐØ�IŠI�d‰OŒOˆOØˆE�$ˆJŒ×Ò˜dÑ#Ô#Ð#Ø�d˜C™CØ€Jr2   ©Ú*c           	     ó  ‡— ddl m} d}|                      ¦   «         D ]�\  Š}t          |t          j        j        ¦  «        rkt          |dd¦  «        |k    rVt          ˆfd„|D ¦   «         ¦  «        r;t          |d¦  «        s+|j
        |_        t          t          |¦  «        |_
        |dz  }Œ�|rPdat          ¦   «         rBt!          j        ¦   «         \  a}t$          �d	nd
|› d�}t'          d|› d|› d|› d�¦  «         dS dS dS )a~  Protect a model or subtree whose plain INT8 linears need finer activation scales.

    Install before MMGP profiling/LoRA hooks. Other qtypes and the user's Disabled,
    Triton or WGPKernels selections retain their normal forwards (WGPKernels
    use per-block activation scales at most); Kitchen uses Triton for these linears,
    or native Quanto when Triton is unavailable.
    r   rÕ   r×   Nc              3  ó8   •K  — | ]}t          ‰|¦  «        V — Œd S r4   r   )rx   ÚpatternÚnames     €r   ú	<genexpr>z)require_blockwise_int8.<locals>.<genexpr>C  s-   øè è € ÐRÐR°w� D¨'Ñ2Ô2ÐRÐRÐRÐRÐRÐRr2   Ú_int8_precision_forwardrG   Tr   ú	PyTorch (ú)z[INT8] Precision Rule: z for z linear layers - rÙ   )rÚ   rÖ   Únamed_modulesrÜ   r+   rÝ   rÞ   rÄ   Úanyrß   rà   r  r   Ú_blockwise_module_forwardÚ_blockwise_requiredr‰   r   r   Ú_blockwise_tritonr/   )	râ   r„   Úmodule_patternsrÖ   rã   rc   Úunavailabler…   r	  s	           @r   Úrequire_blockwise_int8r  5  sa  ø€ ð %Ð$Ð$Ð$Ð$Ð$à€EØ×+Ò+Ñ-Ô-ð ð ‰ˆˆfÝ�v�uœxœÑ/Ô/ð 	µG¸FÀNÐTXÑ4YÔ4YÐ]bÒ4bÐ4bÝÐRÐRÐRÐRÀ/ÐRÑRÔRÑRÔRð 5cå Ð(AÑBÔBð 5cð .4¬^ˆFÔ*Ý'Õ(AÀ6ÑJÔJˆFŒNØ�Q‰JˆEøØð ]Ø"ÐÝÑÔð 	]Ý-3Ô-IÑ-KÔ-KÑ*Ð˜{Ý"3Ð"?�h�hÐE_ÐQ\ÐE_ÐE_ÐE_ˆGÝÐ[¨GÐ[Ð[¸%Ð[Ð[ÐRXÐ[Ð[Ð[Ñ\Ô\Ð\Ð\Ð\ð]ð ]ð	]ð 	]r2   c                óÖ   — t           €'t          j        j                             | ||¦  «        }n#t                                | ||| j        ¬¦  «        }|�|                     |¦  «         |S )Nrî   )r  r+   rµ   ÚquantoÚ	qbytes_mmÚfused_quant_scaled_mmrE   rù   )rg   r—   r¥   r¦   rh   s        r   Ú_blockwise_linear_implr  P  s`   € ÝÐ ÝŒiÔ×(Ò(¨¨F°EÑ:Ô:ˆˆå×5Ò5°a¸ÀÐRSÔRYÐ5ÑZÔZˆØÐØ�Š�‰ŒˆØ€Jr2   c                óŒ  — t           dk    sLt          |¦  «        t          j        us1|j        r*|j        t          j        t          j        t          j        fvr|  	                    |¦  «        S | j
        }|                     d|j        d         ¦  «        }t          j                             ¦   «         st          j        |¦  «        r7t          j        j                             ||j        |j        | j        ¦  «        }n!t/          ||j        |j        | j        ¦  «        } |j        g |j        d d…         ¢|j        d         ‘R Ž S )Nr   r�   r   )r#   r�   r+   r�   r“   rE   rZ   r[   rY   r  ræ   r²   r]   r,   r´   r   r‘   rµ   rç   Úblockwise_linearr”   Ú_scaler¦   r  )rc   r–   r—   rg   rh   s        r   r  r  Z  s  € Ý�IÒÐ¥ e¡¤µE´LÐ!@Ð!@ÈÌÐ!@ØŒ{¥5¤=µ%´.Å%Ä-Ð"PÐPÐPØ×-Ò-¨eÑ4Ô4Ð4ØŒ^€FØ�Š�b˜%œ+ bœ/Ñ*Ô*€AÝ„~×"Ò"Ñ$Ô$ð R­Ô(>¸uÑ(EÔ(Eð RÝŒiÔ&×7Ò7¸¸6¼<ÈÌÐX^ÔXcÑdÔdˆˆå$ Q¨¬°f´mÀVÄ[ÑQÔQˆØˆ3Œ;Ð:˜œ C R CÔ(Ð:¨&¬,°q¬/Ð:Ð:Ð:Ð:r2   c                óZ  — t           j                             ¦   «         rt          j        | j        ||¦  «        S | j                             dd ¦  «        }|€Tddlm	} | 
                    ¦   «         5  t          j        | j        ||¦  «        x}| _        d d d ¦  «         n# 1 swxY w Y   |S )NÚ_kitchen_scaler   )Ú	allocator)r+   r,   r´   r   Ú_prepare_weight_scaler  Ú__dict__ÚgetÚmmgpr   Únormal_allocationsr  )r—   r½   rF   r¥   r   s        r   r±   r±   g  sþ   € õ „~×"Ò"Ñ$Ô$ð FÝÔ+¨F¬M¸1¸fÑEÔEÐEØŒO×ÒÐ 0°$Ñ7Ô7€EØ€}Ø"Ð"Ð"Ð"Ð"Ð"à×)Ò)Ñ+Ô+ð 	cð 	cÝ,2Ô,HÈÌÐXYÐ[aÑ,bÔ,bÐbˆE�FÔ)ð	cð 	cð 	cñ 	cô 	cð 	cð 	cð 	cð 	cð 	cð 	cøøøð 	cð 	cð 	cð 	cà€Ls   Á1#B Â B$Â'B$c                óV  — ddl m}  || d¦  «        }|                     d|j        d         ¦  «        }t	          ||j        d         | j        ¦  «        }t                               ||j        || j	        ¬¦  «        }|�||z  } |j        g | j        dd…         ¢|j        d         ‘R Ž S )zFUse Triton for ConvRot rows beyond Kitchen's graph-safe CUTLASS width.r   ©Ú_rotate_activationrC   r�   rî   N)
Úshared.qtypes.int8_convrotr(  r²   r]   r±   rF   Ú_wide_convrot_tritonr  r”   rE   )r–   r—   r¦   r(  Úrotatedrg   r¥   rh   s           r   Ú_wide_convrot_linearr,  u  s¸   € à=Ð=Ð=Ð=Ð=Ð=à Ð  ¨Ñ,Ô,€GØ�Š˜˜GœM¨"Ô-Ñ.Ô.€AÝ˜& &¤,¨q¤/°5´<Ñ@Ô@€EÝ
×
4Ò
4°Q¸¼ÀeÐW\ÔWbÐ
4Ñ
cÔ
c€CØÐØˆt‰ˆØˆ3Œ;Ð:˜œ C R CÔ(Ð:¨&¬,°q¬/Ð:Ð:Ð:Ð:r2   c                óö  — t           j                             ¦   «         s]t          j        ¦   «         rJt	          j        |¦  «        s6|j        r/|j        |vs&t          | dd¦  «        rt          | dd¦  «        i k    rdS t          dk    r]t          | dd¦  «        }|dk    o#t          t          | dd¦  «        d	d¦  «        d
k    }|t          j        v o|dv s|o| j        dz  dk    S |dk    oŽt          j        ¦   «         o{t          ¦   «         omt           oft          | dd¦  «        dk    oQd| j        cxk    odk    nc o<| j        dz  dk    o.| j        dz  dk    o t$                               || j        d¦  «        S )a  Keep qtype/backend details out of model code; MMGP still owns module calls.
    Callers widen `dtypes` only for activation dtypes validated with their model; act is the input operation they want fused
    ('rms_norm', 'swiglu', or 'gelu', which only WGPKernels provide).
    WGPKernels fuse with the precision of their unfused linears (full precision or per-block activations, input operations in FP32,
    one rounding), whatever the kernel precision setting; Comfy Kitchen's rowwise fusions need the fast (approximate) setting.Ú_mm_lora_old_forwardNÚ_mm_lora_dataFr   Ú_convrot_group_sizer   r×   r	  rÖ   ©rŽ   rí   rC   rŽ   ÚgelurC   é @  rš   )r+   r,   r´   Úis_inference_mode_enabledr   r‘   r“   rE   rÄ   r#   r¢   ÚACTSÚin_featuresr   Úallow_approximater‰   Ú_sm120_cutlassÚout_featuresrô   Ú!_convrot_fused_shared_memory_fits)rc   rg   Údtypesr¬   r§   r  s         r   Úcan_fuse_linearr<  ‚  sã  € õ 	Œ×#Ò#Ñ%Ô%ð ­UÔ-LÑ-NÔ-Nð ÕRXÔRhÐijÑRkÔRkð ÐstÔs|ð ØŒw˜fÐ$Ð$­°Ð9OÐQUÑ)VÔ)VÐ$Õ[bÐciÐkzð  }Añ  \Bô  \Bð  FHò  \Hð  \HØˆuÝ�<ÒÐÝ˜Ð 5°qÑ9Ô9ˆØ˜!’Ðg¥­°¸ÈÑ(MÔ(MÈvÐW[Ñ \Ô \Ð`gÒ gˆØ•d”iÐÐg U¨mÐ%;Ð%E¸vÐgÈ6ÔK]Ð`bÑKbÐfgÒKgÐgØ�6ŠMð W�mÔ=Ñ?Ô?ð WÅOÑDUÔDUð WÕZhð WÝ˜Ð 5°qÑ9Ô9¸SÒ@ðWà�vÔ)Ð2Ð2Ò2Ð2¨UÒ2Ð2Ð2Ð2ðWà7=Ô7IÈCÑ7OÐSTÒ7TðWð Ô# aÑ'¨1Ò,ðWõ ×:Ò:¸1¸fÔ>PÐRUÑVÔVð	Xr2   )rÌ   r­   r¼   r¯   r°   rh   c               ó&  — t          | dd ¦  «        }||||||fx}	| _        t          t          | dd ¦  «        dd ¦  «        dk    rt          | dd ¦  «        nd }
|
�|	|
_        	  | |¦  «        |€| `n|| _        |
�|
`S S # |€| `n|| _        |
�|
`w xY w)NÚ_wangp_linear_fusionr×   r	  rÖ   ræ   )rÄ   r>  )rc   rg   rÌ   r­   r¼   r¯   r°   rh   ÚpreviousÚfusionræ   s              r   Úlinear_with_fusionrA  –  sî   € õ �vÐ5°tÑ<Ô<€HØ,5°zÀ7ÈHÐVdÐfiÐ+jÐj€FˆVÔ(å29½'À&È.ÐZ^Ñ:_Ô:_ÐagÐimÑ2nÔ2nÐryÒ2yÐ2y�g�f˜i¨Ñ.Ô.Ð.ð  @D€GØÐØ'-ˆÔ$ð-Øˆv�a‰yŒyàÐØÐ+Ð+à*2ˆFÔ'ØÐØÐ,Ð,ð øð	 ÐØÐ+Ð+à*2ˆFÔ'ØÐØÐ,Ð,Ð,Ð,Ð,s   Á"
A> Á>Bc                ó¼   — | j         j        dk    r| j        n| }t          |dd¦  «        }||_        	  | |¦  «         |€|`dS ||_        dS # |€|`n||_        w xY w)z|module(x) with its input already prepared (and its output tile): the module's own hooks run, MMGP's weight loading included.rÖ   Ú_wangp_prequantized_inputN)r×   r	  ræ   rÄ   rC  )rc   rg   rÑ   Útargetr?  s        r   Ú_call_preparedrE  «  s�   € à%Ô2Ô7¸7ÒBÐBˆVŒ^ˆ^È€FÝ�vÐ:¸DÑAÔA€HØ'/€FÔ$ð8Øˆˆq‰	Œ	ˆ	àÐØÐ0Ð0Ð0à/7ˆFÔ,Ð,Ð,øð ÐØÐ0Ð0à/7ˆFÔ,Ð7Ð7Ð7Ð7s   ³A ÁAc           	     ó    — | j         }t                               ||j        t	          ||j        j        d         |j        ¦  «        d||¦  «        S )zhThe WGPKernels path for x and this module's weight, used for every projection of an input prepared once.r   N)ræ   r¢   Úchoose_pathr”   r±   r]   rF   )rc   rg   r§   r¨   r—   s        r   Ú	_wgp_pathrH  ¹  sG   € àŒ^€FÝ×Ò˜A˜vœ|­]¸6À6Ä<ÔCUÐVWÔCXÐZ[ÔZbÑ-cÔ-cÐeiÐkpÐryÑzÔzÐzr2   c                ó  ‡‡— t           dk    �r t          | d         dd¦  «        Št          ˆfd„| D ¦   «         ¦  «        rˆfd„| D ¦   «         S t          ‰¦  «        Šˆfd„| D ¦   «         }t	          dt
          d‰j        d	         z  d
z   z  dz  dz  ¦  «        }t          d‰j        d         |¦  «        D ]u}‰|||z   …         }t           	                    |‰t          | d         |‰¦  «        ¦  «        }t          | |¦  «        D ]#\  }}t          |||||||z   …         f¦  «         Œ$~Œv|S ˆfd„| D ¦   «         }t	          dt
          ‰j        d	         d
z   z  dz  dz  ¦  «        }t          d‰j        d         |¦  «        D ]|}t          ||z   ‰j        d         ¦  «        }	‰||	…         }t                               |d¦  «        \  }
}t          | |¦  «        D ]!\  }}t          |||
||||	…         f¦  «         Œ"~
~Œ}|S )zKReuse one bounded INT8 input tile across projections, retaining MMGP hooks.r   r   r0  c              3  óB   •K  — | ]}t          |d d¦  «        ‰k    V — ŒdS )r0  r   N)rÄ   )rx   rc   r§   s     €r   r
  zlinear_multi.<locals>.<genexpr>Ã  s5   øè è € ÐXÐXÀf�w�vÐ4°aÑ8Ô8¸EÒAÐXÐXÐXÐXÐXÐXr2   c                ó&   •— g | ]} |‰¦  «        ‘ŒS rw   rw   ©rx   rc   rg   s     €r   ú
<listcomp>z linear_multi.<locals>.<listcomp>Ä  s!   ø€ Ð4Ð4Ð4 &�F�F˜1‘I”IÐ4Ð4Ð4r2   c                ó^   •— g | ])}‰                      ‰j        d          |j        f¦  «        ‘Œ*S ©r   ©rñ   r]   r9  rL  s     €r   rM  z linear_multi.<locals>.<listcomp>Æ  s3   ø€ ÐXÐXÐXÀf�1—;’; ¤¨¤
¨FÔ,?Ð@ÑAÔAÐXÐXÐXr2   é€   é   r�   r   c                ó^   •— g | ])}‰                      ‰j        d          |j        f¦  «        ‘Œ*S rO  rP  rL  s     €r   rM  z linear_multi.<locals>.<listcomp>Ï  s3   ø€ ÐTÐTÐTÀ&ˆq�{Š{˜AœG AœJ¨Ô(;Ð<Ñ=Ô=ÐTÐTÐTr2   rH   rC   )r#   rÄ   r  r    rï   rð   r]   rò   r¢   ÚpreparerH  ÚziprE  ró   rô   Úquantize_int8_rowwise_convrot64)rÛ   rg   Úoutputsrf   rý   rÿ   rÑ   rc   Úoutputrþ   r   Úscalesr§   s    `          @r   Úlinear_multirZ  ¿  sR  øø€ å�<ÒÑÝ˜ œ
Ð$9¸1Ñ=Ô=ˆÝÐXÐXÐXÐXÐPWÐXÑXÔXÑXÔXð 	5Ø4Ð4Ð4Ð4¨GÐ4Ñ4Ô4Ð4Ý˜‰OŒOˆØXÐXÐXÐXÐPWÐXÑXÔXˆÝ�3�¨1¨q¬w°r¬{©?¸QÑ+>Ñ?À3ÑFÈÑLÑMÔMˆÝ˜1˜aœg aœj¨$Ñ/Ô/ð 	ð 	ˆEØ�U˜5 4™<Ð'Ô(ˆDÝ—|’| D¨%µ¸7À1¼:ÀtÈUÑ1SÔ1SÑTÔTˆHÝ"% g¨wÑ"7Ô"7ð Uð U‘�˜Ý˜v t¨h¸¸uÀUÈTÁ\Ð?QÔ8RÐ-SÑTÔTÐTÐTØ�ØˆØTÐTÐTÐTÈGÐTÑTÔT€GÝˆr•N q¤w¨r¤{°Q¡Ñ7¸BÑ>ÀÑCÑDÔD€DÝ�q˜!œ' !œ* dÑ+Ô+ð ð ˆÝ�5˜4‘< ¤¨¤Ñ,Ô,ˆØ��t�Œ}ˆÝ×<Ò<¸TÀ3ÑGÔG‰	ˆˆ6Ý! '¨7Ñ3Ô3ð 	Jð 	J‰NˆF�FÝ˜6 4¨!¨V°V¸EÀ$¸JÔ5GÐ)HÑIÔIÐIÐIØˆvˆvØ€Nr2   c                ó`  — t          | | j        j        d         |j        ¦  «        }|�&t                               ||j        |j        ¦  «        n$t                               |j        |j        ¦  «        }t          j        }t          j	         
                     ||¦  «         || j        ¦  «         ||¦  «         ||¦  «         ||¦  «         ||¦  «        t          j        |j                 t          j	                             |j        j        ¦  «        ¦  «        }|st          d¦  «        ‚|S )Nr   z8Comfy Kitchen rejected the shared-input INT8 output tile)r±   r”   r]   rF   rô   Ú_gemm_vector_argrE   Ú_empty_cuda_tensorÚ_wrap_for_dlpackr5   Úcutlass_int8_dequantÚDTYPE_TO_CODEr+   Ú_cuda_getCurrentRawStreamÚindexÚRuntimeError)	r—   r¦   r   rY  rh   r¥   Úbias_argÚwrapÚuseds	            r   rÐ   rÐ   Û  s  € Ý˜& &¤,Ô"4°QÔ"7¸¼ÑDÔD€EØJNÐJZ•×)Ò)¨$°´
¸C¼IÑFÔFÐFÝ×0Ò0°´¸S¼YÑGÔGð åÔ$€DÝŒ;×+Ò+ØˆˆQ‰Œ���f”lÑ#Ô# T T¨&¡\¤\°4°4¸±;´;ÀÀÀXÁÄÐPTÐPTÐUXÑPYÔPYÝÔ˜sœyÔ)­5¬8×+MÒ+MÈcÌjÔN^Ñ+_Ô+_ñaô a€Dð ð WÝÐUÑVÔVÐVØ€Jr2   c           
     óô  ‡ ‡‡— t          ‰ d         dd¦  «        Št          j        ¦   «         o”t          j                             ¦   «          out          j        ‰¦  «         o`‰j        oY‰j        dk    oN‰ 	                    ¦   «         o:t          ‰ ¦  «        ‰j        d         k    ot          ˆˆ ˆfd„‰ D ¦   «         ¦  «        }t          dk    �r‰|�r†‰dv �r�‰j        t          j        t          j        fv �ra‰j        d         dz  dk    �rL‰                     ¦   «         dz  t$          k    �r+‰                     ¦   «         dz  dk    �r‰j        d	         }t(                               ‰                     d
‰j        d         ¦  «        ‰t/          ‰ d         ‰d         ‰¦  «        ¦  «        }‰                     ‰j        d         |‰ d         j        f¦  «        }t5          t7          ‰ ‰                     ¦   «         |                     ¦   «         ¦  «        ¦  «        D ]9\  }\  }}}	t;          |||                     ||z  |d	z   |z  ¦  «        |	f¦  «         Œ:|S |o›‰dk    o•t?          j         ¦   «         o‚tC          ¦   «         ottD          om‰j        t          j        k    oX‰j        d	         dk    oG‰j        d         dz  dk    o3‰                     ¦   «         dz  t$          k    o‰ d         j        dz  dk    }
|
s>t          j#        d„ t7          ‰ ‰                     ¦   «         ¦  «        D ¦   «         ¦  «        S ddl$m%}  |‰d¦  «                             d
‰j        d
         ¦  «        }tL           '                    |¦  «        \  }}~| (                    ‰¦  «        |                     ‰j        d         ‰j        d	         d
¦  «        }}‰                     ‰j        d         ‰j        d	         ‰ d         j        f¦  «        }t7          ‰ ‰                     ¦   «         |                     ¦   «         |                     ¦   «         |                     ¦   «         ¦  «        D ]\  }}}}}	t;          |||||	f¦  «         Œ|S )a¤  Independent small projections with batched input preparation and normal module hooks.

    Group-64 ConvRot normally launches rotation and quantization for each module.
    Kitchen retains its BF16 rotation boundary but prepares all input rows together;
    WGPKernels prepare all rows at once for their path (rows are independent). No
    weights are packed or cached, so MMGP retains their lifetime and residency.
    r   r0  r|   c              3  óx  •K  — | ]´}t          |d d¦  «        ‰k    ošt          t          |dd¦  «        dd¦  «        dk    ov|j        ‰j        d         k    o`|j        ‰d         j        k    oJt          |dd¦  «         o8t          |dd¦  «         o&t          |d	d¦  «         pt          |d
d¦  «        i k    V — ŒµdS )r0  r   r×   Nr	  Úqint8_convrotrR  r>  rC  r.  r/  )rÄ   r6  r]   r9  )rx   rc   r§   rÛ   rg   s     €€€r   r
  z!linear_grouped.<locals>.<genexpr>ó  s  øè è € ð cð cð QWõ ˜fÐ&;¸QÑ?Ô?À5ÒHð LÝ!¥'¨&°.À$Ñ"GÔ"GÈÐQUÑVÔVÐZiÒiðLà Ô,°´¸´
Ò:ðLà?EÔ?RÐV]Ð^_ÔV`ÔVmÒ?mðLõ & fÐ.DÀdÑKÔKÐKðLõ & fÐ.IÈ4ÑPÔPÐPð	Lõ
  ' vÐ/EÀtÑLÔLÐLð KÝ% f¨o¸tÑDÔDÈÒJðcð cð cð cð cð cr2   r   r1  rR  rŽ   rG   r�   rí   r   rš   c                ó*   — g | ]\  }} ||¦  «        ‘ŒS rw   rw   )rx   rc   Úparts      r   rM  z"linear_grouped.<locals>.<listcomp>  s$   € ÐVÐVÐV©\¨V°T˜F˜F 4™LœLÐVÐVÐVr2   r'  ))rÄ   r+   r4  r,   r´   r   r‘   r“   Úndimr·   Úlenr]   r_   r#   rE   r[   rZ   Únumelrð   r�   r¢   rT  r²   rH  rñ   r9  Ú	enumeraterU  ÚunbindrE  rf   r   r7  r‰   Ú_direct_cutlassÚstackr)  r(  rô   rõ   Ú
reshape_as)rÛ   rg   ÚcommonÚtokensrÑ   rX  rb  rc   rk  rh   Ú	supportedr(  r+  r   rY  Ú	quantizedr¥   r§   s   ``               @r   Úlinear_groupedrx  è  s�  øøø€ õ �G˜A”JÐ 5°qÑ9Ô9€EÝÔ-Ñ/Ô/ð c½¼×8SÒ8SÑ8UÔ8UÐ4Uð cÕ^dÔ^tÐuvÑ^wÔ^wÐZwð cÐ|}ô  }Fð cØ”&˜A’+ðcØ"#§/¢/Ñ"3Ô"3ðcÝ8;¸G¹¼ÈÌÐPQÌ
Ò8Rðcåð cð cð cð cð cð cð [bðcñ cô cñ cô cð õ 	�LÒ Ñ  VÑ °¸Ð0FÑ0FÈ1Ì7ÕW\ÔWeÕglÔgtÐVuÐKuÑKuÐz{ô  {Bð  CDô  {Eð  HJñ  {Jð  NOò  {Oñ  {OØ—’‘	”	˜A‘¥Ò/Ñ/°A·J²J±L´LÀ2Ñ4EÈÒ4JÑ4JØ”˜”ˆÝ—<’< §	¢	¨"¨a¬g°a¬jÑ 9Ô 9¸5Å)ÈGÐTUÌJÐXYÐZ[ÔX\Ð^cÑBdÔBdÑeÔeˆØ—’˜aœg aœj¨&°'¸!´*Ô2IÐJÑKÔKˆÝ*3µC¸ÀÇÂÁÄÈVÏ]Ê]É_Ì_Ñ4]Ô4]Ñ*^Ô*^ð 	eð 	eÑ&ˆEÑ&�F˜D #Ý˜6 4¨(¯-ª-¸À¹ÈÐQRÉÐV\ÑH\Ñ*]Ô*]Ð_bÐ)cÑdÔdÐdÐdØˆØð Z˜E RšKð Z­MÔ,KÑ,MÔ,Mð ZÕRaÑRcÔRcð ZÕhwð ZØ”W¥¤Ò.ðZØ34´7¸1´:À²>ðZØFGÄgÈaÄjÐSUÁoÐYZÒFZðZà—W’W‘Y”Y ‘]¥nÒ4ðZà9@À¼Ô9PÐSTÑ9TÐXYÒ9Yð ð ð XÝŒ{ÐVÐV½SÀÈ!Ï(Ê(É*Ì*Ñ=UÔ=UÐVÑVÔVÑWÔWÐWØ=Ð=Ð=Ð=Ð=Ð=Ø Ð   BÑ'Ô'×/Ò/°°A´G¸B´KÑ@Ô@€GÝ×.Ò.¨wÑ7Ô7�I€A€vØØ—’˜Q‘” §¢°´¸´
¸A¼GÀA¼JÈÑ!KÔ!K€v€AØ�[Š[˜!œ' !œ* a¤g¨a¤j°'¸!´*Ô2IÐJÑKÔK€FÝ/2°7¸A¿HºH¹J¼JÈÏÊÉ
Ì
ÐTZ×TaÒTaÑTcÔTcÐek×erÒerÑetÔetÑ/uÔ/uð >ð >Ñ+ˆ��i ¨Ý�v˜t i°¸Ð%<Ñ=Ô=Ð=Ð=Ø€Mr2   c	                óÒ	  — t           st          d¦  «         da t          ||j        j        d         | j        ¦  «        }	|                      d| j        d         ¦  «        }
|j        j        \  }}|�`|j        g | j        dd…         ¢|‘R k    s4|j        | j        k    s$|j        | j        k    s|                     ¦   «         st          d¦  «        ‚||€"|  
                    |
j        d         |f¦  «        n|                     d|¦  «        }}|�|                     d|¦  «        nd}|dz   |
                     ¦   «         s"|
j        d         |
                     ¦   «         z  ndz   }t          dt          |z  dz  dz  ¦  «        }t          j        }t           j                             | j        j        ¦  «        }|�&t                               || j        | j        ¦  «        n$t                               | j        | j        ¦  «        }t                               ||| j        | j        ¦  «        }|�&t                               || j        | j        ¦  «        nd}t/          d|
j        d         |¦  «        D �]3}t1          ||z   |
j        d         ¦  «        }|
||…                              ¦   «         }t!          j        ||z
  |f| j        t           j        ¬	¦  «        }t!          j        ||z
  d
f| j        t           j        ¬	¦  «        }t          j                              ||¦  «         ||¦  «         ||¦  «        ddt                               |¦  «        d ||¦  «        t?          |¦  «        |¦
  «
         |�›t          j                               ||¦  «         ||j        ¦  «         ||¦  «         ||	¦  «         ||¦  «         ||¦  «         ||||…         ¦  «         ||||…         ¦  «        t          j!        | j                 |¦
  «
        }n~t          j         "                     ||¦  «         ||j        ¦  «         ||¦  «         ||	¦  «         ||¦  «         ||||…         ¦  «        t          j!        | j                 |¦  «        }|stG          d¦  «        ‚~~~�Œ5|�$t           j$        j%         &                    |¦  «          |j        g | j        dd…         ¢|‘R Ž S )zLBound temporary quantization/output storage while retaining Kitchen fusions.z_[INT8] Comfy Kitchen fused normalization/activation and residual linear kernels are being used.Tr   r�   NzQFused INT8 output must match the output shape, dtype and device and be contiguousr   rH   ©rF   rE   rG   rC   Fz7Comfy Kitchen rejected the fused SM120 INT8 output tile)'rÊ   r/   r±   r”   r]   rF   r²   rE   r·   r~   rñ   Úelement_sizerï   rð   rô   r^  r+   r5   ra  rb  r\  r]  Ú_act_weight_argrò   ró   Ú
contiguousÚemptyr   rY   rV  Ú_input_act_coderö   Úcutlass_int8_dequant_residualr`  r_  rc  r¹   rº   r»   )r–   r—   r¦   rÌ   r­   r¼   r¯   r°   rh   r¥   rg   r½   rü   ÚsuppliedÚrÚ	row_bytesrf   re  Ústreamrd  Úact_argÚresidual_argrý   rþ   rÿ   r   r  rf  s                               r   rË   rË     sš  € õ ð ÝÐoÑpÔpÐpØˆÝ˜& &¤,Ô"4°QÔ"7¸¼ÑFÔF€EØ�Š�b˜%œ+ bœ/Ñ*Ô*€AØŒ<Ô�D€A€qØ
€˜CœIÐ)?¨5¬;°s¸°sÔ+;Ð)?¸QÐ)?Ð)?Ò?Ð?À3Ä9ÐPUÔP[ÒC[ÐC[Ø"œz¨U¬\Ò9Ð9À×ARÒARÑATÔATÐ9ÝÐlÑmÔmÐmØ¸S¸[˜Ÿš¨!¬'°!¬*°a¨Ñ9Ô9Ð9ÈcÏkÊkÐZ\Ð^_ÑN`ÔN`ˆc€HØ#+Ð#7ˆ×Ò˜˜QÑÔÐ¸T€Aà�A‘¸q¿ºÑ?PÔ?PÐW˜œ œ q§~¢~Ñ'7Ô'7Ñ7Ð7ÐVWÑX€IÝˆr•N iÑ/°BÑ6¸Ñ;Ñ<Ô<€DÝÔ$€DÝŒX×/Ò/°´Ô0BÑCÔC€FØNRÐN^•×)Ò)¨$°´¸e¼kÑJÔJÐJÝ×0Ò0°´¸u¼{ÑKÔKð å×&Ò& y°*¸e¼lÈEÌKÑXÔX€Gà�}õ ×-Ò-¨n¸e¼lÈEÌKÑXÔXÐXØ*.ð å�q˜!œ' !œ* dÑ+Ô+ð ñ ˆÝ�5˜4‘< ¤¨¤Ñ,Ô,ˆØ��t�Œ}×'Ò'Ñ)Ô)ˆÝŒK˜˜e™ Q˜°´ÅEÄJÐOÑOÔOˆÝŒ[˜$˜u™* a˜°´ÅUÄ]ÐSÑSÔSˆÝŒ×3Ò3ØˆD�‰JŒJ˜˜˜Q™œ   b¡¤¨3°Ý×$Ò$ YÑ/Ô/°°D°D¸±M´MÅ5ÈÁ>Ä>ÐSYñ	[ô 	[ð 	[ð ˆ=Ý”;×<Ò<Ø��Q‘”˜˜˜fœlÑ+Ô+¨T¨T°"©X¬X°t°t¸E±{´{ÀDÀDÈÁNÄNØ��\Ñ"Ô" D D¨¨5°¨:¬Ñ$7Ô$7¸¸¸cÀ%ÈÀ*¼oÑ9NÔ9NÝÔ& u¤{Ô3°Vñ=ô =ˆDˆDõ
 ”;×3Ò3Ø��Q‘”˜˜˜fœlÑ+Ô+¨T¨T°"©X¬X°t°t¸E±{´{ÀDÀDÈÁNÄNØ��S˜˜t˜”_Ñ%Ô%¥xÔ'=¸e¼kÔ'JÈFñTô TˆDð ð 	ZÝÐXÑYÔYÐYØˆr�4‘4ØÐÝŒÔ×.Ò.¨xÑ8Ô8Ð8Øˆ3Œ;Ð,˜œ C R CÔ(Ð,¨!Ð,Ð,Ð,Ð,r2   c                ó¾  — | j         \  }}|j         d         }||d|                      ¦   «         z   z  z   dz   }t          r&|d|z  |                      ¦   «         z  d|dz  z  z   z  }t          dt          |z  ¦  «        }	|	dk    r|	dz  dz  }	||	k    r%t
                               | |||| j        |¬¦  «        S t          rš| j        t          j
        t          j        fv r{|                      ¦   «         rg|                     ¦   «         rS|dz  dk    rJ|dz  dk    rAd|cxk    rd	k    r4n n1|rt
                               | |d¦  «        rt          | ||||¦  «        S |                      ||f¦  «        }
t!          d||	¦  «        D ]]}t#          ||	z   |¦  «        }|
||…                              t
                               | ||…         |||| j        |¬¦  «        ¦  «         Œ^|
S )
Nr   r   rR  rC   rG   rH   ©r¦   rJ   rK   rš   r3  )r]   r{  Ú_kitchen_hiprï   rð   rô   r\   rE   rq  r+   r[   rZ   r·   r:  Ú_cutlass_linear_chunkedrñ   rò   ró   rú   )rg   r—   r¥   r¦   rK   rû   rü   r½   rƒ  rf   rh   rý   rþ   s                r   Ú_linear_implr‹  C  s"  € ØŒ7�D€A€qØŒ�QŒ€Aà�A˜˜QŸ^š^Ñ-Ô-Ñ-Ñ.Ñ.°Ñ2€IÝð ?à�Q˜‘U˜QŸ^š^Ñ-Ô-Ñ-°°Q¸#±X±Ñ>Ñ>ˆ	Ýˆq•. IÑ-Ñ.Ô.€DØˆr‚z€zØ�r‰z˜B‰ˆØˆD‚y€yÝ×#Ò# A v¨u¸4È1Ì7Ð\cÐ#ÑdÔdÐdÝð H˜AœG­¬½¼Ð'FÐFÐFØ—’Ñ!Ô!ð GØ&,×&:Ò&:Ñ&<Ô&<ð GØABÀQÁÈ!ÂÀØ�C‘˜1’� ¨Ð!2Ð!2Ò!2Ð!2¨UÒ!2Ð!2Ð!2Ð!2Ð!2Øð "3Ý (× JÒ JÈ1ÈaÐQTÑ UÔ Uð "3å& q¨&°%¸¸wÑGÔGÐGØ
�+Š+�q˜!�fÑ
Ô
€CÝ�q˜!˜TÑ"Ô"ð Zð ZˆÝ�5˜4‘< Ñ#Ô#ˆØˆE�$ˆJŒ×Ò�h×2Ò2Øˆe�DˆjŒM˜6 5¨t¸q¼wÐPWð 3ñ Yô Yñ 	Zô 	Zð 	Zð 	Zà€Jr2   c                ó¦  — | j         \  }}t          dt          |dz   z  dz  dz  ¦  «        }|                      ||j         d         f¦  «        }t          j        }	|�&t                               || j        | j        ¦  «        n$t           	                    | j        | j        ¦  «        }
t          j                             | j        j        ¦  «        }t          d||¦  «        D ]ï}t          ||z   |¦  «        }|r't                               | ||…         d¦  «        \  }}n%t                               | ||…         ¦  «        \  }}t          j                              |	|¦  «         |	|¦  «         |	|¦  «         |	|¦  «         |	|
¦  «         |	|||…         ¦  «        t          j        | j                 |¦  «        }|st)          d¦  «        ‚~~Œð|S )zîWrite Kitchen's CUTLASS result directly into output slices on supported CUDA GPUs.

    Only quantized row tiles and their scales occupy scratch space. Avoiding a
    temporary output permits wide GEMMs without tiny, slow row chunks.
    rH   r   r   NrC   z3Comfy Kitchen rejected the INT8 CUTLASS output tile)r]   rï   rð   rñ   rô   r^  r\  rF   rE   r]  r+   r5   ra  rb  rò   ró   rV  rõ   r_  r`  rc  )rg   r—   r¥   r¦   rK   rû   rü   rf   rh   re  rd  r„  rý   rþ   r   rY  rf  s                    r   rŠ  rŠ  ]  sº  € ð Œ7�D€A€qÝˆr•N q¨1¡uÑ-°"Ñ4°rÑ9Ñ:Ô:€DØ
�+Š+�q˜&œ, qœ/Ð*Ñ
+Ô
+€CÝÔ$€DØFJÐFV•×)Ò)¨$°´¸!¼'ÑBÔBÐBÝ×0Ò0°´¸1¼7ÑCÔCð åŒX×/Ò/°´´Ñ?Ô?€FÝ�q˜!˜TÑ"Ô"ð ð ˆÝ�5˜4‘< Ñ#Ô#ˆØð 	FÝ ×@Ò@ÀÀ5ÈÀ:ÄÐPSÑTÔT‰IˆAˆvˆvå ×6Ò6°q¸¸t¸´}ÑEÔE‰IˆAˆvÝŒ{×/Ò/ØˆD�‰GŒG�T�T˜&‘\”\ 4 4¨¡<¤<°°°e±´¸d¸dÀ8¹n¼nØˆD��U˜4�Z”Ñ!Ô!¥8Ô#9¸!¼'Ô#BÀFñLô Lˆð ð 	VÝÐTÑUÔUÐUØˆvˆvØ€Jr2   c                óh  ‡‡‡— | j         t          f}t          dk    st          st          j        s	|t          v rdS | j        Št          j	         
                    ‰¦  «        dk    �r²t          j	                             ‰¦  «        j        }dd|z  dz
  z  dcŠŠt          j        }t          j        ¦   «         5  t          j        ‰‰ft          j        ‰¬¦  «        }t          j        ‰dft          j        ‰¬¦  «        }ˆˆˆfd	„d
D ¦   «         }|                      t          j	                             ‰¦  «        ¦  «         |D ]Œ\  }}}	}
t          j                              ||¦  «         ||¦  «         ||¦  «         ||	¦  «         ||
¦  «        t          j        t          j                 || j         ¦  «        st3          d|› d�¦  «        ‚Œ�t          j	                             ‰¦  «                             | ¦  «         ddd¦  «         n# 1 swxY w Y   t                               |¦  «         dS )a�  Kitchen's stream-K GEMMs keep a workspace per stream that they allocate on first need and enlarge later: a CUDA graph capture
    cannot allocate it, and an enlargement would free the workspace of the graphs captured before. The capture stream gets the largest
    one once (a block per SM with the widest stream-K tile); a capture that would still need more fails instead of replaying stale memory.r   N©rš   r   rQ  rR  rG   i   rD   c           
     óä   •— g | ]l\  }}|t          j        |‰ft           j        ‰¬ ¦  «        t          j        |t           j        ‰¬ ¦  «        t          j        ‰|ft           j        ‰¬ ¦  «        f‘ŒmS )rD   )r+   rW   r   rX   rY   r~  r[   )rx   Úconfigr½   rF   rü   rû   s      €€€r   rM  z*prepare_capture_stream.<locals>.<listcomp>ˆ  s™   ø€ ð yð yð yÙT]ÐTZÐ\]ð �eœk¨1¨a¨&½¼
È6ÐRÑRÔRÕTYÔT^Ð_`ÕhmÔhuð  Eð  UFñ  UFô  UFÝ”k 1 a &µ´ÀvÐNÑNÔNðPð yð yð yr2   ))rì   rQ  )é   rC   zFComfy Kitchen could not prepare the stream-K workspace of tile config z for CUDA graphs)Úcuda_streamÚrevisionr#   r‰  rô   Ú_DISABLE_CUTLASS_INT8Ú_prepared_capture_streamsrF   r+   rB   rO   rm   Úmulti_processor_countr^  rU   rW   r   rX   rY   Úwait_streamÚcurrent_streamr5   Úcutlass_int8_dequant_configr`  r[   rc  Úadd)r„  ÚkeyÚsmsre  ÚaÚxsÚcallsr�  ÚbÚwsrh   rF   rü   rû   s              @@@r   Úprepare_capture_streamr¢  y  s¯  øøø€ ð Ô�xÐ
(€CÝ�9ÒÐ¥ÐµÔ0NÐÐRUÕYrÐRrÐRrØˆØŒ]€FÝ„z×'Ò'¨Ñ/Ô/°6Ò9Ñ9ÝŒj×.Ò.¨vÑ6Ô6ÔLˆØ�a˜#‘g ‘kÑ" Cˆˆˆ1ÝÔ(ˆÝÔ!Ñ#Ô#ð 
	Bð 
	BÝ”˜Q ˜F­%¬*¸VÐDÑDÔDˆAÝ”˜Q ˜F­%¬-ÀÐGÑGÔGˆBðyð yð yð yð yð yØawðyñ yô yˆEà×Ò�uœz×8Ò8¸Ñ@Ô@ÑAÔAÐAØ&+ð Jð JÑ"�˜˜2˜sÝ”{×>Ò>¸t¸tÀA¹w¼wÈÈÈQÉÌÐQUÐQUÐVXÑQYÔQYÐ[_Ð[_Ð`bÑ[cÔ[cÐeiÐeiÐjmÑenÔenÝ?GÔ?UÕV[ÔVdÔ?eÐgmÐouô  pBñCô Cð Jå&ð  (IÐpvð  (Ið  (Ið  (Iñ  Jô  Jð  JðJõ ŒJ×%Ò% fÑ-Ô-×9Ò9¸&ÑAÔAÐAð
	Bð 
	Bð 
	Bñ 
	Bô 
	Bð 
	Bð 
	Bð 
	Bð 
	Bð 
	Bð 
	Bøøøð 
	Bð 
	Bð 
	Bð 
	Bõ ×!Ò! #Ñ&Ô&Ð&Ð&Ð&s   Â7E
HÈHÈHc                 ó(  — 	 t          j        dt           j                             ¦   «         ¦  «        } t          j        d| t           j        ¬¦  «                             d¦  «                             d¦  «                             d¦  «         	                    dd¦  «        }t          j        d	| t           j
        ¬¦  «        d
z  dz
                       t           j        ¦  «         	                    dd¦  «        }t          j        dd| t           j        ¬¦  «        }t           j        t           j        fD ]³}|                     |¦  «        }d|                     |¦  «        fD ]„}dD ]}t                                ||||||¬¦  «        }t%          |||||¦  «        }	t          j        |	¦  «                             ¦   «         rt          j        |	|¦  «        st-          d¦  «        ‚Œ€Œ…Œ´dS # t,          $ r}
t/          d|
› d�¦  «         Y d}
~
dS d}
~
ww xY w)zMCheck the actual output-buffer kernel at backend setup, never during capture.rB   i A  rz  éû   é}   rQ  éA   rC   i  éÿ   é   é  )r©  g�@ €?NrI   rˆ  z/output-buffer kernel did not match Kitchen INT8Tz([INT8] Direct Output Tiles Unavailable: z". Using existing Kitchen batching.F)r+   rF   rB   rV   ÚarangerY   Ú
remainder_Úsub_Údiv_r²   Úint32r³   r   ÚfullrZ   r[   rô   r\   rŠ  r^   r_   Úequalrc  r/   )rF   rg   r—   r¥   rE   Úinputsr¦   rK   ÚexpectedÚactualri   s              r   Ú_probe_direct_cutlassr´  “  s  € ðÝ”˜f¥e¤j×&?Ò&?Ñ&AÔ&AÑBÔBˆÝŒL˜¨&½¼ÐFÑFÔF×QÒQÐRUÑVÔV×[Ò[Ð\_Ñ`Ô`×eÒeÐfiÑjÔj×rÒrÐsuÐwzÑ{Ô{ˆÝ”,˜y°½u¼{ÐKÑKÔKÈcÑQÐTWÑW×[Ò[Õ\aÔ\fÑgÔg×oÒoÐpsÐuxÑyÔyˆÝ”
˜6 7°6ÅÄÐOÑOÔOˆÝ”m¥U¤^Ð4ð 	^ð 	^ˆEØ—T’T˜%‘[”[ˆFØ˜uŸxšx¨™œÐ/ð ^ð ^�Ø,ð ^ð ^�GÝ'×3Ò3°F¸FÀEÐPTÐ`eÐovÐ3ÑwÔw�HÝ4°V¸VÀUÈDÐRYÑZÔZ�FÝ œ>¨&Ñ1Ô1×5Ò5Ñ7Ô7ð ^½u¼{È6ÐS[Ñ?\Ô?\ð ^Ý*Ð+\Ñ]Ô]Ð]ð^ð^ð^ð ˆtøÝð ð ð ÝÐ`¸Ð`Ð`Ð`ÑaÔaÐaØˆuˆuˆuˆuˆuøøøøðøøøs   ‚G%G) Ç)
HÇ3HÈHc                 óô  — t           rd S t          j                             dd¬¦  «        dd„¦   «         } | j        d„ ¦   «         }t          j                             dd¬¦  «        dd„¦   «         }|j        d„ ¦   «         }t          j                             dd¬¦  «        dd„¦   «         }|j        d„ ¦   «         }t          j                             dd¬¦  «        dd„¦   «         }|j        d„ ¦   «         }da d S )Nzwan2gp_wgp::linearrw   )Úmutates_argsrg   útorch.Tensorr—   r¥   r¦   útorch.Tensor | Noner§   Úintr¨   ÚboolÚreturnc                ó*   — t          | |||||¦  «        S r4   )r©   r¤   s         r   Úwgp_linear_opz$_register_ops.<locals>.wgp_linear_op¯  s   € õ ˜˜6 5¨$°°wÑ?Ô?Ð?r2   c                ó\   — |                       | j        d         |j        d         f¦  «        S ©Nr   ©rñ   r]   r¤   s         r   Úwgp_fakez_register_ops.<locals>.wgp_fake´  ó%   € à�{Š{˜AœG AœJ¨¬°Q¬Ð8Ñ9Ô9Ð9r2   zwan2gp_kitchen::linearrK   c                ó(   — t          | ||||¦  «        S r4   )r‹  ©rg   r—   r¥   r¦   rK   s        r   r£   z_register_ops.<locals>.linear¸  s   € õ ˜A˜v u¨d°GÑ<Ô<Ð<r2   c                ó\   — |                       | j        d         |j        d         f¦  «        S r¿  rÀ  rÄ  s        r   Úfakez_register_ops.<locals>.fake½  rÂ  r2   z wan2gp_kitchen::blockwise_linearc                ó&   — t          | |||¦  «        S r4   )r  ©rg   r—   r¥   r¦   s       r   r  z'_register_ops.<locals>.blockwise_linearÁ  s   € õ & a¨°¸Ñ=Ô=Ð=r2   c                ó\   — |                       | j        d         |j        d         f¦  «        S r¿  rÀ  rÈ  s       r   Úblockwise_fakez%_register_ops.<locals>.blockwise_fakeÆ  rÂ  r2   zwan2gp_kitchen::residual_linearc                ó&   — t          | |||¦  «        S r4   )ré   rÈ  s       r   rè   z&_register_ops.<locals>.residual_linearÊ  s   € õ % Q¨°°tÑ<Ô<Ð<r2   c                ó\   — |                       | j        d         |j        d         f¦  «        S r¿  rÀ  rÈ  s       r   Úresidual_fakez$_register_ops.<locals>.residual_fakeÏ  rÂ  r2   T)rg   r·  r—   r·  r¥   r·  r¦   r¸  r§   r¹  r¨   rº  r»  r·  )rg   r·  r—   r·  r¥   r·  r¦   r¸  rK   rº  r»  r·  )
rg   r·  r—   r·  r¥   r·  r¦   r¸  r»  r·  )Ú_ops_registeredr+   ÚlibraryÚ	custom_opÚregister_fake)r½  rÁ  r£   rÆ  r  rÊ  rè   rÍ  s           r   Ú_register_opsrÒ  ©  sz  € åð Øˆõ „]×ÒÐ1ÀÐÑCÔCð@ð @ð @ñ DÔCð@ð Ô ð:ð :ñ !Ô ð:õ „]×ÒÐ5ÀBÐÑGÔGð=ð =ð =ñ HÔGð=ð Ôð:ð :ñ Ôð:õ „]×ÒÐ?ÈbÐÑQÔQð>ð >ð >ñ RÔQð>ð Ô#ð:ð :ñ $Ô#ð:õ „]×ÒÐ>ÈRÐÑPÔPð=ð =ð =ñ QÔPð=ð Ô"ð:ð :ñ #Ô"ð:ð €O€O€Or2   )rK   c               óð  — |r˜t           r‘| j        d         dk    r€t          �Ct          j                             ¦   «         s%t          j        | ¦  «        st          | ||¦  «        S ddl	m
} t          j        j                              || d¦  «        ||¦  «        S |j        j        d         }t          ||| j        ¦  «        }|                      d| j        d         ¦  «        }t          j                             ¦   «         st          j        | ¦  «        r.t          j        j                             ||j        |||¦  «        }nt)          ||j        |||¦  «        } |j        g | j        d d…         ¢|‘R Ž S )Nr�   r3  r   r'  rC   )r8  r]   r*  r+   r,   r´   r   r‘   r,  r)  r(  rÝ   Ú
functionalr£   r”   r±   rF   r²   rµ   rç   r‹  )	r–   r—   r¦   rK   r(  r½   r¥   rg   rh   s	            r   Úkitchen_linearrÕ  Ö  sj  € Øð X•>ð X e¤k°"¤o¸Ò&=Ð&=Ý Ð,µU´^×5PÒ5PÑ5RÔ5RÐ,ÝÔ.¨uÑ5Ô5ð -å'¨¨v°tÑ<Ô<Ð<ð 	BÐAÐAÐAÐAÐAÝŒxÔ"×)Ò)Ð*<Ð*<¸UÀCÑ*HÔ*HÈ&ÐRVÑWÔWÐWØŒÔ˜1Ô€AÝ˜& ! U¤\Ñ2Ô2€EØ�Š�b˜%œ+ bœ/Ñ*Ô*€AÝ„~×"Ò"Ñ$Ô$ð B­Ô(>¸uÑ(EÔ(Eð BÝŒiÔ&×-Ò-¨a°´¸uÀdÈGÑTÔTˆˆå˜1˜fœl¨E°4¸ÑAÔAˆØˆ3Œ;Ð,˜œ C R CÔ(Ð,¨!Ð,Ð,Ð,Ð,r2   c                óþ  — t          |¦  «        t          j        u rÑ|j        rÊ|j        t          j        t          j        t          j        fv r |j        j        r”|j        j        t          j	        k    rz|j
        d         dk    st          sb|j        j
        d         t          rdndz  dk    r@|j        j
        d         dz  dk    r'|                      ||¦  «         t          |||¦  «        S t          | |||¦  «        S )Nr�   r3  rŽ   r   r   )r�   r+   r�   r“   rE   rZ   r[   rY   r”   r   r]   r8  r‰  Úsave_for_backwardrÕ  Ú_original_forward)Úctxr–   r—   r¦   s       r   Ú_quanto_forwardrÚ  é  sç   € ÝˆU‰Œ•u”|Ð#Ð#¨¬Ð#Ø”¥¤­u¬~½u¼}ÐMÐMÐMØ”Ô$ð NØ)/¬Ô);½u¼zÒ)IÐ)IØ”˜R” EÒ)Ð)µÐ)Ø”Ô" 2Ô&µÐ*C¨"¨"À!ÑDÈÒIÐIÈfÌlÔN`ÐabÔNcÐfgÑNgÐklÒNlÐNlØ×Ò˜e VÑ,Ô,Ð,Ý˜e V¨TÑ2Ô2Ð2Ý˜S %¨°Ñ6Ô6Ð6r2   c                ób  — t          ||¦  «        rŽ|                      ||¦  «         |j                             d¦  «        }|�t	          ||g|¢R Ž S |j                             d¦  «        }|€t          |||¦  «        S |\  }}}}	}
}t          |||||||	|
|¬¦	  «	        S t          | |||¦  «        S )NrC  r>  rÉ   )r˜   r×  r"  r#  rÓ   r¿   rØ  )rÙ  r–   r—   r¦   rÑ   r@  rÌ   r­   r¼   r¯   r°   rh   s               r   Ú_wgp_quanto_forwardrÜ  ô  sâ   € Ý�U˜FÑ#Ô#ð 
BØ×Ò˜e VÑ,Ô,Ð,Ø”?×&Ò&Ð'BÑCÔCˆØÐÝ& v¨tÐ?°hÐ?Ð?Ð?Ð?Ø”×$Ò$Ð%;Ñ<Ô<ˆØˆ>Ý˜e V¨TÑ2Ô2Ð2ØHNÑEˆ	�:˜w¨°.À#Ý˜% ¨°9ÈÐ]dÐowØ)7¸SðBñ Bô Bð 	Bå˜S %¨°Ñ6Ô6Ð6r2   c                  ój   — e Zd ZdZed„ ¦   «         Zed„ ¦   «         Zed„ ¦   «         Zed„ ¦   «         ZdS )Ú_WgpRowProjectionaÔ  mmgp row projection (offload.linear_rows) of the INT8 ConvRot and Quanto INT8 layers computed with WGPKernels: the input is
    prepared once (rotated and quantized for the path chosen with the first module's weight), shared by the modules of the same rotation
    group and precision, and each range of output rows is the same linear on a view of the weight rows. The prepared input is at most as
    large as the input it replaces (prepare_linear_input releases it).c                ó  — t          | dd ¦  «        }t          dk    sN|�L|j        dvsCt          j                             ¦   «         s%t          j        |¦  «        st          | dd ¦  «        �dS |j        dk    rt          | dd ¦  «        �dS | j        }t          ||¦  «        ok| 
                    ¦   «         oW|                     ¦   «         dz  d	k    o<|j         
                    ¦   «         o#|j        dk    p|j        d
         | j        z  d	k    S )Nr×   r   ©rÖ   ri  Úactivation_qtypeFrÖ   Ú_router_forward_implrŽ   r   r�   )rÄ   r#   r	  r+   r,   r´   r   r‘   ræ   r˜   r·   r�   r”   r]   r0  )rc   rg   Úqtyper—   s       r   Úsupportsz_WgpRowProjection.supports  s.  € å˜ °Ñ5Ô5ˆÝ˜Ò$Ð$¨¨¸¼ÐKeÐ9eÐ9eÕinÔiw÷  jEò  jEñ  jGô  jGÐ9eÝÔ)¨!Ñ,Ô,ð :fÝ07¸Ð@RÐTXÑ0YÔ0YÐ0eØ�5ØŒ:˜Ò Ð ¥W¨VÐ5KÈTÑ%RÔ%RÐ%^Ø�5Ø”ˆÝ˜a Ñ(Ô(ð ]¨Q¯_ª_Ñ->Ô->ð ]À1Ç:Â:Á<Ä<ÐRTÑCTÐXYÒCYð ]Ð^dÔ^j×^xÒ^xÑ^zÔ^zð ]Ø”Z 7Ò*Ð[¨a¬g°b¬k¸FÔ<VÑ.VÐZ[Ò.[ð	^r2   c                óZ   — | j         j        dk    r| j        nd}d|t          | d¦  «        fS )Nri  r   Úwgp_rowsrØ   )r×   r	  r0  rß   ©rc   r§   s     r   r›  z_WgpRowProjection.key  s8   € à.4Ô.AÔ.FÈ/Ò.YÐ.Y�Ô*Ð*Ð_`ˆØ˜5¥'¨&Ð2JÑ"KÔ"KÐKÐKr2   c           
     ó–   — t                                | ¦  «        \  }}}t                               ||t	          | |||¦  «        ¦  «        S r4   )rÞ  r›  r¢   rT  rH  )rc   rg   ry   r§   r¨   s        r   rT  z_WgpRowProjection.prepare  s@   € å-×1Ò1°&Ñ9Ô9Ñˆˆ5�'Ý�|Š|˜A˜u¥i°¸¸5À'Ñ&JÔ&JÑKÔKÐKr2   c                ó$  — | j         | j        }}t          ||j        j        d         |j        j        ¦  «        ||…         }|€d n!|||…                              |j        ¦  «        }t           	                    ||j        ||…         ||¦  «        S r¿  )
ræ   r¦   r±   r”   r]   rF   r³   rE   r¢   rÏ   )rc   rÑ   rý   rþ   r—   r¦   r¥   s          r   rf   z_WgpRowProjection.rows  s†   € à”~ v¤{�ˆÝ˜f f¤lÔ&8¸Ô&;¸V¼\Ô=PÑQÔQÐRWÐX\ÐR\Ô]ˆØ�|ˆtˆt¨¨e°D¨jÔ)9×)<Ò)<¸X¼^Ñ)LÔ)LˆÝ×#Ò# H¨f¬l¸5À¸:Ô.FÈÈtÑTÔTÐTr2   N©	Ú__name__Ú
__module__Ú__qualname__Ú__doc__Ústaticmethodrä  r›  rT  rf   rw   r2   r   rÞ  rÞ    sš   € € € € € ðJð Jð ð	^ð 	^ñ „\ð	^ð ðLð Lñ „\ðLð ðLð Lñ „\ðLð ðUð Uñ „\ðUð Uð Ur2   rÞ  c                  ój   — e Zd ZdZed„ ¦   «         Zed„ ¦   «         Zed„ ¦   «         Zed„ ¦   «         ZdS )Ú_Int8RowProjectionz]Preserve PyTorch weight-only or Triton blockwise activation arithmetic for output row ranges.c                óD  — t          | dd ¦  «        }t          dvsN|�L|j        dvsCt          j        |¦  «        s/t
          j                             ¦   «         st          | dd ¦  «        �dS |j        dk    rt          | dd ¦  «        �dS t          dk    r?|j        d	k    r4t          j        r(t          | d
d¦  «        dk    r|j	        d         dk    rdS t          |¦  «        t
          j        u o3| j        j        j        t
          j        k    o| j        j        |j        k    S )Nr×   )r   r   rà  rá  FrÖ   râ  r   ri  r0  r   rC   r   )rÄ   r#   r	  r   r‘   r+   r,   r´   ÚFUSED_CONVROT_ENABLEDr]   r�   r�   ræ   r”   rE   r   )rc   rg   rã  s      r   rä  z_Int8RowProjection.supports+  s'  € å˜ °Ñ5Ô5ˆÝÐ1Ð1Ð1°U°]ÀeÄjÐXrÐFrÐFrÝÔ)¨!Ñ,Ô,ð GsÝ05´×0KÒ0KÑ0MÔ0Mð GsÝQXÐY_ÐasÐuyÑQzÔQzð  RGØ�5ØŒ:˜Ò Ð ¥W¨VÐ5KÈTÑ%RÔ%RÐ%^Ø�5Ý˜Ò Ð  U¤Z°?Ò%BÐ%BÅvÔGcÐ%BÝ˜FÐ$9¸1Ñ=Ô=ÀÒDÐDÈÌÐQRÌÐWXÊÈØ�5Ý�A‰wŒw�%œ,Ð&Ðw¨6¬>Ô+?Ô+EÍÌÒ+SÐwÐX^ÔXfÔXlÐpqÔpwÒXwÐwr2   c                ób   — t          | dd¦  «        }t          dk    s|rdt          || j        fnd S )Nr0  r   r   Ú	int8_rows)rÄ   r#   r9  rç  s     r   r›  z_Int8RowProjection.key8  s=   € å˜Ð 5°qÑ9Ô9ˆÝFNÐRZÒFZÐFZÐ^cÐFZ��X u¨fÔ.AÐBÐBÐimÐmr2   c                óÞ   — ddl m}  ||t          | dd¦  «        ¦  «        }t          dk    r@|j        r9t          j        |¦  «        s%t
          j                             || j	        ¦  «        S |S )Nr   r'  r0  r   )
r)  r(  rÄ   r#   r“   r   Ú_prefer_native_quanto_pathÚ_TRITON_MODULEÚprepare_blockscale_inputr9  )rc   rg   r(  s      r   rT  z_Int8RowProjection.prepare=  sw   € àAÐAÐAÐAÐAÐAØÐ˜q¥'¨&Ð2GÈÑ"KÔ"KÑLÔLˆÝ˜Ò Ð  Q¤YÐ µvÔ7XÐYZÑ7[Ô7[Ð ÝÔ(×AÒAÀ!ÀVÔEXÑYÔYÐYØˆr2   c                ó  — | j         }|j        ||…         }|j                             ¦   «         dk    r|j        n|j        ||…         }t	          |t
          ¦  «        rjt          j        |||z
  |j        ¦  «        }t          j	         
                    |||¦  «        }| j        €|n!|                     | j        ||…         ¦  «        S |j        j        dk    rK||                     |j        ¦  «        |                     |j        ¦  «        z                       ¦   «         z  }n>t"          j        j                             |||                     |j        ¦  «        ¦  «        }t          j        || j        €d n| j        ||…         ¦  «        S )NrG   Úmps)ræ   r”   r  rn  rÜ   Útupler   r!  rF   rø  Úprepared_blockscale_mmr¦   rù   r�   r³   rE   Útr+   rµ   r  r  Ú_add_bias_in_place_or_fallback)rc   rÑ   rý   rþ   r—   Údatar¥   rh   s           r   rf   z_Int8RowProjection.rowsE  sX  € à”ˆØŒ|˜E $˜JÔ'ˆØ!'¤×!4Ò!4Ñ!6Ô!6¸!Ò!;Ð!;�”�ÀÄÈuÐUYÈzÔAZˆÝ�h¥Ñ&Ô&ð 	WÝÔ0°¸¸u¹ÀdÄkÑRÔRˆEÝÔ'×>Ò>¸xÈÈuÑUÔUˆCØ œ+Ð-�3�3°3·8²8¸F¼KÈÈdÈ
Ô<SÑ3TÔ3TÐTØŒ_Ô! UÒ*Ð*Ø˜dŸgšg h¤nÑ5Ô5¸¿ºÀÄÑ8PÔ8PÑP×SÒSÑUÔUÑUˆCˆCå”)Ô"×,Ò,¨X°t¸U¿XºXÀhÄnÑ=UÔ=UÑVÔVˆCÝÔ4°SÀ&Ä+ÐBU¸$¸$Ð[aÔ[fÐglÐmqÐgqÔ[rÑsÔsÐsr2   Nrê  rw   r2   r   rñ  rñ  )  s�   € € € € € ØgÐgØð
xð 
xñ „\ð
xð ðnð nñ „\ðnð ðð ñ „\ðð ðtð tñ „\ðtð tð tr2   rñ  c                  ój   — e Zd ZdZed„ ¦   «         Zed„ ¦   «         Zed„ ¦   «         Zed„ ¦   «         ZdS )Ú_KitchenRowProjectiona¶  mmgp row projection (offload.linear_rows) of the INT8 ConvRot and Quanto INT8 layers that kitchen_linear computes with
    _cutlass_linear_chunked: the input is quantized once, each range of output rows is the same CUTLASS GEMM on a view of the weight
    rows, so the rows equal those of the layer's own output. The quantized input is as large as half the input it replaces: callers
    release the input (prepare_linear_input takes it).c           	     ó¢  — t          | dd ¦  «        }t          �qt          rj|�ht          j                             ¦   «         sJ|j        rCt          | d¦  «        s3|j        t          j	        t          j
        fvs|                     ¦   «         sdS |j        dk    }|r%t          | dd¦  «        dk    st          ¦   «         sdS |s|j        dk    st          | d	d ¦  «        �dS | j        }|j        }|j        |j        d         c\  }}}t#          d
t$          ||d|                     ¦   «         z   z  z   dz   z  ¦  «        }	|	dk    r|	dz  dz  n|	}	||	k    o€|j        t          j        k    ok|j        od|                     ¦   «         oP|j        |j        k    o@|dz  dk    o7|dz  dk    o.d|cxk    odk    nc o| pt                               ||d¦  «        S )Nr×   rØ   Fri  r0  r   rC   rÖ   râ  rG   r   rH   rš   r3  )rÄ   rô   rq  r+   r,   r´   r“   rß   rE   r[   rZ   r·   r	  r‰   ræ   r”   r]   rï   rð   r{  r   r:  )
rc   rg   rã  rK   r—   r   rû   rü   r½   rf   s
             r   rä  z_KitchenRowProjection.supports^  s0  € å˜ °Ñ5Ô5ˆÝÐ¥OÐ°u°}ÍÌ×HcÒHcÑHeÔHe°}ÐmnÔmv°}Ý˜6Ð#;Ñ<Ô<ð 8Eà”7¥5¤>µ5´=Ð"AÐAÐAÈÏÊÑIZÔIZÐAØ�5Ø”* Ò/ˆØð 	� Ð(=¸qÑAÔAÀSÒHÐHÕP_ÑPaÔPaÐHØ�5Øð 	˜EœJ¨'Ò1Ð1µW¸VÐE[Ð]aÑ5bÔ5bÐ5nØ�5Ø”ˆØŒ|ˆØ”G˜TœZ¨œ]ˆ	‰ˆˆA�Ý�1•n¨¨Q°!°a·n²nÑ6FÔ6FÑ2FÑ-GÑ)GÈ!Ñ)KÑLÑMÔMˆØ"&¨"¢* *ˆt�r‰z˜B‰ˆ°$ˆØ�D’ð Q˜TœZ­5¬:Ò5ð Q¸$¼,ð QÈ4×K]ÒK]ÑK_ÔK_ð QÐdjÔdpÐtuÔt{Òd{ð QØ˜‘E˜Q’JðQØ#$ s¡7¨a¢<ðQØ47¸1Ð4EÐ4EÒ4EÐ4EÀÒ4EÐ4EÐ4EÐ4EðQØOVÈ;ð  LPÕZb÷  [Eò  [Eð  FGð  IJð  LOñ  [Pô  [Pð	Rr2   c                ó*   — | j         j        dk    rdndS )Nri  Úkitchen_int8_convrotÚkitchen_int8)r×   r	  )rc   s    r   r›  z_KitchenRowProjection.keyr  s   € à)/Ô)<Ô)AÀ_Ò)TÐ)TÐ%Ð%ÐZhÐhr2   c                ó¤  — |j         | j        j        dk    c\  }}}t          dt          |dz   z  dz  dz  ¦  «        }t          j        ||ft
          j        |j        ¬¦  «        d }}t          d||¦  «        D ]Ï}t          ||z   |¦  «        }	|r#t                               |||	…         d¦  «        n!t                               |||	…         ¦  «        \  }
}|€2t          j        |g|j         dd …         ¢R |j        |j        ¬¦  «        }|||	…                              |
¦  «         |||	…                              |¦  «         ~
~ŒÐ|||j        fS )Nri  rH   r   rD   r   rC   rG   )r]   r×   r	  rï   rð   r+   r~  r   rF   rò   ró   rô   rV  rõ   rE   rú   )rc   rg   rû   rü   rK   rf   r   rY  rý   rþ   Útile_qÚtile_scaless               r   rT  z_KitchenRowProjection.preparev  s‰  € àœ' 6Ô#6Ô#;¸Ò#Nˆ‰ˆˆA�Ý�2�¨1¨q©5Ñ1°bÑ8¸2Ñ=Ñ>Ô>ˆÝ”K  A ­e¬jÀÄÐJÑJÔJÈDˆ6ˆÝ˜1˜a Ñ&Ô&ð 	$ð 	$ˆEÝ�u˜t‘| QÑ'Ô'ˆDØbið  #]¥(×"JÒ"JÈ1ÈUÐSWÈZÌ=ÐZ]Ñ"^Ô"^Ð"^Õow÷  pNò  pNð  OPð  QVð  W[ð  Q[ô  O\ñ  p]ô  p]ÑˆF�KØˆ~Ýœ aÐ%@¨+Ô*;¸A¸B¸BÔ*?Ð%@Ð%@ÈÔHYÐbcÔbjÐkÑkÔk�Øˆe�DˆjŒM×Ò Ñ'Ô'Ð'Ø�5˜�:Ô×$Ò$ [Ñ1Ô1Ð1Ø˜˜Ø�&˜!œ'Ð!Ð!r2   c                ó’  — |\  }}}| j         | j        }}|j        |j        c\  }	}
}|j        ||…         }t          ||j        j        d         |¦  «        ||…         }|�$t                               |||…         ||¦  «        nt                               ||¦  «        }t          j
        |	||z
  f||¬¦  «        }t          j        }t          j                             |j        ¦  «        }t          dt           |
dz   z  dz  dz  ¦  «        }t#          d|	|¦  «        D ]¨}t%          ||z   |	¦  «        }t          j                              ||||…         ¦  «         ||¦  «         ||||…         ¦  «         ||¦  «         ||¦  «         ||||…         ¦  «        t          j        |         |¦  «        st+          d¦  «        ‚Œ©|S )Nr   rD   rH   r   z1Comfy Kitchen rejected the INT8 CUTLASS row range)ræ   r¦   r]   rF   r”   r±   rô   r\  r]  r+   r~  r^  r5   ra  rb  rï   rð   rò   ró   r_  r`  rc  )rc   rÑ   rý   rþ   r   rY  rE   r—   r¦   rû   rü   rF   r   r¥   rd  rh   re  r„  rÿ   ÚrowÚends                        r   rf   z_KitchenRowProjection.rows…  sï  € à#Ñˆˆ6�5Ø”~ v¤{�ˆØœ !¤(ˆ‰ˆˆA�ØŒ|˜E $˜JÔ'ˆÝ˜f f¤lÔ&8¸Ô&;¸VÑDÔDÀUÈ4ÀZÔPˆØRVÐRb•H×-Ò-¨d°5¸°:Ô.>ÀÈÑNÔNÐNÝ!×4Ò4°V¸UÑCÔCð 	åŒk˜1˜d U™lÐ+°5ÀÐHÑHÔHˆÝÔ(ˆÝ”×3Ò3°F´LÑAÔAˆÝ�2�¨1¨q©5Ñ1°bÑ8¸2Ñ=Ñ>Ô>ˆÝ˜˜A˜tÑ$Ô$ð 	Xð 	XˆCÝ�c˜D‘j !Ñ$Ô$ˆCÝ”;×3Ò3°D°D¸¸3¸s¸7¼Ñ4DÔ4DÀdÀdÈ4ÁjÄjÐRVÐRVÐW]Ð^aÐbeÐ^eÔWfÑRgÔRgÐimÐimÐnsÑitÔitÐvzÐvzð  |Dñ  wEô  wEØ59°T¸#¸cÀ#¸g¼,Ñ5GÔ5GÍÔI_Ð`eÔIfÐhnñpô pð Xå"Ð#VÑWÔWÐWðXð ˆ
r2   Nrê  rw   r2   r   r  r  X  s�   € € € € € ð:ð :ð
 ðRð Rñ „\ðRð& ðið iñ „\ðið ð"ð "ñ „\ð"ð ðð ñ „\ðð ð r2   r  )Úresolvedc               ó2  — |€t          | ¦  «        n|\  }}t          j        | ¦  «         t          }t          �t          d         t          d         _        d at          �&ddlm} t          t          ¦  «        |j
        _        d at          j        ¦   «          |dk    rdndt          j        d<   d\  aaad	ad	ad	ad a|dk    r%t          j        |¦  «        st/          d
¦  «        ‚�ng|dk    rBddlm} t1          ¦   «          |a|j
        j        at          t2          ¦  «        |j
        _        �n|dk    �rddlm} t1          ¦   «          |at4          j        j        d uat$          s||j        fat:          |_        t$          sF|j        s?t4          j                              ¦   «         }|dk    at(          p|dk    otC          ¦   «         at(          rt          j"        ¦   «         \  a}tF          r8t          j"        ¦   «         \  a$}	tH          �dnd|	› d�}
tK          d|
› d�¦  «         |j
        j        at          tL          ¦  «        |j
        _        |addl'm(}m)} ddl*m+} | ,                    tZ          ¦  «          |dk    r|j,        n|j.        t^          ¦  «          |dk    r|j,        n|j.        t`          ¦  «         | 1                    td          ¦  «         ||k    r
tf          dz  a3dt$          rdnddddœ|         }
tK          d|
› d| › d�¦  «         |dk    S )NrG   r   )Úqbytesr   Ú1Ú0ÚWAN2GP_QUANTO_INT8_KERNEL)r   NNFz$Failed to enable Triton INT8 kernelsr   r   )rì   r   rŽ  r   r  r  z[INT8] Precision Rules: z for protected linear layers.)Úcuda_graphsÚoffload)Úquanto_rowsr   zComfy Kitchen HIPzComfy Kitchen CUDAÚPyTorch)r   r   r   r   z[INT8] Backend: z (setting: r!   r   )4r†   r   Úconfigure_familiesr#   Ú_kitchen_dlpack_exportr^  rØ  Úoptimum.quanto.tensor.weightsr  rï  ÚWeightQBytesLinearFunctionrà   r   Údisable_quanto_int8_kernelr&   r.   rô   r¢   r‰  rq  r8  r*  Úmaybe_enable_quanto_int8_kernelrc  rÒ  rÜ  r+   r@   rN   r:   r”  rB   rO   r´  r   r  r  r/   rÚ  r$  r  r  Úshared.qtypesr  Úregister_row_projectionÚ_INT8_ROW_PROJECTIONÚunregister_row_projectionÚ_KITCHEN_ROW_PROJECTIONÚ_WGP_ROW_PROJECTIONÚregister_capture_setupr¢  r“  )r‚   Úverbose_levelr  r…   rc   Úprevious_backendr  Ú
capabilityry   r„   Úlabelr  r  r  s                 r   Ú	configurer(  �  s›  € ð 5=Ð4D•o iÑ0Ô0Ð0È(�O€GˆVÝÔ$ YÑ/Ô/Ð/ÝÐÝÐ)Ý5KÈAÔ5NÕ˜qÔ!Ô2Ø!%ÐÝÐ$Ø8Ð8Ð8Ð8Ð8Ð8Ý4@ÕARÑ4SÔ4SˆÔ)Ô1Ø ÐÝ
Ô%Ñ'Ô'Ð'Ø5<ÀÒ5HÐ5H¨c¨cÈc…B„JÐ*Ñ+Ø4Ñ€Hˆh˜Ø€LØ€OØ€NØÐØ�(ÒÐÝÔ5°mÑDÔDð 	GÝÐEÑFÔFÐFñ	Gà	�LÒ	 Ð	 Ø8Ð8Ð8Ð8Ð8Ð8Ý‰ŒˆØˆØ"Ô=ÔEÐÝ4@ÕATÑ4UÔ4UˆÔ)Ô1Ñ1Ø	�IÒ	Ñ	Ø8Ð8Ð8Ð8Ð8Ð8Ý‰ŒˆØˆÝ”}Ô(°Ð4ˆÝð 	5Ø&,¨fÔ.EÐ%FÐ"Ý&4ˆFÔ#Ýð 	c FÔ$@ð 	cÝœ×9Ò9Ñ;Ô;ˆJØ'¨7Ò2ˆNÝ,Ðb°¸vÒ1EÐ1aÕJ_ÑJaÔJaˆOÝð 	EÝ&,Ô&BÑ&DÔ&DÑ#Ð  !Ýð 	SÝ(.Ô(DÑ(FÔ(FÑ%Ð˜vÝ 1Ð =�H�HÐCXÈvÐCXÐCXÐCXˆEÝÐQ¨UÐQÐQÐQÑRÔRÐRØ"Ô=ÔEÐÝ4@ÅÑ4QÔ4QˆÔ)Ô1Ø€HØ)Ð)Ð)Ð)Ð)Ð)Ð)Ð)Ø)Ð)Ð)Ð)Ð)Ð)Ø×#Ò#Õ$8Ñ9Ô9Ð9Øc¨°9Ò(<Ð(<€WÔ$Ð$À'ÔBcÕe|Ñ}Ô}Ð}Øf¨°<Ò(?Ð(?€WÔ$Ð$ÀWÔEfÕh{Ñ|Ô|Ð|Ø×&Ò&Õ'=Ñ>Ô>Ð>ØÐ"Ò"Ð"Ý�A‰ˆØ'Í<Ð4qÐ4GÐ4GÐ]qð  ~FØ!ð#ð #Ø#*ô,€Eå	Ð
<˜UÐ
<Ð
<¨yÐ
<Ð
<Ð
<Ñ=Ô=Ð=Ø�iÒÐr2   )	Nr   FNNrª   NNNr4   )r  )FrO  )Urî  Ú
__future__r   rR   r&   Úfnmatchr   Útypesr   r+   Úshared.kernelsr   r   r   r   r}   r€   r#   r“  r¢   rô   r‰  rq  r8  rØ  rÎ  rÊ   r%   r"   r*  r  r  r  Úsetr•  r)   rð   r1   r:   rj   rt   r†   r‰   r‹   r˜   r    r©   r¿   rÅ   rÇ   rÍ   rÓ   rä   rá   ré   r  r  r  r±   r,  r[   r<  rA  rE  rH  rZ  rÐ   rx  rË   r‹  rŠ  r¢  rU   r´  rÒ  rÕ  rÚ  rÜ  rÞ  r"  rñ  r  r  r!  r(  rw   r2   r   ú<module>r.     s4  ðà HÐ HØ "Ð "Ð "Ð "Ð "Ð "à Ð Ð Ð Ø 	€	€	€	Ø Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð à €€€à FÐ FÐ FÐ FÐ FÐ FÐ FÐ Fð1ð 1ð 1ð EÐFgØx°~°~Ñ7GÔ7GÐ(\¨¨È_ÐxÐxÐxð  {Gð  HØ4Ð6sðu€ð &°/ÈXÐ
VÐ
V€Ø€Ø€Ø€Ø€Ø€Ø€Ø€ØÐ Ø€Ø€ØÐ ØÐ ØÐ ØÐ ØÐ ØÐ Ø˜C™EœEÐ àÐ ð "€ðLð Lð Lð$Tð Tð Tðð ð ðBð ð ð(ð ð ð*!ð !ð !ð$ð $ð $ðið ið ið~ð ~ð ~ð
Mð Mð Mð HLØðð ð ð ðD>ð >ð >ðbð bð bð bð	tð 	tð 	tð 	tð@ð @ð @ðcð cð cð&-ð -ð -ð"ð ð ð2]ð ]ð ]ð ]ð6ð ð ð
;ð 
;ð 
;ðð ð ð
;ð 
;ð 
;ð 
;ð (-¤~Ð&7¸Tð Xð Xð Xð Xð( 04ÀÈcØ $°T¸tð-ð -ð -ð -ð -ð*8ð 8ð 8ð{ð {ð {ð {ðð ð ð8
ð 
ð 
ð'ð 'ð 'ðT.-ð .-ð .-ð .-ðbð ð ð4ð ð ð8'ð 'ð 'ð4 €ÔÑÔðð ñ Ôðð**ð *ð *ðZ-¸ð -ð -ð -ð -ð -ð&7ð 7ð 7ð 7ð7ð 7ð 7ð 7ð Uð  Uð  Uð  Uð  Uñ  Uô  Uð  UðF (Ð'Ñ)Ô)Ð ð)tð )tð )tð )tð )tñ )tô )tð )tðX *Ð)Ñ+Ô+Ð ð?ð ?ð ?ð ?ð ?ñ ?ô ?ð ?ðD 0Ð/Ñ1Ô1Ð ð= °dð = ð = ð = ð = ð = ð = ð = r2   