§
     ’Þj6  ã                   ó   — d Z ddlZddlmc mZ ddlmZmZm	Z	m
Z ddlmZmZmZ 	 ddlmZ n# eeef$ r dZY nw xY w	 ddlmZmZ n# eeef$ r dxZZY nw xY w	 ddlmZmZ n# eeef$ r dxZZY nw xY w	 dd	lmZ n# eeef$ r dZY nw xY wej         ej!        ej"        fZ#ej         ej!        fZ$d
Z%d„ Z&d!d„Z'd!d„Z(d„ Z)d„ Z*d„ Z+d„ Z,d„ Z-d"d„Z.d"d„Zd#d„Z/d„ Z0d„ Z1d„ Zddœd„Zd„ Z2 e2¦   «         \  Z3Z4dddœd „Z5dS )$zÒShared dispatch for video/audio activation kernels.

Streaming schedules, temporal history, and neighborhood topology stay with the
model. This module owns kernel eligibility and fallbacks, never those caches.
é    Né   )Úkernel_policyÚopsÚ
native_opsÚvideo_kitchen)ÚchooseÚcuda_inferenceÚtake)Úvideo_triton)Úconv3d_tritonÚstream_triton)Úsnake_beta_aaÚsnake_)Úgroup_norm_silui    c                 ó,   — t          j        | |¦  «        S ©N)Ú_kitchenÚprepare_rotary)ÚcosÚsins     ú>C:\Users\johnn\OneDrive\Documents\AI\shared\kernels\vae_ops.pyr   r   #   s   € ÝÔ" 3¨Ñ,Ô,Ð,ó    c                 ó"  — t          | ¦  «        }t          dt          j        ||¦  «        ¬¦  «        rt          j        |||¦  «        S t          j        ||j        |j        |j        d|j	        ¬¦  «        }|€|n| 
                    |¦  «        S )NzVideo Conv3D)Úkitchenr   )ÚstrideÚpaddingÚdilation)r
   r   r   Ú
can_conv3dÚconv3dÚFÚweightÚbiasr   r   Úadd_)Úx_listÚconvÚresidualÚxs       r   r   r   '   s„   € ÝˆV‰Œ€AÝˆn¥hÔ&9¸!¸TÑ&BÔ&BÐCÑCÔCð 2ÝŒ˜q $¨Ñ1Ô1Ð1Ý	Œ��D”K ¤°4´;ÈÐTXÔTaÐbÑbÔb€AØÐ ˆ1ˆ1 a§f¢f¨XÑ&6Ô&6Ð6r   c           	      óê  — t          | ¦  «        }d}|�^|\  }}}}}	t          d uoLt          |t          ¬¦  «        o6||cxk    o|cxk    o|k    nc o|t	          |j        dd …         ¦  «        k     }|d uot          j        |||¦  «        }
|�t          d|
|o|
 ¬¦  «        nd }|dk    rt          j	        |||¦  «        S |j        \  }}}}}| 
                    ddd	d
d¦  «                             ¦   «                              ||z  |d	||¦  «        }t          j        ||j        |j        |j        |j        ¦  «        }|�"|\  }}}}}	|�t          j        |||||	¦  «        S |                     |||||¦  «         
                    ddd	d
d¦  «                             ¦   «         }|�Lt          j        |¦  «        }t          j        |||||ddfd¬¦  «        }t          j        |dddd|	df¦  «        }|S )NF©ÚdtypeséþÿÿÿzFrame GroupNorm + SiLU + Pad)r   Úowned_tritonr   r   é   r   é   é   Úreflect)Úmode)r
   Ú_videor	   Ú_DTYPESÚminÚshaper   Ú	availabler   Úgroup_norm_silu_pad3dÚpermuteÚ
contiguousÚviewr    Ú
group_normÚ
num_groupsr!   r"   ÚepsÚsilu_pad_framesÚsiluÚpad)r$   ÚnormÚsilu_padr'   Ú
triton_padÚleftÚrightÚtopÚbottomÚfrontr   ÚfamilyÚbatchÚchannelsÚframesÚheightÚwidths                    r   Úframe_group_normrO   /   sM  € ÝˆV‰Œ€AØ€JØÐØ*2Ñ'ˆˆe�S˜& %Ý DÐ(ð V­^¸AÅgÐ-NÑ-NÔ-Nð VØ %Ð8Ð8Ò8Ð8¨3Ð8Ð8Ò8Ð8°&Ò8Ð8Ð8Ð8ðVØ=AÅCÈÌÐPRÐPSÐPSÌÑDUÔDUÒ=Uð 	ð ˜dÐ"ÐL¥xÔ'9¸!¸TÀ8Ñ'LÔ'L€GØqyð  rF�VÐ2¸GÐR\ÐRlÐelÐalÐmÑmÔmÐmð  LP€FØ�ÒÐÝÔ-¨a°°xÑ@Ô@Ð@Ø-.¬WÑ*€Eˆ8�V˜V UØ	�	Š	�!�Q˜˜1˜aÑ Ô ×+Ò+Ñ-Ô-×2Ò2°5¸6±>À8ÈQÐPVÐX]Ñ^Ô^€AÝ	Œ�Q˜œ¨¬°d´iÀÄÑJÔJ€AØÐØ*2Ñ'ˆˆe�S˜& %ØÐÝÔ)¨!¨U°F¸DÀ%ÑHÔHÐHØ	�Šˆu�f˜h¨°Ñ6Ô6×>Ò>¸qÀ!ÀQÈÈ1ÑMÔM×XÒXÑZÔZ€AØÐÝŒF�1‰IŒIˆÝŒE�!�d˜E 3¨°°1Ð5¸IÐFÑFÔFˆÝŒE�!�a˜˜A˜q %¨Ð+Ñ,Ô,ˆØ€Hr   c                 ód  — t           d uo?|d uo;t          | t          ¬¦  «        o%t          j        ||d         j        d         ¦  «        }t          d|t          j        | |¦  «        ¬¦  «        }t          t          t           d uot           j	        ot          | ¦  «        ¬¦  «        ¦  «        }||fS )Nr)   r   éÿÿÿÿzVideo RMSNorm + RoPE)r,   r   ©r,   )
r2   r	   r3   Úcan_rms_roper5   r   r   Úcan_fuse_rms_ropeÚboolÚCAST_ROPE_SUPPORTED)r'   Údim_headÚrotaryÚ
triton_rmsrI   Úcasts         r   Ú	rope_planr[   K   sµ   € Ý Ð$ð I¨°tÐ);ð IÅÈqÕY`Ð@aÑ@aÔ@að IÝÔ)¨(°F¸1´I´OÀBÔ4GÑHÔHð åÐ*¸ÍXÔMgÐhiÐkqÑMrÔMrÐsÑsÔs€FÝ•¥F°$Ð$6Ð$k½6Ô;UÐ$kÕZhÐijÑZkÔZkÐlÑlÔlÑmÔm€DØ�4ˆ<Ðr   c                 óÜ   ‡‡— t          | ¦  «        }|\  }}|�|S |j        Š|sddlm}  ||ˆˆfd„¦  «         |S  ‰|                     ¦   «         ¦  «        }t          j        ||¦  «        S )zZConsume a (batch, tokens, heads, head_dim) projection, normalizing each row independently.Nr   )Úrows_c                 ód   •—  ‰|                       ¦   «         ¦  «                             ‰¦  «        S r   )ÚfloatÚto)ÚpartÚfirstÚlastÚdtyperA   s      €€r   ú<lambda>z&normalize_projection.<locals>.<lambda>]   s'   ø€ ¨4¨4°·
²
±´Ñ+=Ô+=×+@Ò+@ÀÑ+GÔ+G€ r   )r
   rd   Úshared.attention_kitr]   r_   r2   Ú	cast_rope)	r$   rA   rX   Úplanr'   rI   rZ   r]   rd   s	    `      @r   Únormalize_projectionri   S   s�   øø€ åˆV‰Œ€AØ�L€FˆDØÐØˆØŒG€EØð Ø.Ð.Ð.Ð.Ð.Ð.àˆˆaÐGÐGÐGÐGÐGÑHÔHÐHØˆØˆˆQ�WŠW‰YŒY‰Œ€AÝÔ˜A˜vÑ&Ô&Ð&r   c                 ó  ‡ — |\  }}|dk    rt          j        ‰ |||¦  «        S |�t          j        ‰ |||¦  «         nÂ|sÀ|�¾ˆ fd„|D ¦   «         \  }}|j        d         }	‰ |fD ]›}
|
dd |	…f                              dd¬¦  «        \  }}|j        d         }||dd |…f         z  ||dd |…f         z  z
  }||d|d …f         z  ||d|d …f         z  z   }|                     |¦  «         |                     |¦  «         Œœ‰ |fS )Nr   c              3   óL   •K  — | ]}|                      ‰j        ¦  «        V — Œd S r   )r`   rd   )Ú.0ÚtableÚqs     €r   ú	<genexpr>zfinish_rope_.<locals>.<genexpr>j   s1   øè è € Ð:Ð:¨%�E—H’H˜QœWÑ%Ô%Ð:Ð:Ð:Ð:Ð:Ð:r   rQ   .r-   ©Údim)r   Úrms_roper2   Ú	rms_rope_r5   ÚchunkÚcopy_)rn   ÚkrX   r=   rh   rI   rZ   r   r   rq   r'   rb   ÚsecondÚhalfÚaÚbs   `               r   Úfinish_rope_r{   c   sJ  ø€ Ø�L€FˆDØ�ÒÐÝÔ   A v¨sÑ3Ô3Ð3ØÐÝÔ˜˜A˜v sÑ+Ô+Ð+Ð+Øð 
�fÐ(Ø:Ð:Ð:Ð:°6Ð:Ñ:Ô:‰ˆˆSØŒi˜Œmˆà�Q�ð 	ð 	ˆAØ˜c 4 C 4˜iœL×.Ò.¨q°bÐ.Ñ9Ô9‰MˆE�6Ø”;˜r”?ˆDØ˜˜C  $ ˜JœÑ'¨&°3°s¸E¸T¸E°z´?Ñ*BÑBˆAØ˜˜S $ % %˜ZœÑ(¨5°3°s¸D¸E¸E°z´?Ñ+BÑBˆAØ�KŠK˜‰NŒNˆNØ�LŠL˜‰OŒOˆOˆOØˆaˆ4€Kr   c                 óx  — t          dt          d uot          | d         t          ¬¦  «        ¬¦  «        st	          j        | ¦  «        S t          | ¦  «        }|j        t          j	        k    rt          j
        |¦  «        S t          j        ¦   «         rt          j        |¦  «        nt          j        |¦  «        S )NzVideo SwiGLUr   r)   rR   )r   r2   r	   r3   r   Úswiglur
   rd   ÚtorchÚfloat16Ú	swiglu_rnr   Úallow_approximateÚswiglu_)r$   r'   s     r   r}   r}   w   sŸ   € Ý�.­v¸TÐ/AÐ/oÅnÐU[Ð\]ÔU^ÕgnÐFoÑFoÔFoÐpÑpÔpð "ÝŒz˜&Ñ!Ô!Ð!ÝˆV‰Œ€AØ„w•%”-ÒÐÝÔ Ñ"Ô"Ð"Ý,Ô>Ñ@Ô@ÐW�6Œ=˜ÑÔÐÅfÄnÐUVÑFWÔFWÐWr   c                 óö   — t          | ¦  «        }t          t          d uot          |t          ¬¦  «        ¬¦  «        rt          j        |||¦  «        S |                     |¦  «                             |¦  «        S )Nr)   rR   )r
   r   r2   r	   r3   Ú	residual_Úmul_r#   )r$   Úscaler&   r'   s       r   Úscaled_residualr‡   €   sj   € ÝˆV‰Œ€AÝ�6¨Ð-ÐSµ.ÀÍ7Ð2SÑ2SÔ2SÐTÑTÔTð 4ÝÔ  5¨(Ñ3Ô3Ð3Ø�6Š6�%‰=Œ=×Ò˜hÑ'Ô'Ð'r   é   c                 ó.  — |                       d¦  «        t          d||                      ¦   «         z  ¦  «        }}t          d|                     ¦   «         |¦  «        D ]6}||||z   …         }|                     t          j        |¦  «        ¦  «         Œ7| S )z\x * sigmoid(x) written into x by chunks: the same values, without two full-size temporaries.rQ   r   r   )r:   ÚmaxÚelement_sizeÚrangeÚnumelr…   r~   Úsigmoid)r'   Úchunk_bytesÚflatÚrowsÚstartra   s         r   Úsilu_r“   ‡   sˆ   € à—’˜‘”�S  K°1·>²>Ñ3CÔ3CÑ$CÑDÔDˆ$€DÝ�q˜$Ÿ*š*™,œ,¨Ñ-Ô-ð 'ð 'ˆØ�E˜% $™,Ð&Ô'ˆØ�	Š	•%”- Ñ%Ô%Ñ&Ô&Ð&Ð&Ø€Hr   c                 ó4  — t           duo!|j        ot          j        | |j        |¦  «        }t	          d|¬¦  «        �-t          j        | |j        |j        |j        |j        |¦  «        S t          j
        | |¬¦  «        }| j        d         |j        z  }t          d|| dd…d|…f                              ¦   «         dz  z  ¦  «        }t          d|j        |¦  «        D ]¸}||z  t          |j        ||z   ¦  «        |z  }
}	t!          t#          j        | dd…|	|
…f                              ¦   «         |
|	z
  |z  |j        |	|
…                              ¦   «         |j        |	|
…                              ¦   «         |j        ¦  «        |¦  «        |dd…|	|
…f<   Œ¹|S )aŽ  silu(norm(x)) of a 2-D GroupNorm (N, C, H, W) as computed under autocast: normalization and SiLU in FP32, rounded once to dtype;
    x is borrowed. The WanGP kernel allocates only the output (shared.kernels.group_norm_silu). PyTorch: by chunks of groups (the FP32
    copies autocast would make), each rounded right away, so that neither the FP32 copy of x nor the FP32 result is allocated in full.NzGroupNorm + SiLUrR   )rd   r   r/   r   )Ú
_norm_siluÚaffineÚeligibler<   r   r   r!   r"   r=   r~   Ú
empty_liker5   rŠ   r�   rŒ   r4   r“   r    r;   r_   )r'   rA   rd   r�   ÚvalidÚoutÚstepÚgroupsr’   Úc0Úc1s              r   r   r   �   sÅ  € õ ˜dÐ"Ðe t¤{ÐeµzÔ7JÈ1ÈdÌoÐ_dÑ7eÔ7e€EÝÐ ¨uÐ5Ñ5Ô5ÐAÝÔ)¨!¨T¬_¸d¼kÈ4Ì9ÐVZÔV^Ð`eÑfÔfÐfÝ
Ô
˜1 EÐ
*Ñ
*Ô
*€CØŒ7�1Œ:˜œÑ(€DÝ��K A a a a¨¨$¨ h¤K×$5Ò$5Ñ$7Ô$7¸!Ñ$;Ñ<Ñ=Ô=€FÝ�q˜$œ/¨6Ñ2Ô2ð að aˆØ˜‘�s 4¤?°E¸F±NÑCÔCÀdÑJˆBˆÝ�aœl¨1¨Q¨Q¨Q°°2°¨X¬;×+<Ò+<Ñ+>Ô+>ÀÀbÁÈTÑ@QÐSWÔS^Ð_aÐbdÐ_dÔSe×SkÒSkÑSmÔSmÐosÔoxÐy{Ð|~Ðy~Ôo÷  pFò  pFñ  pHô  pHð  JNô  JRñ  Sô  Sð  U`ñ  aô  aˆˆAˆAˆAˆr�"ˆuˆH‰ˆØ€Jr   é   c           	      óÌ  — t           j        j        }|                     | ||dddd¦  «        }t	          d|| j        d         | j        d         z  |j        d         z  dz  z  ¦  «        }|| j        d         k    sO| j        j        dk    s?|                     |¦  «        s*| 	                    |¦  «        s| 
                    |¦  «        rt          j        | ||¦  «        S |                      | j        dd	…         |j        d	d…         z   ¦  «        }t          d| j        d         |¦  «        D ]5}t          j        | d
|||z   …dd…f         ||¦  «        |d
|||z   …dd…f<   Œ6|S )a¿  F.scaled_dot_product_attention(q, k, v) of (batch, heads, tokens, dim) tensors: one call where SDPA has a flash, memory-efficient or
    cuDNN kernel for them. Elsewhere (BF16 on Turing...) its math path materializes every score in FP32, 2.4 GiB for the single-head
    mid-block attention of a VAE's 1024 px tile: there, by chunks of queries whose scores take at most chunk_bytes, each attending all the
    keys (values differ by rounding only).Ng        Fr   r   r+   r/   ÚcudarQ   .)r~   Úbackendsr¡   Ú
SDPAParamsrŠ   r5   ÚdeviceÚtypeÚcan_use_flash_attentionÚcan_use_efficient_attentionÚcan_use_cudnn_attentionr    Úscaled_dot_product_attentionÚ	new_emptyrŒ   )	rn   rv   Úvr�   ÚsdpaÚparamsr‘   rš   r’   s	            r   Ú	attentionr®       s¢  € õ
 Œ>Ô€DØ�_Š_˜Q  1 d¨C°¸Ñ>Ô>€FÝˆq�+ !¤'¨!¤*¨q¬w°q¬zÑ"9¸A¼GÀB¼KÑ"GÈ!Ñ"KÑLÑMÔM€DØˆqŒw�rŒ{ÒÐ˜aœhœm¨vÒ5Ð5¸×9UÒ9UÐV\Ñ9]Ô9]Ð5Ðae÷  bBò  bBð  CIñ  bJô  bJÐ5ð  NR÷  Njò  Njð  kqñ  Nrô  NrÐ5ÝÔ-¨a°°AÑ6Ô6Ð6Ø
�+Š+�a”g˜c˜r˜c”l Q¤W¨R¨S¨S¤\Ñ1Ñ
2Ô
2€CÝ�q˜!œ' "œ+ tÑ,Ô,ð nð nˆÝ*+Ô*HÈÈ3ÐPUÐV[Ð^bÑVbÐPbÐdeÐdeÐdeÐKeÔIfÐhiÐklÑ*mÔ*mˆˆC��u˜t‘|Ð# Q Q QÐ&Ñ'Ð'Ø€Jr   c                ó.  — t          | ¦  «        }t          |t          d¬¦  «        oC|j        dk    o8|                     ¦   «         dk    o |ot          |t          j        j        ¦  «        }t          d|ot          j        |j        ¦  «        |o!t          d uot          j        |j        ¦  «        ¬¦  «        }|dk    r%t          j                             ||j        ¦  «        S |dk    rt          j        ||j        ¦  «        S  | ||¦  «        ¦  «        S )	NT©r*   Úapproximateé   r   zPixelNorm + SiLU)Ú
wgpkernelsÚtritonr³   r´   )r
   r	   r3   Úndimr�   Ú
isinstancer~   ÚnnÚSiLUr   r   r6   r¤   Ú_streamÚ_convÚ	supportedÚimplementationÚpixel_norm_silur=   Ú	norm_silu)r$   rA   ÚactÚchannel_normr'   r™   rI   s          r   r½   r½   °   s  € ÝˆV‰Œ€AÝ˜A¥g¸4Ð@Ñ@Ô@ð 0ÀQÄVÈqÂ[ð 0ÐUV×U\ÒU\ÑU^ÔU^ÐabÒUbð 0Ðgsð 0Ý˜C¥¤¤Ñ/Ô/ð 
åÐ&°5Ð3[½ZÔ=QÐRSÔRZÑ=[Ô=[Ø ÐV¥W°DÐ%8ÐV½U¼_ÈQÌXÑ=VÔ=VðXñ Xô X€Fà�ÒÐÝÔ(×8Ò8¸¸D¼HÑEÔEÐEØ�ÒÐÝÔ   D¤HÑ-Ô-Ð-Øˆ3ˆtˆt�A‰wŒw‰<Œ<Ðr   c           	      óh  — |d         j         \  }}}}}t          d„ |D ¦   «         ¦  «        |d         }}t          |t          d¬¦  «        oÊ|dz  dk    oÁ| j        dz  dk    o³| j        dk    o¨||dz
  z  |z  |z  t          k    o‘| j        dk    o†| j        dk    o{| j	        d	k    op| j
        d
k    oe| j        dv o\| j        j        |j        k    oG| j        j        |j        k    o2| j        du p)| j        j        |j        k    o| j        j        |j        k    }	|	o*t           duo!| j        duot!          j        |j        ¦  «        }
t%          d|
¬¦  «        }|€Jt'          |¦  «        d	k    rt)          j        |d¬¦  «        n|}|                     ¦   «           | |¦  «        S t)          j        ||||dz   |dz   f|j        |j        t(          j        ¬¦  «        }d\  }}|D ]4}||dd…dd…|||j         d         z   …d	d…d	d…f<   ||j         d         z  }Œ5~|                     ¦   «          | j        dk    rC|ddd	d…f         |ddd	d…f         c|ddd	d…f<   |ddd	d…f<   |d         |d         c|d<   |d<   nd\  |dddd…f<   |dddd…f<   |d<   |d<   t!          j        || j        | j        ¦  «        S )zDConsume temporal parts without keeping their original tensors alive.r   c              3   ó0   K  — | ]}|j         d          V — ŒdS )r-   N)r5   )rl   ra   s     r   ro   z temporal_conv.<locals>.<genexpr>À   s(   è è € Ð/Ð/ ˆtŒz˜!Œ}Ð/Ð/Ð/Ð/Ð/Ð/r   Tr°   é    )r   r   r   r-   )r.   r.   r.   r   )r   r   r   )Úzerosr0   NzStreamed Conv3D)r´   rp   )rd   r¤   Úmemory_format)Nr   rQ   r0   .éýÿÿÿ).r-   ).rÆ   ).r   ).rQ   )r   r   r   r   )r5   Úsumr	   r3   Úout_channelsr   Ú_STREAM_CONV_MIN_PIXELSÚkernel_sizer   rœ   r   Úpadding_moder!   rd   r¤   r"   rº   Úconv_supportedr   Úlenr~   ÚcatÚclearÚemptyÚchannels_last_3dr%   )r%   Úpartsrz   ÚcÚ_ÚhÚwÚtr'   r™   Útriton_convrI   Úpaddedr’   ra   s                  r   Útemporal_convrÚ   ½   s\  € à˜!”H”N�M€A€qˆ!ˆQ�ÝÐ/Ð/¨Ð/Ñ/Ô/Ñ/Ô/°°q´€q€AÝ˜A¥g¸4Ð@Ñ@Ô@ð fÀQÈÁVÈqÂ[ð fÐUYÔUfÐikÑUkÐopÒUpð fØ” 	Ò)ðfØ./°1°q±5©k¸A©oÀÑ.AÕE\Ò.\ðfàÔ! YÒ.ðfà37´=ÀIÒ3Mðfð ” Ò!ðfð '+¤l°iÒ&?ðfð EIÔDUÐYmÐDmðfð ”Ô" a¤gÒ-ð	fð 37´+Ô2DÈÌÒ2Pð	fð
 ”)˜tÐ#Ðd¨¬	¬¸1¼7Ò(BÐ(cÀtÄyÔGWÐ[\Ô[cÒGcð 
ð Ðj�E¨Ð-Ðj°$´)À4Ð2GÐjÍEÔL`ÐabÔaiÑLjÔLj€Kõ Ð%¨kÐ:Ñ:Ô:€FØ€~Ý'*¨5¡z¤z°A¢~ ~�EŒI�e Ð#Ñ#Ô#Ð#¸1ˆØ�Š‰ŒˆØˆt�A‰wŒwˆÝŒ[˜!˜Q  1 q¡5¨!¨a©%Ð0¸¼ÈÌÕ`eÔ`vÐwÑwÔw€FØ�H€A€uØð ð ˆØ@Dˆˆqˆqˆq�!�!�!�U˜5 4¤:¨a¤=Ñ0Ð0°!°B°$¸¸"¸Ð<Ñ=Ø�”˜A”ÑˆˆØØ	‡K‚K�M„M€MØÔ˜IÒ%Ð%Ø6<¸SÀ!ÀQÀrÀT¸\Ô6JÈFÐSVÐXZÐ\]Ð^`Ð\`ÐS`ÔLaÐ3ˆˆs�A�q˜�tˆ|Ñ˜f S¨"¨a°¨d ]Ñ3Ø*0°¬.¸&À¼/Ð'ˆˆv‰˜˜w™˜àQ[ÑNˆˆs�A�q�q�qˆyÑ˜6 # r¨1¨1¨1 *Ñ-¨v°f©~¸vÀg¹ÝŒ:�f˜dœk¨4¬9Ñ5Ô5Ð5r   c                óô   — t          | ¦  «        }t          d|ot          d uo|j        ¬¦  «        r,t          ||j        |j        |j        |j        j        ¦  «        S  | | ||¦  «        ¦  «        ¦  «        S )NzSnakeBeta + ResamplingrR   )r
   r   Ú_snakeÚis_cudaÚalphaÚbetaÚfilterÚlowpass)r$   Ú
activationÚupsampleÚ
downsampleÚfusabler'   s         r   r   r   Þ   s~   € ÝˆV‰Œ€Aõ Ð&°WÐ5aÅÈtÐASÐ5aÐXYÔXaÐbÑbÔbð hÝ�a˜Ô)¨:¬?¸H¼OÈZÔM_ÔMfÑgÔgÐgØˆ:�j�j  ¨!¡¤Ñ-Ô-Ñ.Ô.Ð.r   T)Ú	log_scalec                ó�  — t          dt          duoIt          | t          ¬¦  «        o3|                      ¦   «         o|j        | j        k    o|j        | j        k    ¬¦  «        rt          | |||¦  «        S |rt          j        |¦  «        n|}|rt          j        |¦  «        n|}t          j        | | 	                    ddd¦  «        ¦  «        }| 
                    ¦   «                              |¦  «                             | 	                    ddd¦  «        dz                        ¦   «         ¦  «         |                      |¦  «        S )al  Snake activation in place on x (batch, channels, length): x += sin(a x)^2 / (b + 1e-9), a and b per channel from alpha and beta
    (shape (1, channels, 1) or (channels,)), stored in log scale (a = exp(alpha), b = exp(beta)) or directly. The Triton kernel computes
    in FP32 from the stored parameters and rounds once at the store, in place of PyTorch's five rounded kernels (exp, mul, sin,
    square, scale, add: the Oobleck VAE of ACE-Step 1.5), so it is at least as precise in every dtype (checked against FP64 on random
    and edge inputs, tests/test_snake_op.py); the PyTorch path reproduces the module's chain.ÚSnakeNr)   rR   r   rQ   g•Ö&è.>)r   Ú_snake_plainr	   Ú_AUDIO_DTYPESÚis_contiguousr¤   r~   ÚexpÚmulÚreshapeÚsin_r…   Ú
reciprocalr#   )r'   rÞ   rß   ræ   ry   rz   Úwaves          r   r   r   ç   s6  € õ ˆg¥L¸Ð$<ð %GÅÐPQÕZgÐAhÑAhÔAhð %GÐmn×m|Òm|Ñm~Ôm~ð %GØ”, !¤(Ò*ð%GØ/3¬{¸a¼hÒ/FðHñ Hô Hð 7å˜A˜u d¨IÑ6Ô6Ð6Ø%Ð0�Œ	�%ÑÔÐ¨5€AØ$Ð.�Œ	�$‰Œˆ¨$€AÝŒ9�Q˜Ÿ	š	 ! R¨Ñ+Ô+Ñ,Ô,€DØ‡I‚I�K„K×Ò�TÑÔ×Ò §¢¨1¨b°!Ñ!4Ô!4°tÑ!;× GÒ GÑ IÔ IÑJÔJÐJØ�6Š6�$‰<Œ<Ðr   c                  ó:  — ddl m} m} 	 dd l}n# t          t
          f$ r Y dS w xY w	  ||j        ¦  «        }n# | $ r dd|j        › �fcY S w xY w| |d¦  «        k     rdt          |¦  «        fS t          j	         
                    ¦   «         t          |¦  «        fS )Nr   )ÚInvalidVersionÚVersion)Fznot installedFzunknown version z3.2.0)Úpackaging.versionró   rô   r´   ÚImportErrorÚOSErrorÚ__version__Ústrr~   r¡   Úis_available)ró   rô   r´   Úversions       r   Úneighborhood_statusrü   ÷   sé   € Ø9Ð9Ð9Ð9Ð9Ð9Ð9Ð9ð&ØˆˆˆˆøÝ�Ð!ð &ð &ð &Ø%Ð%Ð%ð&øøøð>Ø�'˜&Ô,Ñ-Ô-ˆˆøØð >ð >ð >ØÐ=¨Ô);Ð=Ð=Ð=Ð=Ð=Ð=ð>øøøà��˜Ñ!Ô!Ò!Ð!Ø•c˜'‘l”lÐ"Ð"ÝŒ:×"Ò"Ñ$Ô$¥c¨'¡l¤lÐ2Ð2s   Š �$£$¨9 ¹AÁA)Úkeyframe_timesÚslotsc                óB  — t          dt          o| d         j        ¬¦  «        rbddlm} |€|                     | |¦  «        S | d         j        d         }|\  }}|                     |  |||¦  «         |||¦  «        |¦  «        S |€ || |¦  «        S  || ||¦  «        S )Nz3D Neighborhood Attentionr   rR   r   )Úneighborhood_triton)r   Ú_NA_AVAILABLErÝ   Ú r   Úna3dr5   Ú
joint_na3d)	Úqkv_listrÊ   Úfallbackrý   rþ   r   ÚtimeÚvideo_slotsÚkeyframe_slotss	            r   Úneighborhood_attentionr
  	  sà   € ÝÐ)½Ð8]È(ÐSTÌ+ÔJ]Ð^Ñ^Ô^ð NØ)Ð)Ð)Ð)Ð)Ð)ØÐ!Ø&×+Ò+¨H°kÑBÔBÐBØ˜Œ{Ô  Ô#ˆØ&+Ñ#ˆ�^Ø"×-Ò-¨h¸¸ÀNÐTXÑ8YÔ8YÐ[iÐ[iÐjxÐz~Ñ[Ô[ð  BMñ  Nô  Nð  	NØÐØˆx˜ +Ñ.Ô.Ð.Øˆ8�H˜n¨kÑ:Ô:Ð:r   r   )rˆ   )rŸ   )6Ú__doc__r~   Útorch.nn.functionalr·   Ú
functionalr    r  r   r   r   r   r   Údispatchr   r	   r
   r   r2   rö   r÷   ÚRuntimeErrorr   rº   r   r¹   Úaudio_tritonr   rÜ   r   ré   r   r•   Úbfloat16r   Úfloat32rê   r3   rÉ   r   r   rO   r[   ri   r{   r}   r‡   r“   r®   r½   rÚ   rü   r  Ú_NA_VERSIONr
  © r   r   ú<module>r     s'  ððð ð
 €€€Ø Ð Ð Ð Ð Ð Ð Ð Ð Ø GÐ GÐ GÐ GÐ GÐ GÐ GÐ GÐ GÐ GÐ GÐ GØ 2Ð 2Ð 2Ð 2Ð 2Ð 2Ð 2Ð 2Ð 2Ð 2ðØ(Ð(Ð(Ð(Ð(Ð(Ð(øØ�W˜lÐ+ð ð ð Ø€F€F€FðøøøðØBÐBÐBÐBÐBÐBÐBÐBÐBøØ�W˜lÐ+ð ð ð ØÐ€EˆGˆGˆGðøøøð!ØMÐMÐMÐMÐMÐMÐMÐMÐMøØ�W˜lÐ+ð !ð !ð !Ø Ð €Fˆ\ˆ\ˆ\ð!øøøðØ/Ð/Ð/Ð/Ð/Ð/Ð/øØ�W˜lÐ+ð ð ð Ø€J€J€Jðøøøð ” ¤°´Ð>€àŒ>˜5œ=Ð
)€ØÐ ð-ð -ð -ð7ð 7ð 7ð 7ðð ð ð ð8ð ð ð'ð 'ð 'ð ð ð ð(Xð Xð Xð(ð (ð (ðð ð ð ðð ð ð ð ð ð ð ð 
ð 
ð 
ð6ð 6ð 6ðB/ð /ð /ð )-ð ð ð ð ð ð 3ð 3ð 3ð 1Ð0Ñ2Ô2Ñ €ˆ{ð OSÐZ^ð 
;ð 
;ð 
;ð 
;ð 
;ð 
;ð 
;sA   §. ®
;º;¿A ÁAÁAÁA$ Á$A3Á2A3Á7A> Á>
BÂ
B