§
    ÎÐj   ã            
      ó€   — d Z ddlmZ ddlZdZdZd„ Zdd„Zd„ Zdd	d
dddddddœ	d„Z	dd„Z
d„ Zdd	ddddœd„Zg d¢ZdS )z9Public interface for the bundled Sol-Attn Triton kernels.é    )ÚannotationsNé@   ))é   é   )r   é	   )r   r   )é
   r   )é   r   )r	   é   c                óX  — | j         dk    rt          d¦  «        ‚t          j                             | ¦  «        }|t
          vrGd                     d„ t
          D ¦   «         ¦  «        }t          d|› d|d         › |d         › �¦  «        ‚|t          j        k    rt          d	|› �¦  «        ‚	 dd l}n"# t          $ r}t          d
¦  «        |‚d }~ww xY wt          d„ |j                             d¦  «        d d…         D ¦   «         ¦  «        }|dk     rt          d|j        › �¦  «        ‚|S )NÚcudazSol-Attn requires CUDAú, c              3  ó*   K  — | ]\  }}d |› |› �V — ŒdS ©ÚSMN© ©Ú.0ÚmajorÚminors      úAC:\Users\johnn\OneDrive\Documents\AI\shared\sol_attn\interface.pyú	<genexpr>z#validate_runtime.<locals>.<genexpr>   s4   è è € ÐbÐb±l°e¸UÐ1 5Ð1¨%Ð1Ð1ÐbÐbÐbÐbÐbÐbó    zSol-Attn supports ú; got SMr   r
   z&Sol-Attn requires bfloat16 Q/K/V, got zSol-Attn requires Triton >= 3.6c              3  ó4   K  — | ]}t          |¦  «        V — Œd S ©N)Úint)r   Úparts     r   r   z#validate_runtime.<locals>.<genexpr>   s(   è è € ÐLÐL $•C˜‘I”IÐLÐLÐLÐLÐLÐLr   ú.é   )é   r   z%Sol-Attn requires Triton >= 3.6, got )ÚtypeÚRuntimeErrorÚtorchr   Úget_device_capabilityÚSUPPORTED_CUDA_CAPABILITIESÚjoinÚbfloat16ÚtritonÚImportErrorÚtupleÚ__version__Úsplit)ÚdeviceÚdtypeÚ
capabilityÚ	supportedr(   ÚerrorÚversions          r   Úvalidate_runtimer3      sY  € Ø„{�fÒÐÝÐ3Ñ4Ô4Ð4Ý”×1Ò1°&Ñ9Ô9€JØÕ4Ð4Ð4Ø—I’IÐbÐbÕFaÐbÑbÔbÑbÔbˆ	ÝÐa°	ÐaÐaÀ:ÈaÄ=ÐaÐR\Ð]^ÔR_ÐaÐaÑbÔbÐbØ•”ÒÐÝÐKÀEÐKÐKÑLÔLÐLðIØˆˆˆˆøÝð Ið Ið IÝÐ<Ñ=Ô=À5ÐHøøøøðIøøøåÐLÐL¨&Ô*<×*BÒ*BÀ3Ñ*GÔ*GÈÈÈÔ*KÐLÑLÔLÑLÔL€GØ�ÒÐÝÐWÀ6ÔCUÐWÐWÑXÔXÐXØÐs   Â-B2 Â2
CÂ<CÃCc                ó„  — | j         dk    s | j        |j        k    s| j        |j        k    rt          d¦  «        ‚| j        d         dk    s| j        d         dk    rt          d¦  «        ‚t          d„ | ||fD ¦   «         ¦  «        rt	          d	¦  «        ‚| j        j        d
k    s |j        | j        k    s|j        | j        k    rt          d¦  «        ‚t          d„ | ||fD ¦   «         ¦  «        rt          d¦  «        ‚|dvrt          d¦  «        ‚t          |t          ¦  «        st	          d¦  «        ‚d|cxk    r| j        d         k    sn t          d¦  «        ‚|�pt          |t          ¦  «        st	          d¦  «        ‚d|cxk    r| j        d         k    sn t          d¦  «        ‚||z   | j        d         k    rt          d¦  «        ‚t          |t          ¦  «        rd|cxk    r| j        d         k     sn t          d¦  «        ‚t          t          j                             | j        ¦  «        ¦  «        S )Né   z+q, k, and v must share shape [B, T, H, 128]r
   r   r    é€   z.Sol-Attn requires T > 0 and head dimension 128c              3  ó@   K  — | ]}|j         t          j        k    V — Œd S r   )r.   r#   r'   ©r   Útensors     r   r   z#_validate_inputs.<locals>.<genexpr>&   s+   è è € Ð
BÐ
B¨fˆ6Œ<�5œ>Ò)Ð
BÐ
BÐ
BÐ
BÐ
BÐ
Br   z#q, k, and v must use torch.bfloat16r   z+q, k, and v must be on the same CUDA devicec              3  óH   K  — | ]}|                      d ¦  «        dk    V — ŒdS )éÿÿÿÿr
   N)Ústrider8   s     r   r   z#_validate_inputs.<locals>.<genexpr>*   s3   è è € Ð
:Ð
: fˆ6�=Š=˜ÑÔ Ò!Ð
:Ð
:Ð
:Ð
:Ð
:Ð
:r   z1q, k, and v must have a contiguous head dimension)ÚdiagÚexactz%thresh_type must be 'diag' or 'exact'zsink_tokens must be an integerzsink_tokens must be in [0, T]z%sink_start must be an integer or Nonezsink_start must be in [0, T]z%sink_start + sink_tokens must be <= Tz(query_start must be an integer in [0, T))ÚndimÚshapeÚ
ValueErrorÚanyÚ	TypeErrorr-   r!   Ú
isinstancer   r*   r#   r   r$   )ÚqÚkÚvÚthresh_typeÚsink_tokensÚ
sink_startÚquery_starts          r   Ú_validate_inputsrL   !   s`  € Ø„v�‚{€{�a”g ¤Ò(Ð(¨A¬G°q´wÒ,>Ð,>ÝÐFÑGÔGÐGØ„wˆq„z�Q‚€˜!œ' !œ*¨Ò+Ð+ÝÐIÑJÔJÐJÝ
Ð
BÐ
B¸¸A¸q¸	Ð
BÑ
BÔ
BÑBÔBð ?ÝÐ=Ñ>Ô>Ð>Ø„x„}˜ÒÐ !¤(¨a¬hÒ"6Ð"6¸!¼(ÀaÄhÒ:NÐ:NÝÐFÑGÔGÐGÝ
Ð
:Ð
:°°A°q°	Ð
:Ñ
:Ô
:Ñ:Ô:ð NÝÐLÑMÔMÐMØÐ+Ð+Ð+ÝÐ@ÑAÔAÐAÝ�k¥3Ñ'Ô'ð :ÝÐ8Ñ9Ô9Ð9Ø�Ð)Ð)Ò)Ð)˜qœw qœzÒ)Ð)Ð)Ð)ÝÐ8Ñ9Ô9Ð9ØÐÝ˜*¥cÑ*Ô*ð 	EÝÐCÑDÔDÐDØ�JÐ,Ð,Ò,Ð, !¤'¨!¤*Ò,Ð,Ð,Ð,ÝÐ;Ñ<Ô<Ð<Ø˜Ñ# a¤g¨a¤jÒ0Ð0ÝÐDÑEÔEÐEÝ�k¥3Ñ'Ô'ð E¨q°KÐ/LÐ/LÒ/LÐ/LÀ!Ä'È!Ä*Ò/LÐ/LÐ/LÐ/LÝÐCÑDÔDÐDÝ•”×1Ò1°!´(Ñ;Ô;Ñ<Ô<Ð<r   c                óŽ   — | t           z   dz
  t           z  }|s||fS |€| |z
  n|}|t           z  ||z   t           z   dz
  t           z  fS )Nr
   )Ú
BLOCK_SIZE)ÚtokensrJ   rI   ÚblocksÚstarts        r   Ú_sink_block_rangerR   >   s`   € Ø•zÑ! AÑ%­*Ñ4€FØð Ø�vˆ~ÐØ$.Ð$6ˆF�[Ñ Ð ¸J€EØ•JÑ ¨Ñ!4µzÑ!AÀAÑ!EÍ*Ñ TÐTÐTr   g      ð?r=   F)	ÚscaleÚtaurH   rI   rJ   rK   Ú	recycle_qÚint8_qkÚint8_pvc       	        óº  — |r|
st          d¦  «        ‚|
r¬t          | ||||||¦  «        }|rt          d¦  «        ‚|t          vrGd                     d„ t          D ¦   «         ¦  «        }t	          d|› d|d         › |d         › �¦  «        ‚dd	lm} t          | j        d         ||¦  «        } || |||||d
||¬¦	  «	        S dd	l	m}  || |||||||||	¬¦
  «
        S )Nzint8_pv requires int8_qkz7The optimized INT8 Sol-Attn path requires query_start=0r   c              3  ó*   K  — | ]\  }}d |› |› �V — ŒdS r   r   r   s      r   r   zsol_attn.<locals>.<genexpr>O   s4   è è € Ð!fÐ!f¹,¸%ÀÐ"5 uÐ"5¨eÐ"5Ð"5Ð!fÐ!fÐ!fÐ!fÐ!fÐ!fr   zTriton Sol-Attn supports r   r   r
   )Úsol_attnT)rS   rT   rH   rV   rW   Úsink_blocks)rS   rT   rH   rI   rJ   rK   rU   )
rA   rL   r%   r&   r"   ÚsaganakirZ   rR   r@   Útriton_kernels)rE   rF   rG   rS   rT   rH   rI   rJ   rK   rU   rV   rW   r/   r0   Úoptimized_sol_attnr[   Útriton_sol_attns                    r   rZ   rZ   F   sZ  € àð 5�wð 5ÝÐ3Ñ4Ô4Ð4Øð ZÝ% a¨¨A¨{¸KÈÐU`ÑaÔaˆ
Øð 	XÝÐVÑWÔWÐWØÕ8Ð8Ð8ØŸ	š	Ð!fÐ!fÕJeÐ!fÑ!fÔ!fÑfÔfˆIÝÐl¸9ÐlÐlÈjÐYZÌmÐlÐ]gÐhiÔ]jÐlÐlÑmÔmÐmØ<Ð<Ð<Ð<Ð<Ð<å'¨¬°¬
°JÀÑLÔLˆØ!Ð! ! Q¨°¸CÈ[Ø*.¸È[ðZñ Zô Zð 	Zð <Ð;Ð;Ð;Ð;Ð;àˆ?˜1˜a ¨%°SÀkØ'2¸zÐWbÐnwðyñ yô yð yr   c                ób   — ddl m}  |t          j                             | ¦  «        ||¦  «        S )zÛTrue when sol_attn(q, k, v, int8_qk=True) can be split into prepare_kv and sol_attn_prepared on this device: K/V are
    prepared and k released before q is computed, and the output can be written into q (same results).r
   )Úinline_q_path)Úsaganaki.fwdra   r#   r   r$   )r-   rO   rH   ra   s       r   Ústaged_supportedrc   ]   s:   € ð ,Ð+Ð+Ð+Ð+Ð+àˆ=�œ×9Ò9¸&ÑAÔAÀ6È;ÑWÔWÐWr   c                ó&   — ddl m}  || |¦  «        S )zOThe K/V side of the INT8 path (see staged_supported): k is not read afterwards.r
   )Úprepare_int8_kv)Úsaganaki.preprocessre   )rF   rG   re   s      r   Ú
prepare_kvrg   e   s&   € à4Ð4Ð4Ð4Ð4Ð4àˆ?˜1˜aÑ Ô Ð r   )rS   rT   rI   rJ   Úoutc               óf   — ddl m}  || ||||t          | j        d         ||¦  «        |¬¦  «        S )zjsol_attn(q, k, v, int8_qk=True, thresh_type="diag") from prepared = prepare_kv(k, v); out may be q itself.r
   )Úsol_attn_prepared)rS   rT   r[   rh   )rb   rj   rR   r@   )	rE   rG   ÚpreparedrS   rT   rI   rJ   rh   Úprepared_attentions	            r   rj   rj   l   sb   € àEÐEÐEÐEÐEÐEàÐ˜a  H°E¸sÕPaÐbcÔbiÐjkÔblÐnxð  {Fñ  QGô  QGØ"%ð'ñ 'ô 'ð 'r   )rg   rZ   rj   rc   r3   )r   Nr   )r=   )Ú__doc__Ú
__future__r   r#   rN   r%   r3   rL   rR   rZ   rc   rg   rj   Ú__all__r   r   r   ú<module>rp      s  ðð @Ð ?à "Ð "Ð "Ð "Ð "Ð "à €€€ð €
ØQÐ ðð ð ð&=ð =ð =ð =ð:Uð Uð Uð  $¨¸&ÈaÐ\`ÐnoØ e°Uðyð yð yð yð yð.Xð Xð Xð Xð!ð !ð !ð 04¸È!ÐX\Ðbfð 'ð 'ð 'ð 'ð 'ð bÐ
aÐ
a€€€r   