ó
    ‡Ž¡jõk  ã                   óº  • S r SSKJr  SSKrSSKJr  SSKJr  SSKJ	r	J
r
  SSKJrJr  SS	KJr  SS
KJr  SSKJrJrJrJr  SSKJrJr  SSKJr  SSKJrJrJrJ r J!r!  SSK"J#r#  SSK$J%r%  SSK&J'r'J(r(J)r)J*r*  SSK+J,r,  SSK-J.r.J/r/J0r0  SSK1J2r2  \!Rf                  " \45      r5\ " S S\5      5       r6 " S S\Rn                  5      r8 " S S\,5      r9 " S S\(5      r: " S S \)5      r; " S! S"\'5      r< " S# S$\Rn                  5      r= " S% S&\5      r> " S' S(\65      r? " S) S*\5      r@ " S+ S,\65      rA\" S-S.9 " S/ S0\65      5       rB\" S1S.9 " S2 S3\6\25      5       rC/ S4QrDg)5zPyTorch Dia model.é    )ÚCallableN)Únné   )Úinitialization)ÚDynamicCacheÚEncoderDecoderCache)Úcreate_bidirectional_maskÚcreate_causal_mask)ÚFlashAttentionKwargs)ÚGradientCheckpointingLayer)ÚBaseModelOutputÚ)BaseModelOutputWithPastAndCrossAttentionsÚSeq2SeqLMOutputÚSeq2SeqModelOutput)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tupleÚis_torchdynamo_compilingÚlogging)Úmerge_with_config_defaults)Úcapture_outputsé   )ÚLlamaAttentionÚLlamaRMSNormÚLlamaRotaryEmbeddingÚeager_attention_forward)ÚPhi3MLPé   )Ú	DiaConfigÚDiaDecoderConfigÚDiaEncoderConfig)ÚDiaGenerationMixinc                   óX   ^ • \ rS rSr% \\S'   SrSrSrSr	Sr
SrSrSS/rU 4S jrS	rU =r$ )
ÚDiaPreTrainedModelé3   ÚconfigÚmodelTÚ	input_idsÚDiaEncoderLayerÚDiaDecoderLayerc                 ó.  >• [         TU ]  U5        [        U[        5      (       ap  [        R
                  " U R                  R                  [        R                  S9U R                  R                  -  n[        R                  " UR                  U5        g g )N©Údtype)ÚsuperÚ_init_weightsÚ
isinstanceÚDiaMultiChannelEmbeddingÚtorchÚaranger)   Únum_channelsÚlongÚ
vocab_sizeÚinitÚcopy_Úoffsets)ÚselfÚmoduler<   Ú	__class__s      €Ú±C:\Users\johnn\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.13_qbz5n2kfra8p0\LocalCache\local-packages\Python313\site-packages\transformers/models/dia/modular_dia.pyr2   Ú DiaPreTrainedModel._init_weights?   se   ø€ Ü‰Ñ˜fÔ%Ü�fÔ6×7Ñ7Ü—l’l 4§;¡;×#;Ñ#;Ä5Ç:Á:ÑNÐQU×Q\ÑQ\×QgÑQgÑgˆGÜ�JŠJ�v—~‘~ wÕ/ð 8ó    © )Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r"   Ú__annotations__Úbase_model_prefixÚsupports_gradient_checkpointingÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_can_compile_fullgraphÚmain_input_nameÚ_no_split_modulesr2   Ú__static_attributes__Ú__classcell__©r?   s   @r@   r'   r'   3   sG   ø‡ àÓØÐØ&*Ð#ØÐØ€NØÐØ!ÐØ!€OØ*Ð,=Ð>Ð÷0ó 0rB   r'   c                   ón   ^ • \ rS rSrSrS\4U 4S jjrS\R                  S\R                  4S jr	Sr
U =r$ )	r4   éF   aâ  In order to efficiently compute the audio embedding from the 9 different channels,
we vectorize the embedding process by using a single embedding layer and an offset.
Example:
- num_embeds = 4
- vocab_size = 8
- num_channels = 3
We would have offsets = [0, 8, 16]
If audio_codes = [0, 1, 2, 3], [1, 3, 4, 7], [5, 6, 7, 8],
then tokens = audio_codes + offsets
            = [0, 1, 2, 3, 9, 11, 12, 15, 21, 22, 23, 24]
This allows us to use a single embedding layer for all channels.
r)   c                 óˆ  >• [         TU ]  5         [        R                  " UR                  UR
                  -  UR                  5      U l        UR                  U l        UR
                  U l        [        R                  " UR
                  [        R                  S9UR                  -  n[        R                  " USS9U l        g )Nr/   F)Ú
persistent)r1   Ú__init__r   Ú	Embeddingr9   r7   Úhidden_sizeÚembedr5   r6   r8   ÚBufferr<   )r=   r)   r<   r?   s      €r@   rX   Ú!DiaMultiChannelEmbedding.__init__T   sŠ   ø€ Ü‰ÑÔÜ—\’\ &×"3Ñ"3°f×6IÑ6IÑ"IÈ6×K]ÑK]Ó^ˆŒ
Ø!×-Ñ-ˆÔØ"×/Ñ/ˆÔÜ—,’,˜v×2Ñ2¼%¿*¹*ÑEÈ×HYÑHYÑYˆÜ—y’y °UÑ;ˆ�rB   Úaudio_codesÚreturnc                 óV  • XR                   R                  UR                  5      -   R                  SUR                  S   UR                  S   -  5      nU R                  U5      R                  UR                  S   UR                  S   SU R                  5      nUR                  SS9$ )Néÿÿÿÿr!   r   r   )Údim)r<   ÚtoÚdeviceÚviewÚshaper[   rZ   Úsum)r=   r^   ÚtokensÚembedss       r@   ÚforwardÚ DiaMultiChannelEmbedding.forward\   s•   € Ø§¡§¡°×0BÑ0BÓ CÑC×IÑIØ�×!Ñ! !Ñ$ {×'8Ñ'8¸Ñ';Ñ;ó
ˆð —‘˜FÓ#×(Ñ(¨¯©°a©¸+×:KÑ:KÈAÑ:NÐPRÐTX×TdÑTdÓeˆØ�z‰z˜aˆzÐ Ð rB   )r[   rZ   r7   r<   )rD   rE   rF   rG   Ú__doc__r#   rX   r5   ÚTensorrj   rQ   rR   rS   s   @r@   r4   r4   F   s5   ø† ñð<Ð/÷ <ð! 5§<¡<ð !°E·L±L÷ !ò !rB   r4   c                   ó   • \ rS rSrSrg)ÚDiaMLPéd   rC   N©rD   rE   rF   rG   rQ   rC   rB   r@   ro   ro   d   ó   † ÚrB   ro   c                   ó   • \ rS rSrSrg)Ú
DiaRMSNorméh   rC   Nrq   rC   rB   r@   rt   rt   h   rr   rB   rt   c                   ó   • \ rS rSrSrg)ÚDiaRotaryEmbeddingél   rC   Nrq   rC   rB   r@   rw   rw   l   rr   rB   rw   c                   ó8   • \ rS rSrSrS	S\\-  S\S\4S jjr	Sr
g)
ÚDiaSelfAttentionép   ú=Multi-headed attention from 'Attention Is All You Need' paperr)   Ú	layer_idxÚ	is_causalc                 ó¤  • [         R                  R                  U 5        Xl        X l        UR
                  U l        U R                  R                  U l        U R                  R                  =(       d    U R                  U l        U R                  U R                  -  U l	        [        USUR
                  U R                  -  5      U l        SU l        SU l        X0l        [         R                  " U R
                  U R                  U R                  -  SS9U l        [         R                  " U R
                  U R                  U R                  -  SS9U l        [         R                  " U R
                  U R                  U R                  -  SS9U l        [         R                  " U R                  U R                  -  U R
                  SS9U l        g )NÚhead_dimr!   ç        F©Úbias)r   ÚModulerX   r)   r}   rZ   Únum_attention_headsÚ	num_headsÚnum_key_value_headsÚnum_key_value_groupsÚgetattrr€   ÚscalingÚattention_dropoutr~   ÚLinearÚq_projÚk_projÚv_projÚo_proj)r=   r)   r}   r~   s       r@   rX   ÚDiaSelfAttention.__init__s   sF  € Ü
�	‰	×Ñ˜4Ô ØŒØ"ŒØ!×-Ñ-ˆÔØŸ™×8Ñ8ˆŒØ#'§;¡;×#BÑ#B×#TÀdÇnÁnˆÔ Ø$(§N¡N°d×6NÑ6NÑ$NˆÔ!Ü ¨
°F×4FÑ4FÈ$Ï.É.Ñ4XÓYˆŒØˆŒØ!$ˆÔØ"Œä—i’i × 0Ñ 0°$·.±.À4Ç=Á=Ñ2PÐW\Ñ]ˆŒÜ—i’i × 0Ñ 0°$×2JÑ2JÈTÏ]É]Ñ2ZÐafÑgˆŒÜ—i’i × 0Ñ 0°$×2JÑ2JÈTÏ]É]Ñ2ZÐafÑgˆŒÜ—i’i §¡°·±Ñ >À×@PÑ@PÐW\Ñ]ˆ�rB   )r‹   r)   r€   rZ   r~   rŽ   r}   r†   rˆ   r‡   r�   r�   rŠ   r�   N)F)rD   rE   rF   rG   rl   r$   r#   ÚintÚboolrX   rQ   rC   rB   r@   rz   rz   p   s1   † ÙGñ^Ð/Ð2BÑBð ^Èsð ^Ð_c÷ ^ð ^rB   rz   c                   óè   ^ • \ rS rSrSrS\S\4U 4S jjr  SS\R                  S\R                  S	\R                  S-  S
\
S-  S\\   S\\R                  \R                  S-  4   4S jjrSrU =r$ )ÚDiaCrossAttentioné†   r|   r)   r}   c                 óR  >• [         TU ]  5         Xl        X l        UR                  U l        UR
                  U l        U R                  R                  U l        U R                  R                  U l	        U R                  U R                  -  U l
        UR                  U l        SU l        SU l        SU l        [         R"                  " U R                  U R                  U R                  -  SS9U l        [         R"                  " U R
                  U R                  U R                  -  SS9U l        [         R"                  " U R
                  U R                  U R                  -  SS9U l        [         R"                  " U R                  U R                  -  U R                  SS9U l        g )Nr!   r�   Fr‚   )r1   rX   r)   r}   rZ   Úcross_hidden_sizeÚcross_num_attention_headsr†   Úcross_num_key_value_headsr‡   rˆ   Úcross_head_dimr€   rŠ   r‹   r~   r   rŒ   r�   rŽ   r�   r�   ©r=   r)   r}   r?   s      €r@   rX   ÚDiaCrossAttention.__init__‰   s;  ø€ Ü‰ÑÔØŒØ"ŒØ!×-Ñ-ˆÔØ!'×!9Ñ!9ˆÔØŸ™×>Ñ>ˆŒØ#'§;¡;×#HÑ#HˆÔ Ø$(§N¡N°d×6NÑ6NÑ$NˆÔ!Ø×-Ñ-ˆŒØˆŒØ!$ˆÔØˆŒä—i’i × 0Ñ 0°$·.±.À4Ç=Á=Ñ2PÐW\Ñ]ˆŒÜ—i’i × 6Ñ 6¸×8PÑ8PÐSW×S`ÑS`Ñ8`ÐglÑmˆŒÜ—i’i × 6Ñ 6¸×8PÑ8PÐSW×S`ÑS`Ñ8`ÐglÑmˆŒÜ—i’i §¡°·±Ñ >À×@PÑ@PÐW\Ñ]ˆ�rB   NÚhidden_statesÚcross_attention_statesÚattention_maskÚpast_key_valuesÚkwargsr_   c                 óR  • UR                   S S n/ UQSPU R                  P7n/ UR                   S S QSPU R                  P7nU R                  U5      R                  U5      R	                  SS5      n	Ub%  UR
                  R                  U R                  5      OSn
Ubb  U
(       a[  UR                  R                  U R                     R                  nUR                  R                  U R                     R                  nO¥U R                  U5      R                  U5      R	                  SS5      nU R                  U5      R                  U5      R	                  SS5      nUbB  UR                  R                  UUU R                  5      u  p¼SUR
                  U R                  '   [        R                   " U R"                  R$                  [&        5      nU" U U	UUU4SU R(                  0UD6u  pïUR+                  / UQSP75      R-                  5       nU R/                  U5      nXï4$ )Nra   r!   r   FTrŠ   )rf   r€   r�   re   Ú	transposeÚ
is_updatedÚgetr}   Úcross_attention_cacheÚlayersÚkeysÚvaluesrŽ   r�   Úupdater   Úget_interfacer)   Ú_attn_implementationr   rŠ   ÚreshapeÚ
contiguousr�   )r=   rž   rŸ   r    r¡   r¢   Úinput_shapeÚhidden_shapeÚcross_shapeÚquery_statesr¥   Ú
key_statesÚvalue_statesÚattention_interfaceÚattn_outputÚattn_weightss                   r@   rj   ÚDiaCrossAttention.forwardœ   s  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆØMÐ.×4Ñ4°S°bÐ9ÐM¸2ÐM¸t¿}¹}ÑMˆà—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆàGVÑGb�_×/Ñ/×3Ñ3°D·N±NÔCÐhmˆ
ØÑ&®:à(×>Ñ>×EÑEÀdÇnÁnÑU×ZÑZˆJØ*×@Ñ@×GÑGÈÏÉÑW×^Ñ^‰LàŸ™Ð%;Ó<×AÑAÀ+ÓN×XÑXÐYZÐ\]Ó^ˆJØŸ;™;Ð'=Ó>×CÑCÀKÓP×ZÑZÐ[\Ð^_Ó`ˆLàÑ*à+:×+PÑ+P×+WÑ+WØØ Ø—N‘Nó,Ñ(�
ð >B�×*Ñ*¨4¯>©>Ñ:ä(?×(MÒ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØñ%
ð —L‘Lð%
ð ñ%
Ñ!ˆð "×)Ñ)Ð*<¨KÐ*<¸Ñ*<Ó=×HÑHÓJˆØ—k‘k +Ó.ˆØÐ(Ð(rB   )r‹   r)   r˜   r€   rZ   r~   rŽ   r}   r†   rˆ   r‡   r�   r�   rŠ   r�   ©NN)rD   rE   rF   rG   rl   r#   r’   rX   r5   rm   r   r   r   Útuplerj   rQ   rR   rS   s   @r@   r•   r•   †   sœ   ø† ÙGð^Ð/ð ^¸C÷ ^ð. /3Ø6:ñ1)à—|‘|ð1)ð !&§¡ð1)ð Ÿ™ tÑ+ð	1)ð
 -¨tÑ3ð1)ð Ð-Ñ.ð1)ð 
ˆu�|‰|˜UŸ\™\¨DÑ0Ð0Ñ	1÷1)ó 1)rB   r•   c                   óÚ   ^ • \ rS rSrS\S\4U 4S jjr  SS\R                  S\	\R                  \R                  4   S-  S\R                  S-  S	\
\   S
\R                  4
S jjrSrU =r$ )r,   éÐ   r)   r}   c                 óì   >• [         TU ]  5         [        UR                  UR                  S9U l        [        XSS9U l        [        UR                  UR                  S9U l        [        U5      U l
        g )N©ÚepsF©r~   )r1   rX   rt   rZ   Únorm_epsÚpre_sa_normrz   Úself_attentionÚpost_sa_normro   Úmlprœ   s      €r@   rX   ÚDiaEncoderLayer.__init__Ñ   sZ   ø€ Ü‰ÑÔÜ% f×&8Ñ&8¸f¿o¹oÑNˆÔÜ.¨vÈEÑRˆÔÜ& v×'9Ñ'9¸v¿¹ÑOˆÔÜ˜&“>ˆ�rB   Nrž   Úposition_embeddingsr    r¢   r_   c                 ó´   • UnU R                  U5      nU R                  " U4UUS.UD6u  pxXW-   nUnU R                  U5      nU R                  U5      n	XY-   nU$ )N)rÈ   r    )rÃ   rÄ   rÅ   rÆ   )
r=   rž   rÈ   r    r¢   ÚresidualÚnormed_statesÚself_attn_outputÚ_Úmlp_outs
             r@   rj   ÚDiaEncoderLayer.forwardØ   s€   € ð !ˆØ×(Ñ(¨Ó7ˆØ"×1Ò1Øð
à 3Ø)ñ
ð ñ	
ÑÐð !Ñ3ˆà ˆØ×)Ñ)¨-Ó8ˆØ—(‘(˜=Ó)ˆØ Ñ*ˆàÐrB   )rÆ   rÅ   rÃ   rÄ   rº   )rD   rE   rF   rG   r$   r’   rX   r5   rm   r»   r   r   rj   rQ   rR   rS   s   @r@   r,   r,   Ð   s‰   ø† ð"Ð/ð "¸C÷ "ð IMØ.2ñ	à—|‘|ðð # 5§<¡<°·±Ð#=Ñ>ÀÑEðð Ÿ™ tÑ+ð	ð
 Ð-Ñ.ðð 
�‰÷ó rB   r,   c                   ó¬   ^ • \ rS rSr\\S.rS\4U 4S jjr\	\
\ SS\R                  S\R                  S-  S\\   S	\4S
 jj5       5       5       rSrU =r$ )Ú
DiaEncoderéñ   )rž   Ú
attentionsr)   c           	      ó¬  >• [         TU ]  U5        Xl        [        R                  " UR
                  UR                  5      U l        [        R                  " [        UR                  5       Vs/ s H  n[        X5      PM     sn5      U l        [        UR                  UR                  S9U l        [!        US9U l        U R%                  5         g s  snf ©Nr¿   ©r)   )r1   rX   r)   r   rY   r9   rZ   Ú	embeddingÚ
ModuleListÚrangeÚnum_hidden_layersr,   r¨   rt   rÂ   Únormrw   Ú
rotary_embÚ	post_initrœ   s      €r@   rX   ÚDiaEncoder.__init__÷   s�   ø€ Ü‰Ñ˜Ô ØŒäŸš f×&7Ñ&7¸×9KÑ9KÓLˆŒÜ—m’mÜAFÀv×G_ÑG_ÔA`ÓaÒA`°IŒ_˜VÖ/ÑA`Ñaó
ˆŒô ˜v×1Ñ1°v·±ÑGˆŒ	Ü,°FÑ;ˆŒà�‰Õùò bs   Á.CNr+   r    r¢   r_   c                 óF  • U R                  U5      n[        R                  " UR                  S   UR                  S9S S S 24   n[        U R                  UUS9nU R                  XES9nU R                   H  nU" U4UUUS.UD6nM     U R                  U5      n[        US9$ )Nra   ©rd   )r)   Úinputs_embedsr    ©Úposition_ids)r    rã   rÈ   )Úlast_hidden_state)r×   r5   r6   rf   rd   r	   r)   rÜ   r¨   rÛ   r   )r=   r+   r    r¢   rž   rã   rÈ   Úencoder_layers           r@   rj   ÚDiaEncoder.forward  s·   € ð Ÿ™ yÓ1ˆô
 —|’| I§O¡O°BÑ$7À	×@PÑ@PÑQÐRVÒXYÐRYÑZˆä2Ø—;‘;Ø'Ø)ñ
ˆð
 #Ÿo™o¨m˜oÐWÐà!Ÿ[œ[ˆMÙ)Øðà-Ø)Ø$7ñ	ð
 ñŠMñ )ð Ÿ	™	 -Ó0ˆä°Ñ?Ð?rB   )r)   r×   r¨   rÛ   rÜ   ©N)rD   rE   rF   rG   r,   rz   Ú_can_record_outputsr$   rX   r   r   r   r5   rm   r   r   r   rj   rQ   rR   rS   s   @r@   rÑ   rÑ   ñ   s†   ø† à(Ø&ñÐð
Ð/÷ ð  ØØð /3ñ@à—<‘<ð@ð Ÿ™ tÑ+ð@ð Ð+Ñ,ð	@ð
 
ô@ó ó ó  ö@rB   rÑ   c                   ó\  ^ • \ rS rSrS\S\4U 4S jjr     SS\R                  S\	\R                  \R                  4   S-  S\R                  S-  S	\R                  S-  S
\R                  S-  S\
S-  S\	\R                  \R                  S-  \R                  S-  4   4S jjrSrU =r$ )r-   i)  r)   r}   c                 ót  >• [         TU ]  5         UR                  U l        [	        XSS9U l        [        X5      U l        [        UR                  UR                  S9U l
        [        UR                  UR                  S9U l        [        UR                  UR                  S9U l        [        U5      U l        g )NTrÁ   r¿   )r1   rX   rZ   Ú	embed_dimrz   rÄ   r•   Úcross_attentionrt   rÂ   rÃ   Úpre_ca_normÚpre_mlp_normro   rÆ   rœ   s      €r@   rX   ÚDiaDecoderLayer.__init__*  s�   ø€ Ü‰ÑÔØ×+Ñ+ˆŒÜ.¨vÈDÑQˆÔÜ0°ÓCˆÔÜ% f×&8Ñ&8¸f¿o¹oÑNˆÔÜ% f×&8Ñ&8¸f¿o¹oÑNˆÔÜ& v×'9Ñ'9¸v¿¹ÑOˆÔÜ˜&“>ˆ�rB   Nrž   rÈ   r    Úencoder_hidden_statesÚencoder_attention_maskr¡   r_   c                 óZ  • Un[        U[        5      (       a  UR                  nUn	U R                  U5      n
U R                  " U
UUU40 UD6u  p¼X›-   nUn	U R                  U5      n
U R                  " U
U4UUS.UD6u  pÜX�-   nUn	U R                  U5      n
U R                  U
5      nXž-   nU$ )N)r    r¡   )	r3   r   Úself_attention_cacherÃ   rÄ   rí   rì   rî   rÆ   )r=   rž   rÈ   r    rð   rñ   r¡   r¢   Úself_attn_cacherÊ   rË   rÌ   rÍ   Úcross_statesrÎ   s                  r@   rj   ÚDiaDecoderLayer.forward4  sì   € ð *ˆÜ�oÔ':×;Ñ;Ø-×BÑBˆOà ˆØ×(Ñ(¨Ó7ˆØ"×1Ò1ØØØð ñ
ð ñ
ÑÐð !Ñ3ˆà ˆØ×(Ñ(¨Ó7ˆØ×.Ò.ØØ!ð
ð 2Ø+ñ	
ð
 ñ
‰ˆð !Ñ/ˆà ˆØ×)Ñ)¨-Ó8ˆØ—(‘(˜=Ó)ˆØ Ñ*ˆàÐrB   )rì   rë   rÆ   rí   rî   rÃ   rÄ   ©NNNNN)rD   rE   rF   rG   r#   r’   rX   r5   rm   r»   r   rj   rQ   rR   rS   s   @r@   r-   r-   )  sÕ   ø† ð"Ð/ð "¸C÷ "ð IMØ.2Ø59Ø6:Ø6:ñ+à—|‘|ð+ð # 5§<¡<°·±Ð#=Ñ>ÀÑEð+ð Ÿ™ tÑ+ð	+ð
  %Ÿ|™|¨dÑ2ð+ð !&§¡¨tÑ 3ð+ð -¨tÑ3ð+ð 
ˆu�|‰|˜UŸ\™\¨DÑ0°%·,±,ÀÑ2EÐEÑ	F÷+ó +rB   r-   c                   ó$  ^ • \ rS rSrSr\\\S.rS\	4U 4S jjr
\\\     SS\R                  S\R                   S-  S	\R                  S-  S
\R"                  S-  S\R                   S-  S\S-  S\\   S\\-  4S jj5       5       5       rSrU =r$ )Ú
DiaDecoderib  z-Transformer Decoder Stack using DenseGeneral.)rž   rÓ   Úcross_attentionsr)   c           	      ó¤  >• [         TU ]  U5        UR                  U l        UR                  U l        [	        U5      U l        [        R                  " [        UR                  5       Vs/ s H  n[        X5      PM     sn5      U l        [        UR                  UR                  S9U l        [!        US9U l        U R%                  5         g s  snf rÕ   )r1   rX   r7   r9   r4   Ú
embeddingsr   rØ   rÙ   rÚ   r-   r¨   rt   rZ   rÂ   rÛ   rw   rÜ   rÝ   rœ   s      €r@   rX   ÚDiaDecoder.__init__k  s¡   ø€ Ü‰Ñ˜Ô Ø"×/Ñ/ˆÔØ ×+Ñ+ˆŒÜ2°6Ó:ˆŒÜ—m’mÜAFÀv×G_ÑG_ÔA`ÓaÒA`°IŒ_˜VÖ/ÑA`Ñaó
ˆŒô ˜v×1Ñ1°v·±ÑGˆŒ	Ü,°FÑ;ˆŒà�‰Õùò bs   Á*CNr+   rã   r    rð   rñ   r¡   r¢   r_   c                 ó>  • UR                  5       SS u  p‰Ub  UR                  5       OSn
Uc2  [        R                  " X‘R                  S9U
-   nUR                  S5      nU R                  U5      nUc2  [        5       (       d#  X©-   n[        R                  " XŒUR                  S9n[        U R                  UUUS9n[        U R                  UUUS9nU R                  X²S9nU R                   H  nU" UUUU4UUUS.UD6nM     U R                  U5      n[        UUS	9$ )
zç
input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length, num_codebooks)`):
    The original `decoder_input_ids` in 3D shape to facilitate more efficient computations.

    [What are input IDs?](../glossary#input-ids)
Nra   r   rà   )r)   rá   r    r¡   )r)   rá   r    rð   râ   )rñ   r¡   rã   )rä   r¡   )ÚsizeÚget_seq_lengthr5   r6   rd   Ú	unsqueezerü   r   Úonesr
   r)   r	   rÜ   r¨   rÛ   r   )r=   r+   rã   r    rð   rñ   r¡   r¢   Ú
batch_sizeÚ
seq_lengthÚpast_key_values_lengthrž   Úmask_seq_lengthrÈ   Úlayers                  r@   rj   ÚDiaDecoder.forwardx  sK  € ð( "+§¡Ó!1°#°2Ð!6Ñˆ
ØETÑE` ×!?Ñ!?Ô!AÐfgÐàÑÜ Ÿ<š<¨
×;KÑ;KÑLÐOeÑeˆLØ'×1Ñ1°!Ó4ˆLð Ÿ™¨	Ó2ˆàÑ!Ô*B×*DÑ*Dà4ÑAˆOÜ"ŸZšZ¨
ÈI×L\ÑL\Ñ]ˆNä+Ø—;‘;Ø'Ø)Ø+ñ	
ˆô ";Ø—;‘;Ø'Ø1Ø"7ñ	"
Ðð #Ÿo™o¨m˜oÐWÐà—[”[ˆEÙ!Øð $ØØ%ðð (>Ø /Ø)ñð ñŠMñ !ð Ÿ	™	 -Ó0ˆä8Ø+Ø+ñ
ð 	
rB   )rü   r¨   rÛ   r7   rÜ   r9   r÷   )rD   rE   rF   rG   rl   r-   rz   r•   rè   r#   rX   r   r   r   r5   rm   Ú
LongTensorÚFloatTensorr   r   r   r   r»   rj   rQ   rR   rS   s   @r@   rù   rù   b  sñ   ø† Ù7ð )Ø&Ø-ñÐðÐ/÷ ð  ØØð 15Ø.2Ø:>Ø:>Ø6:ñA
à—<‘<ðA
ð ×&Ñ&¨Ñ-ðA
ð Ÿ™ tÑ+ð	A
ð
  %×0Ñ0°4Ñ7ðA
ð !&× 0Ñ 0°4Ñ 7ðA
ð -¨tÑ3ðA
ð Ð+Ñ,ðA
ð 
3°UÑ	:ôA
ó ó ó  öA
rB   rù   z[
    The bare Dia model outputting raw hidden-states without any specific head on top.
    )Úcustom_introc                   ó&  ^ • \ rS rSrS\4U 4S jjr\\        SS\R                  S-  S\R                  S-  S\R                  S-  S\R                  S-  S	\R                  S-  S
\
\-  S-  S\S-  S\S-  S\\-  4S jj5       5       rSrU =r$ )ÚDiaModeli¿  r)   c                 ó¸   >• [         TU ]  U5        Xl        [        UR                  5      U l        [        UR                  5      U l        U R                  5         g rç   )
r1   rX   r)   rÑ   Úencoder_configÚencoderrù   Údecoder_configÚdecoderrÝ   ©r=   r)   r?   s     €r@   rX   ÚDiaModel.__init__Å  sC   ø€ Ü‰Ñ˜Ô ØŒÜ! &×"7Ñ"7Ó8ˆŒÜ! &×"7Ñ"7Ó8ˆŒØ�‰ÕrB   Nr+   r    Údecoder_input_idsÚdecoder_position_idsÚdecoder_attention_maskÚencoder_outputsr¡   Ú	use_cacher_   c	                 ó  • Uc  Uc  [        S5      eU R                  (       a/  U R                  (       a  U(       a  [        R	                  S5        SnU(       a1  Uc.  [        [        U R                  S9[        U R                  S95      nUc  U R                  " SUUS.U	D6nOK[        U[        5      (       d6  [        US   [        U5      S:”  a  US   OS[        U5      S	:”  a  US	   OSS
9nUS   R                  S   SU R                  R                  R                  pËn
UcA  [        R                   " U
SU4U R                  R                  R"                  U R$                  S9nUR&                  S	:X  a"  UR)                  X¬U5      R+                  SS	5      nU R,                  " SUUUUS   UUUS.U	D6n[/        UR0                  UR2                  UR4                  UR6                  UR8                  US   UR4                  UR6                  S9$ )aÜ  
decoder_input_ids (`torch.LongTensor` of shape `(batch_size * num_codebooks, target_sequence_length)
or (batch_size, target_sequence_length, num_codebooks)`, *optional*):
    1. (batch_size * num_codebooks, target_sequence_length): corresponds to the general use case where
    the audio input codebooks are flattened into the batch dimension. This also aligns with the flat-
    tened audio logits which are used to calculate the loss.

    2. (batch_size, sequence_length, num_codebooks): corresponds to the internally used shape of
    Dia to calculate embeddings and subsequent steps more efficiently.

    If no `decoder_input_ids` are provided, it will create a tensor of `bos_token_id` with shape
    `(batch_size, 1, num_codebooks)`. Indices can be obtained using the [`DiaProcessor`]. See
    [`DiaProcessor.__call__`] for more details.

    [What are decoder input IDs?](../glossary#decoder-input-ids)
decoder_position_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`):
    Indices of positions of each input sequence tokens in the position embeddings.
    Used to calculate the position embeddings up to `config.decoder_config.max_position_embeddings`.

    [What are position IDs?](../glossary#position-ids)
NzXYou should either provide text ids or the cached text encodings. Neither has been found.zZ`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...FrÖ   )r+   r    r   r!   r   )rä   rž   rÓ   ra   )rÿ   Ú
fill_valuerd   )r+   rã   r    rð   rñ   r¡   r  )rä   r¡   Údecoder_hidden_statesÚdecoder_attentionsrú   Úencoder_last_hidden_staterð   Úencoder_attentionsrC   )Ú
ValueErrorÚis_gradient_checkpointingÚtrainingÚloggerÚwarning_oncer   r   r)   r  r3   r   Úlenrf   r  r7   r5   ÚfullÚbos_token_idrd   Úndimr®   r¤   r  r   rä   r¡   rž   rÓ   rú   )r=   r+   r    r  r  r  r  r¡   r  r¢   ÚbszÚseq_lenÚchannelsÚdecoder_outputss                 r@   rj   ÚDiaModel.forwardÌ  s  € ðH Ñ Ñ!8ÜØjóð ð ×)×)¨d¯m¯mÞÜ×#Ñ#Øpôð "�	æ˜Ñ0Ü1´,ÀdÇkÁkÑ2RÔT`Ðhl×hsÑhsÑTtÓuˆOàÑ"Ø"Ÿlšlð Ø#Ø-ñð ñ‰Oô ˜O¬_×=Ñ=Ü-Ø"1°!Ñ"4Ü47¸Ó4HÈ1Ó4L˜o¨aÒ0ÐRVÜ14°_Ó1EÈÓ1I˜?¨1Ò-ÈtñˆOð #2°!Ñ"4×":Ñ":¸1Ñ"=¸rÀ4Ç;Á;×C]ÑC]×CjÑCj�hˆØÑ$Ü %§
¢
Ø˜1˜hÐ'°D·K±K×4NÑ4N×4[Ñ4[Ðdh×doÑdoñ!Ðð ×!Ñ! QÓ&Ø 1× 9Ñ 9¸#ÈÓ Q× [Ñ [Ð\]Ð_`Ó aÐàŸ,š,ð 	
Ø'Ø-Ø1Ø"1°!Ñ"4Ø#1Ø+Øñ	
ð ñ	
ˆô "Ø-×?Ñ?Ø+×;Ñ;Ø"1×"?Ñ"?Ø.×9Ñ9Ø,×=Ñ=Ø&5°aÑ&8Ø"1×"?Ñ"?Ø.×9Ñ9ñ	
ð 		
rB   )r)   r  r  )NNNNNNNN)rD   rE   rF   rG   r"   rX   r   r   r5   r	  r   r»   r   r“   r   rj   rQ   rR   rS   s   @r@   r  r  ¿  sø   ø† ð˜y÷ ð Øð .2Ø26Ø59Ø8<Ø:>Ø:>Ø6:Ø!%ñ]
à×#Ñ# dÑ*ð]
ð ×(Ñ(¨4Ñ/ð]
ð !×+Ñ+¨dÑ2ð	]
ð
 $×.Ñ.°Ñ5ð]
ð !&× 0Ñ 0°4Ñ 7ð]
ð )¨5Ñ0°4Ñ7ð]
ð -¨tÑ3ð]
ð ˜$‘;ð]
ð 
Ð#Ñ	#ô]
ó ó ö]
rB   r  zl
    The Dia model consisting of a (byte) text encoder and audio decoder with a prediction head on top.
    c                   óN  ^ • \ rS rSrSrSrS\4U 4S jjr\\	         SS\
R                  S-  S\
R                  S-  S	\
R                  S-  S
\
R                  S-  S\
R                  S-  S\\-  S-  S\S-  S\S-  S\
R                  S-  S\\-  4S jj5       5       rSrU =r$ )ÚDiaForConditionalGenerationi.  r*   )Úaudior)   c                 óv  >• [         TU ]  U5        Xl        [        U5      U l        UR
                  R                  U l        UR
                  R                  U l        [        R                  " UR
                  R                  U R                  U R                  -  SS9U l        SU l        U R                  5         g )NFr‚   ÚForMaskedLM)r1   rX   r)   r  r*   r  r7   r9   r   rŒ   rZ   Úlogits_denseÚ	loss_typerÝ   r  s     €r@   rX   Ú$DiaForConditionalGeneration.__init__7  s‘   ø€ Ü‰Ñ˜Ô ØŒÜ˜fÓ%ˆŒ
à"×1Ñ1×>Ñ>ˆÔØ ×/Ñ/×:Ñ:ˆŒÜŸIšIØ×!Ñ!×-Ñ-°×0AÑ0AÀDÇOÁOÑ0SÐ[`ñ
ˆÔð 'ˆŒð 	�‰ÕrB   Nr+   r    r  r  r  r  r¡   r  Úlabelsr_   c
                 óP  • U R                   " S	UUUUUUUUS.U
D6nUS   nUR                  S   nU R                  U5      R                  USU R                  U R
                  45      R                  SS5      R                  5       R                  XÐR                  -  SU R
                  5      nSnU	b  U R                  " S	XéU R
                  S.U
D6n[        UUUR                  UR                  UR                  UR                  UR                  UR                  UR                   S9	$ )
a   
decoder_input_ids (`torch.LongTensor` of shape `(batch_size * num_codebooks, target_sequence_length)
or (batch_size, target_sequence_length, num_codebooks)`, *optional*):
    1. (batch_size * num_codebooks, target_sequence_length): corresponds to the general use case where
    the audio input codebooks are flattened into the batch dimension. This also aligns with the flat-
    tened audio logits which are used to calculate the loss.

    2. (batch_size, sequence_length, num_codebooks): corresponds to the internally used shape of
    Dia to calculate embeddings and subsequent steps more efficiently.

    If no `decoder_input_ids` are provided, it will create a tensor of `bos_token_id` with shape
    `(batch_size, 1, num_codebooks)`. Indices can be obtained using the [`DiaProcessor`]. See
    [`DiaProcessor.__call__`] for more details.

    [What are decoder input IDs?](../glossary#decoder-input-ids)
decoder_position_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`):
    Indices of positions of each input sequence tokens in the position embeddings.
    Used to calculate the position embeddings up to `config.decoder_config.max_position_embeddings`.

    [What are position IDs?](../glossary#position-ids)
labels (`torch.LongTensor` of shape `(batch_size * num_codebooks,)`, *optional*):
    Labels for computing the masked language modeling loss. Indices should either be in
    `[0, ..., config.decoder_config.vocab_size - 1]` or -100. Tokens with indices set to `-100`
    are ignored (masked).
)r+   r    r  r  r  r  r¡   r  r   ra   r!   r   N)Úlogitsr6  r9   )	Úlossr8  r¡   r  r  rú   r  rð   r  rC   )r*   rf   r3  re   r7   r9   r¤   r¯   Úloss_functionr   r¡   r  r  rú   r  rð   r  )r=   r+   r    r  r  r  r  r¡   r  r6  r¢   Úoutputsrä   r  Úaudio_logitsr9  s                   r@   rj   Ú#DiaForConditionalGeneration.forwardF  s7  € ðR —*’*ð 

ØØ)Ø/Ø!5Ø#9Ø+Ø+Øñ

ð ñ

ˆð $ A™JÐØ&×,Ñ,¨QÑ/ˆ
ð ×ÑÐ/Ó0ß‰T�:˜r 4×#4Ñ#4°d·o±oÐFÓGß‰Y�q˜!‹_ß‰Z‹\ß‰T�*×0Ñ0Ñ0°"°d·o±oÓFð 	ð ˆØÑØ×%Ò%Ðo¨\ÐUY×UdÑUdÑoÐhnÑoˆDäØØØ#×3Ñ3Ø")×"?Ñ"?Ø&×9Ñ9Ø$×5Ñ5Ø&-×&GÑ&GØ")×"?Ñ"?Ø&×9Ñ9ñ

ð 
	
rB   )r)   r3  r4  r*   r7   r9   )	NNNNNNNNN)rD   rE   rF   rG   rI   Úoutput_modalitiesr"   rX   r   r   r5   r	  r   r»   r   r“   r   rj   rQ   rR   rS   s   @r@   r/  r/  .  s  ø† ð  ÐØ"Ðð˜y÷ ð Øð .2Ø26Ø59Ø8<Ø:>Ø:>Ø6:Ø!%Ø*.ñL
à×#Ñ# dÑ*ðL
ð ×(Ñ(¨4Ñ/ðL
ð !×+Ñ+¨dÑ2ð	L
ð
 $×.Ñ.°Ñ5ðL
ð !&× 0Ñ 0°4Ñ 7ðL
ð )¨5Ñ0°4Ñ7ðL
ð -¨tÑ3ðL
ð ˜$‘;ðL
ð × Ñ  4Ñ'ðL
ð 
�Ñ	 ôL
ó ó öL
rB   r/  )r  r'   r/  )Erl   Úcollections.abcr   r5   r   Ú r   r:   Úcache_utilsr   r   Úmasking_utilsr	   r
   Úmodeling_flash_attention_utilsr   Úmodeling_layersr   Úmodeling_outputsr   r   r   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   r   r   Úutils.genericr   Úutils.output_capturingr   Úllama.modeling_llamar   r   r   r   Úphi3.modeling_phi3r    Úconfiguration_diar"   r#   r$   Úgeneration_diar%   Ú
get_loggerrD   r#  r'   r„   r4   ro   rt   rw   rz   r•   r,   rÑ   r-   rù   r  r/  Ú__all__rC   rB   r@   Ú<module>rQ     sw  ðñ å $ã Ý å &ß <ß JÝ BÝ 9÷ó ÷ GÝ &ß lÕ lÝ 7Ý 5÷ó õ )ß LÑ LÝ .ð 
×	Ò	˜HÓ	%€ð ô0˜ó 0ó ð0ô$!˜rŸy™yô !ô<	ˆWô 	ô	�ô 	ô	Ð-ô 	ô^�~ô ^ô,G)˜Ÿ	™	ô G)ôTÐ0ô ôB5@Ð#ô 5@ôp6Ð0ô 6ôrZ
Ð#ô Z
ñz ðñô
g
Ð!ó g
óð
g
ñT ðñô
a
Ð"4Ð6Hó a
óð
a
òH L�rB   