ó
    oM‡j9  ã                   óÆ  • S SK r S SKJr  S SKJs  Jr  SSKJrJr  SSK	J
r
  SSKJr  SSKJr  SSKJr  S	\ R"                  S
\ R"                  4S jr SS	\ R"                  S\ R"                  S\ R"                  S\S
\ R"                  4
S jjr " S S\R*                  5      r " S S5      r " S S\R*                  \5      r " S S\R*                  5      r " S S\\\
5      rg)é    Né   )ÚConfigMixinÚregister_to_config)ÚPeftAdapterMixiné   )ÚAttentionModuleMixin)Údispatch_attention_fn)Ú
ModelMixinÚhidden_statesÚreturnc                 ó–   • U SS U R                   S   S-  24   nU SU R                   S   S-  S 24   n[        R                  " U* U4SS9$ )N.éÿÿÿÿr   ©Údim)ÚshapeÚtorchÚcat)r   Úhidden_states_1Úhidden_states_2s      ÚËC:\Users\johnn\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.13_qbz5n2kfra8p0\LocalCache\local-packages\Python313\site-packages\diffusers/models/condition_embedders/condition_embedder_anima.pyÚ_rotate_halfr      sb   € Ø# CÐ)G¨=×+>Ñ+>¸rÑ+BÀaÑ+GÐ)GÐ$GÑH€OØ# C¨×)<Ñ)<¸RÑ)@ÀAÑ)EÑ)GÐ$GÑH€OÜ�9Š9�Ð&¨Ð8¸bÑAÐAó    ÚcosÚsinÚunsqueeze_dimc                 ól   • UR                  U5      nUR                  U5      nX-  [        U 5      U-  -   $ ©N)Ú	unsqueezer   )r   r   r   r   s       r   Ú_apply_rotary_pos_embr       s8   € ð �-‰-˜Ó
&€CØ
�-‰-˜Ó
&€CØÑ¤L°Ó$?À#Ñ$EÑFÐFr   c                   ó¨   ^ • \ rS rSrS
S\S\4U 4S jjjrS\R                  S\R                  S\	\R                  \R                  4   4S jr
S	rU =r$ )ÚAnimaRotaryEmbeddingé(   Úhead_dimÚ
rope_thetac           	      óÖ   >• [         TU ]  5         SU[        R                  " SUS[        R                  S9R                  [        R                  S9U-  -  -  nU R                  SUSS9  g )Ng      ð?r   r   ©ÚdtypeÚinv_freqF)Ú
persistent)ÚsuperÚ__init__r   ÚarangeÚint64ÚtoÚfloat32Úregister_buffer)Úselfr#   r$   r(   Ú	__class__s       €r   r+   ÚAnimaRotaryEmbedding.__init__)   se   ø€ Ü‰ÑÔØØœ5Ÿ<š<¨¨8°Q¼e¿k¹kÑJ×MÑMÔTY×TaÑTaÐMÐbÐemÑmÑnñ
ˆð 	×Ñ˜Z¨¸eÐÒDr   r   Úposition_idsr   c                 óä  • U R                   S S S 2S 4   R                  5       R                  UR                  S   SS5      nUR	                  UR
                  5      nUS S 2S S S 24   R                  5       nX4-  R                  SS5      n[        R                  " XU4SS9nUR                  5       nUR                  5       nUR	                  UR                  S9UR	                  UR                  S94$ )Nr   r   é   r   r   r&   )r(   ÚfloatÚexpandr   r.   ÚdeviceÚ	transposer   r   r   r   r'   )	r1   r   r4   Úinv_freq_expandedÚposition_ids_expandedÚfreqsÚembr   r   s	            r   ÚforwardÚAnimaRotaryEmbedding.forward0   sØ   € Ø ŸM™M¨$²°4¨-Ñ8×>Ñ>Ó@×GÑGÈ×HZÑHZÐ[\ÑH]Ð_aÐcdÓeÐØ-×0Ñ0°×1EÑ1EÓFÐØ ,ªQ°²a¨ZÑ 8× >Ñ >Ó @Ðà"Ñ:×EÑEÀaÈÓKˆÜ�iŠi˜˜¨BÑ/ˆØ�g‰g‹iˆØ�g‰g‹iˆà�v‰v˜M×/Ñ/ˆvÐ0°#·&±&¸}×?RÑ?R°&Ð2SÐSÐSr   © )g     ˆÃ@)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Úintr7   r+   r   ÚTensorÚtupler?   Ú__static_attributes__Ú__classcell__©r2   s   @r   r!   r!   (   sd   ø† ñE ð E°%÷ Eð Eð
T U§\¡\ð 
TÀÇÁð 
TÐRWÐX]×XdÑXdÐfk×frÑfrÐXrÑRs÷ 
Tò 
Tr   r!   c                   ó  • \ rS rSrSrSr    SSSS\R                  S\R                  S-  S\R                  S-  S\\R                  \R                  4   S-  S	\\R                  \R                  4   S-  S
\R                  4S jjr	Sr
g)Ú!AnimaTextConditionerAttnProcessoré=   NÚattnÚAnimaTextConditionerAttentionr   Úattention_maskÚencoder_hidden_statesÚposition_embeddingsÚencoder_position_embeddingsr   c                 óþ  • Uc  UOUnUR                   S S nUR                   S S nUR                  U5      n	UR                  U5      n
UR                  U5      nU	R                  " / UQUR
                  PUR                  P76 n	U
R                  " / UQUR
                  PUR                  P76 n
UR                  " / UQUR
                  PUR                  P76 nUR                  U	5      n	UR                  U
5      n
Ub,  Uc  [        S5      eUu  pÍ[        XœUSS9n	Uu  pÍ[        X¬USS9n
[        U	U
UUSSU R                  U R                  S9nUR                  SS5      R                  5       nUR!                  U5      nU$ )	Nr   zL`encoder_position_embeddings` must be provided when using rotary embeddings.r   )r   g        F)Ú	attn_maskÚ	dropout_pÚ	is_causalÚbackendÚparallel_configr   )r   Úq_projÚk_projÚv_projÚviewÚnum_attention_headsÚattention_head_dimÚq_normÚk_normÚ
ValueErrorr   r	   Ú_attention_backendÚ_parallel_configÚflattenÚ
contiguousÚo_proj)r1   rO   r   rQ   rR   rS   rT   Úinput_shapeÚencoder_input_shapeÚqueryÚkeyÚvaluer   r   s                 r   Ú__call__Ú*AnimaTextConditionerAttnProcessor.__call__A   s˜  € ð 2GÑ1N¡ÐTiÐØ#×)Ñ)¨#¨2Ð.ˆØ3×9Ñ9¸#¸2Ð>Ðà—‘˜MÓ*ˆØ�k‰kÐ/Ó0ˆØ—‘Ð1Ó2ˆà—
’
Ð[˜KÐ[¨×)AÑ)AÐ[À4×CZÑCZÒ[ˆØ�hŠhÐ_Ð+Ð_¨T×-EÑ-EÐ_Àt×G^ÑG^Ò_ˆØ—
’
ÐcÐ/Ðc°×1IÑ1IÐcÈ4×KbÑKbÒcˆà—‘˜EÓ"ˆØ�k‰k˜#ÓˆàÑ*Ø*Ñ2Ü Ð!oÓpÐpØ*‰HˆCÜ)¨%°cÈÑKˆEØ2‰HˆCÜ'¨°#ÀQÑGˆCä-ØØØØ$ØØØ×+Ñ+Ø ×1Ñ1ñ	
ˆð &×-Ñ-¨a°Ó3×>Ñ>Ó@ˆØŸ™ MÓ2ˆØÐr   rA   ©NNNN)rB   rC   rD   rE   rd   re   r   rG   rH   rn   rI   rA   r   r   rM   rM   =   s²   † ØÐØÐð /3Ø59ØHLØPTñ,à-ð,ð —|‘|ð,ð Ÿ™ tÑ+ð	,ð
  %Ÿ|™|¨dÑ2ð,ð # 5§<¡<°·±Ð#=Ñ>ÀÑEð,ð &+¨5¯<©<¸¿¹Ð+EÑ%FÈÑ%Mð,ð 
�‰÷,ð ,r   rM   c                   óT  ^ • \ rS rSr\r\/rSr SS\S\S\S\S\S-  4
U 4S	 jjjr	    SS
\
R                  S\
R                  S-  S\
R                  S-  S\\
R                  \
R                  4   S-  S\\
R                  \
R                  4   S-  S\
R                  4S jjrSrU =r$ )rP   ép   FNÚ	query_dimÚcontext_dimr_   r`   Ú	processorc                 óÂ  >• [         TU ]  5         X4-  nX0l        X@l        [        R
                  " XSS9U l        [        R                  " USS9U l        [        R
                  " X&SS9U l	        [        R                  " USS9U l
        [        R
                  " X&SS9U l        [        R
                  " XaSS9U l        Uc  U R                  5       nU R                  U5        g )NF)Úbiasç�íµ ÷Æ°>©Úeps)r*   r+   r_   r`   ÚnnÚLinearr[   ÚRMSNormra   r\   rb   r]   rh   Ú_default_processor_clsÚset_processor)r1   rs   rt   r_   r`   ru   Ú	inner_dimr2   s          €r   r+   Ú&AnimaTextConditionerAttention.__init__u   s´   ø€ ô 	‰ÑÔØ'Ñ<ˆ	à#6Ô Ø"4ÔÜ—i’i 	¸5ÑAˆŒÜ—j’jÐ!3¸Ñ>ˆŒÜ—i’i ¸UÑCˆŒÜ—j’jÐ!3¸Ñ>ˆŒÜ—i’i ¸UÑCˆŒÜ—i’i 	¸5ÑAˆŒàÑØ×3Ñ3Ó5ˆIØ×Ñ˜9Õ%r   r   rQ   rR   rS   rT   r   c           	      ó*   • U R                  U UUUUUS9$ )N©rQ   rR   rS   rT   )ru   )r1   r   rQ   rR   rS   rT   s         r   r?   Ú%AnimaTextConditionerAttention.forward�   s,   € ð �~‰~ØØØ)Ø"7Ø 3Ø(Cð ð 
ð 	
r   )r`   rb   r\   r_   rh   ra   r[   r]   r   rp   )rB   rC   rD   rE   rM   r~   Ú_available_processorsÚ_supports_qkv_fusionrF   r+   r   rG   rH   r?   rI   rJ   rK   s   @r   rP   rP   p   s÷   ø† Ø>ÐØ>Ð?ÐØ Ðð ?Cñ&àð&ð ð&ð !ð	&ð
  ð&ð 5°tÑ;÷&ð &ð6 /3Ø59ØHLØPTñ
à—|‘|ð
ð Ÿ™ tÑ+ð
ð  %Ÿ|™|¨dÑ2ð	
ð
 # 5§<¡<°·±Ð#=Ñ>ÀÑEð
ð &+¨5¯<©<¸¿¹Ð+EÑ%FÈÑ%Mð
ð 
�‰÷
ó 
r   rP   c                   ób  ^ • \ rS rSr    SS\S\S\S\S\S\4U 4S jjjr    SS
\R                  S\R                  S\R                  S	-  S\R                  S	-  S\
\R                  \R                  4   S	-  S\
\R                  \R                  4   S	-  S\R                  4S jjrSrU =r$ )ÚAnimaTextConditionerBlockéŸ   Ú
source_dimÚ	model_dimr_   Ú	mlp_ratioÚuse_self_attentionÚuse_layer_normc           
      ó$  >• [         T	U ]  5         XPl        U(       a  [        R                  O[        R
                  nU(       a  0 OSS0nU(       a!  U" U40 UD6U l        [        UUUX#-  S9U l        U" U40 UD6U l	        [        UUUX#-  S9U l
        U" U40 UD6U l        [        R                  " [        R                  " U[        X$-  5      5      [        R                  " 5       [        R                  " [        X$-  5      U5      5      U l        g )Nrz   rx   )rs   rt   r_   r`   )r*   r+   r�   r{   Ú	LayerNormr}   Únorm_self_attnrP   Ú	self_attnÚnorm_cross_attnÚ
cross_attnÚnorm_mlpÚ
Sequentialr|   rF   ÚGELUÚmlp)
r1   rŠ   r‹   r_   rŒ   r�   rŽ   Únorm_clsÚnorm_kwargsr2   s
            €r   r+   Ú"AnimaTextConditionerBlock.__init__    së   ø€ ô 	‰ÑÔØ"4ÔÞ#1”2—<’<´r·z±zˆÞ*‘b°¸°ˆæÙ"*¨9Ñ"D¸Ñ"DˆDÔÜ:Ø#Ø%Ø$7Ø#,Ñ#Cñ	ˆDŒNñ  (¨	ÑA°[ÑAˆÔÜ7ØØ"Ø 3Ø(Ñ?ñ	
ˆŒñ ! Ñ:¨kÑ:ˆŒÜ—=’=Ü�IŠI�i¤ YÑ%:Ó!;Ó<Ü�GŠG‹IÜ�IŠI”c˜)Ñ/Ó0°)Ó<ó
ˆ�r   Nr   rR   Útarget_attention_maskÚsource_attention_maskrS   Úsource_position_embeddingsr   c                 ó
  • U R                   (       a'  U R                  U5      nU R                  UUUUS9nX-   nU R                  U5      nU R	                  UUUUUS9nX-   nXR                  U R                  U5      5      -   nU$ )N)rQ   rS   rT   rƒ   )r�   r‘   r’   r“   r”   r˜   r•   )	r1   r   rR   rœ   r�   rS   rž   Únorm_hidden_statesÚattn_hidden_statess	            r   r?   Ú!AnimaTextConditionerBlock.forwardÅ   s¦   € ð ×"×"Ø!%×!4Ñ!4°]Ó!CÐØ!%§¡Ø"Ø4Ø$7Ø,?ð	 "0ð "Ðð *Ñ>ˆMà!×1Ñ1°-Ó@ÐØ!Ÿ_™_ØØ0Ø"7Ø 3Ø(Bð -ð 
Ðð &Ñ:ˆØ%¯©°·±¸}Ó1MÓ(NÑNˆØÐr   )r”   r˜   r“   r•   r‘   r’   r�   )é   ç      @TFrp   )rB   rC   rD   rE   rF   r7   Úboolr+   r   rG   rH   r?   rI   rJ   rK   s   @r   rˆ   rˆ   Ÿ   sþ   ø† ð
 $&ØØ#'Ø$ñ#
àð#
ð ð#
ð !ð	#
ð
 ð#
ð !ð#
ð ÷#
ð #
ðR 6:Ø59ØHLØOSñà—|‘|ðð  %Ÿ|™|ðð  %Ÿ|™|¨dÑ2ð	ð
  %Ÿ|™|¨dÑ2ðð # 5§<¡<°·±Ð#=Ñ>ÀÑEðð %*¨%¯,©,¸¿¹Ð*DÑ$EÈÑ$Lðð 
�‰÷ó r   rˆ   c                   ój  ^ • \ rS rSrSrSrS/r\          SS\S\S\S\S	\S
\	S\S\
S\
S\4U 4S jjj5       r\S\R                  S-  S\R                  S-  4S j5       r  SS\R                  S\R                  S\R                  S-  S\R                  S-  S\R                  4
S jjrSrU =r$ )ÚAnimaTextConditioneréå   a§  
Text conditioner used by Anima to map Qwen3 hidden states and T5 token ids to Cosmos text embeddings.

Anima reuses the Cosmos Predict2 DiT. The only model-specific conditioning module is this LLM adapter, which
cross-attends from learned T5 token embeddings to Qwen3 text encoder hidden states before the diffusion loop.
`target_dim` is the conditioner output dimension and must match the transformer's `text_embed_dim`.
Trˆ   rŠ   Ú
target_dimr‹   Ú
num_layersr_   rŒ   Útarget_vocab_sizer�   rŽ   Úmin_sequence_lengthc                 óê  >• [         TU ]  5         [        R                  " Xr5      U l        X2:w  a  [        R
                  " X#5      O[        R                  " 5       U l        [        X5-  5      U l	        [        R                  " [        U5       Vs/ s H  n[        UUUUUU	S9PM     sn5      U l        [        R
                  " X25      U l        [        R                  " USS9U l        SU l        g s  snf )N)rŠ   r‹   r_   rŒ   r�   rŽ   rx   ry   F)r*   r+   r{   Ú	EmbeddingÚembedr|   ÚIdentityÚin_projr!   Ú
rotary_embÚ
ModuleListÚrangerˆ   ÚblocksÚout_projr}   ÚnormÚgradient_checkpointing)r1   rŠ   r©   r‹   rª   r_   rŒ   r«   r�   rŽ   r¬   Ú_r2   s               €r   r+   ÚAnimaTextConditioner.__init__ñ   sÏ   ø€ ô 	‰ÑÔÜ—\’\Ð"3Ó@ˆŒ
Ø;DÓ;R”r—y’y Ô7ÔXZ×XcÒXcÓXeˆŒÜ.¨yÑ/OÓPˆŒÜ—m’mô ˜zÔ*ó
ò +�Aô *Ø)Ø'Ø(;Ø'Ø'9Ø#1ôñ +ñ
ó
ˆŒô Ÿ	š	 )Ó8ˆŒÜ—J’J˜z¨tÑ4ˆŒ	Ø&+ˆÕ#ùò
s   ÂC0rQ   Nr   c                 ó¬   • U c  g U R                  [        R                  5      n U R                  S:X  a   U R	                  S5      R	                  S5      n U $ )Nr   r6   )r.   r   r¥   Úndimr   )rQ   s    r   Ú_prepare_attention_maskÚ,AnimaTextConditioner._prepare_attention_mask  sN   € àÑ!ØØ'×*Ñ*¬5¯:©:Ó6ˆØ×Ñ !Ó#Ø+×5Ñ5°aÓ8×BÑBÀ1ÓEˆNØÐr   Úsource_hidden_statesÚtarget_input_idsrœ   r�   c           
      óB  • U R                  U5      nU R                  U5      nU R                  U5      R                  UR                  S9nU R	                  U5      n[
        R                  " UR                  S   UR                  S9R                  S5      n[
        R                  " UR                  S   UR                  S9R                  S5      nU R                  XV5      nU R                  XW5      n	U R                   HR  n
[
        R                  " 5       (       a*  U R                  (       a  U R                  U
UUUUUU	5      nMG  U
" UUUUUU	S9nMT     U R                  U R!                  U5      5      nUb@  XSR#                  S5      R#                  S5      R                  U5      R                  S5      -  nUR                  S   U R$                  R&                  :  a?  [(        R*                  " USSSU R$                  R&                  UR                  S   -
  45      nU$ )ah  
Args:
    source_hidden_states (`torch.Tensor` of shape `(batch_size, source_sequence_length, source_dim)`):
        Qwen3 text encoder hidden states to condition on.
    target_input_ids (`torch.Tensor` of shape `(batch_size, target_sequence_length)`):
        T5 token ids used as learned query tokens.
    target_attention_mask (`torch.Tensor`, *optional*):
        Attention mask for the target T5 token ids.
    source_attention_mask (`torch.Tensor`, *optional*):
        Attention mask for the source Qwen3 hidden states.

Returns:
    `torch.Tensor`: Text conditioning embeddings for the Cosmos transformer.
r&   r6   )r9   r   )rœ   r�   rS   rž   r   )r½   r¯   r.   r'   r±   r   r,   r   r9   r   r²   rµ   Úis_grad_enabledr¸   Ú_gradient_checkpointing_funcr·   r¶   ÚsqueezeÚconfigr¬   ÚFÚpad)r1   r¿   rÀ   rœ   r�   r   r4   Úsource_position_idsrS   rž   Úblocks              r   r?   ÚAnimaTextConditioner.forward  së  € ð* !%× <Ñ <Ð=RÓ SÐØ $× <Ñ <Ð=RÓ SÐàŸ
™
Ð#3Ó4×7Ñ7Ð>R×>XÑ>XÐ7ÐYˆØŸ™ ]Ó3ˆä—|’| M×$7Ñ$7¸Ñ$:À=×CWÑCWÑX×bÑbÐcdÓeˆÜ#ŸlšlÐ+?×+EÑ+EÀaÑ+HÐQ^×QeÑQeÑf×pÑpÐqrÓsÐØ"Ÿo™o¨mÓJÐØ%)§_¡_°]Ó%XÐ"à—[”[ˆEÜ×$Ò$×&Ñ&¨4×+F×+FØ $× AÑ AØØ!Ø(Ø)Ø)Ø'Ø.ó!’ñ !&Ø!Ø(Ø*?Ø*?Ø(;Ø/Iñ!’ñ !ð* Ÿ	™	 $§-¡-°Ó">Ó?ˆà Ñ,Ø)×,IÑ,IÈ!Ó,L×,TÑ,TÐUVÓ,W×,ZÑ,ZÐ[hÓ,i×,sÑ,sÐtvÓ,wÑwˆMà×Ñ˜qÑ! D§K¡K×$CÑ$CÓCÜŸEšE -°!°Q¸¸4¿;¹;×;ZÑ;ZÐ]j×]pÑ]pÐqrÑ]sÑ;sÐ1tÓuˆMàÐr   )rµ   r¯   r¸   r±   r·   r¶   r²   )
é   rË   rË   é   r£   r¤   i€}  TFi   )NN)rB   rC   rD   rE   Ú__doc__Ú _supports_gradient_checkpointingÚ_no_split_modulesr   rF   r7   r¥   r+   Ústaticmethodr   rG   r½   r?   rI   rJ   rK   s   @r   r§   r§   å   sC  ø† ñð (,Ð$Ø4Ð5Ðàð ØØØØ#%ØØ!&Ø#'Ø$Ø#&ñ ,àð ,ð ð ,ð ð	 ,ð
 ð ,ð !ð ,ð ð ,ð ð ,ð !ð ,ð ð ,ð !÷ ,ó ð ,ðD ð°·±¸tÑ0Cð ÈÏÉÐW[ÑH[ó ó ðð 6:Ø59ñ=à#Ÿl™lð=ð  Ÿ,™,ð=ð  %Ÿ|™|¨dÑ2ð	=ð
  %Ÿ|™|¨dÑ2ð=ð 
�‰÷=ó =r   r§   )r6   )r   Útorch.nnr{   Útorch.nn.functionalÚ
functionalrÆ   Úconfiguration_utilsr   r   Úloadersr   Ú	attentionr   Úattention_dispatchr	   Úmodeling_utilsr
   rG   r   rF   r   ÚModuler!   rM   rP   rˆ   r§   rA   r   r   Ú<module>rÚ      sá   ðó Ý ß Ð ç BÝ 'Ý ,Ý 6Ý 'ðB §¡ð B°·±ô Bð ]^ñGØ—<‘<ðGØ&+§l¡lðGØ9>¿¹ðGØVYðGà
‡\�\õGôT˜2Ÿ9™9ô T÷*0ñ 0ôf,
 B§I¡IÐ/Cô ,
ô^C §	¡	ô CôLu˜: {Ð4Dõ ur   