ó
    oM‡jÆ_  ã                   ó¨  • S SK r S SKrS SKJr  S SKJs  Jr  SSKJrJ	r	  SSK
Jr  SSKJrJr   " S S\R                  5      r " S	 S
\R"                  5      r " S S\R                  5      r " S S\R                  5      r " S S\R                  5      r " S S\R                  5      r " S S\R                  5      r " S S\R                  5      r " S S\R                  5      r " S S\R                  5      r " S S\R                  5      r " S S\R                  5      r\	" SS 5      (       a  \R"                  rO " S! S"\R                  5      r " S# S$\R                  5      r " S% S&\R                  5      r " S' S(\R                  5      r " S) S*\R                  5      r      S2S+\!S,\"S-  S-\#S.\$S/\$S0\R                  4S1 jjr%g)3é    Né   )Úis_torch_npu_availableÚis_torch_versioné   )Úget_activation)ÚCombinedTimestepLabelEmbeddingsÚ)PixArtAlphaCombinedTimestepSizeEmbeddingsc                   óÞ   ^ • \ rS rSrSr     SS\S\S-  S\S-  S\S\S	\4U 4S
 jjjr SS\	R                  S\	R                  S-  S\	R                  S-  S\	R                  4S jjrSrU =r$ )ÚAdaLayerNormé   a…  
Norm layer modified to incorporate timestep embeddings.

Parameters:
    embedding_dim (`int`): The size of each embedding vector.
    num_embeddings (`int`, *optional*): The size of the embeddings dictionary.
    output_dim (`int`, *optional*):
    norm_elementwise_affine (`bool`, defaults to `False):
    norm_eps (`bool`, defaults to `False`):
    chunk_dim (`int`, defaults to `0`):
NÚembedding_dimÚnum_embeddingsÚ
output_dimÚnorm_elementwise_affineÚnorm_epsÚ	chunk_dimc                 ó>  >• [         TU ]  5         X`l        U=(       d    US-  nUb  [        R                  " X!5      U l        OS U l        [        R                  " 5       U l        [        R                  " X5      U l	        [        R                  " US-  XT5      U l        g ©Nr   )ÚsuperÚ__init__r   ÚnnÚ	EmbeddingÚembÚSiLUÚsiluÚLinearÚlinearÚ	LayerNormÚnorm)Úselfr   r   r   r   r   r   Ú	__class__s          €Ú¬C:\Users\johnn\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.13_qbz5n2kfra8p0\LocalCache\local-packages\Python313\site-packages\diffusers/models/normalization.pyr   ÚAdaLayerNorm.__init__(   st   ø€ ô 	‰ÑÔà"ŒØ×4 =°1Ñ#4ˆ
àÑ%Ü—|’| NÓBˆD�HàˆDŒHä—G’G“IˆŒ	Ü—i’i Ó:ˆŒÜ—L’L ¨q¡°(ÓTˆ�	ó    ÚxÚtimestepÚtembÚreturnc                 óP  • U R                   b  U R                  U5      nU R                  U R                  U5      5      nU R                  S:X  a+  UR	                  SSS9u  pEUS S 2S S S 24   nUS S 2S S S 24   nOUR	                  SSS9u  pTU R                  U5      SU-   -  U-   nU$ )Nr   r   ©Údimr   )r   r   r   r   Úchunkr   )r    r%   r&   r'   ÚshiftÚscales         r"   ÚforwardÚAdaLayerNorm.forward?   s¨   € ð �8‰8ÑØ—8‘8˜HÓ%ˆDà�{‰{˜4Ÿ9™9 T›?Ó+ˆà�>‰>˜QÓð  Ÿ:™: a¨Q˜:Ð/‰LˆEØš!˜T¢1˜*Ñ%ˆEØš!˜T¢1˜*Ñ%‰EàŸ:™: a¨Q˜:Ð/‰LˆEà�I‰I�a‹L˜A ™IÑ&¨Ñ.ˆØˆr$   )r   r   r   r   r   )NNFçñhãˆµøä>r   )NN)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__ÚintÚboolÚfloatr   ÚtorchÚTensorr/   Ú__static_attributes__Ú__classcell__©r!   s   @r"   r   r      s¹   ø† ñ
ð &*Ø!%Ø(-ØØñUàðUð ˜d™
ðUð ˜$‘Jð	Uð
 "&ðUð ðUð ÷Uð Uð0 bfñØ—‘ðØ).¯©¸Ñ)<ðØKPÏ<É<ÐZ^ÑK^ðà	�‰÷ó r$   r   c                   óN   • \ rS rSrS\R
                  S\R
                  4S jrSrg)ÚFP32LayerNorméT   Úinputsr(   c                 óL  • UR                   n[        R                  " UR                  5       U R                  U R
                  b  U R
                  R                  5       OS U R                  b  U R                  R                  5       OS U R                  5      R                  U5      $ ©N)	ÚdtypeÚFÚ
layer_normr9   Únormalized_shapeÚweightÚbiasÚepsÚto)r    rB   Úorigin_dtypes      r"   r/   ÚFP32LayerNorm.forwardU   su   € Ø—|‘|ˆÜ�|Š|Ø�L‰L‹NØ×!Ñ!Ø#'§;¡;Ñ#:ˆD�K‰K×ÑÔÀØ!%§¡Ñ!6ˆD�I‰I�O‰OÔ¸DØ�H‰Hó
÷ ‰"ˆ\Ó
ð	r$   © N)r2   r3   r4   r5   r:   r;   r/   r<   rO   r$   r"   r@   r@   T   s   † ð˜eŸl™lð ¨u¯|©|÷ r$   r@   c            	       ó¬   ^ • \ rS rSrSrSS\S\S\SS4U 4S jjjr SS	\	R                  S
\	R                  S-  S\\	R                  S4   4S jjrSrU =r$ )ÚSD35AdaLayerNormZeroXé`   zÁ
Norm layer adaptive layer norm zero (AdaLN-Zero).

Parameters:
    embedding_dim (`int`): The size of each embedding vector.
    num_embeddings (`int`): The size of the embeddings dictionary.
r   Ú	norm_typerJ   r(   Nc                 óò   >• [         TU ]  5         [        R                  " 5       U l        [        R
                  " USU-  US9U l        US:X  a  [        R                  " USSS9U l        g [        SU S35      e)	Né	   ©rJ   rG   Fç�íµ ÷Æ°>©Úelementwise_affinerK   úUnsupported `norm_type` (z-) provided. Supported ones are: 'layer_norm'.©
r   r   r   r   r   r   r   r   r   Ú
ValueError©r    r   rS   rJ   r!   s       €r"   r   ÚSD35AdaLayerNormZeroX.__init__i   sg   ø€ Ü‰ÑÔä—G’G“IˆŒ	Ü—i’i ¨q°=Ñ/@ÀtÑLˆŒØ˜Ó$ÜŸš ]ÀuÐRVÑWˆD�IäÐ8¸¸ÐCpÐqÓrÐrr$   Úhidden_statesr   .c           	      ó   • U R                  U R                  U5      5      nUR                  SSS9u	  p4pVpxpšnU R                  U5      nUSUS S 2S 4   -   -  US S 2S 4   -   nUSU
S S 2S 4   -   -  U	S S 2S 4   -   nXXgX�U4$ )NrU   r   r*   ©r   r   r,   r   )r    r_   r   Ú	shift_msaÚ	scale_msaÚgate_msaÚ	shift_mlpÚ	scale_mlpÚgate_mlpÚ
shift_msa2Ú
scale_msa2Ú	gate_msa2Únorm_hidden_statesÚnorm_hidden_states2s                 r"   r/   ÚSD35AdaLayerNormZeroX.forwards   s·   € ð
 �k‰k˜$Ÿ)™) C›.Ó)ˆØlo×luÑluØ�1ð mvð m
Ñiˆ	˜h°9È
Ð`ið "ŸY™Y }Ó5ÐØ*¨a°)ºA¸t¸GÑ2DÑ.DÑEÈ	ÒRSÐUYÐRYÑHZÑZˆØ0°A¸
Â1ÀdÀ7Ñ8KÑ4KÑLÈzÒZ[Ð]aÐZaÑObÑbÐØ¨	¸hÐ]fÐfÐfr$   ©r   r   r   ©rG   TrD   )r2   r3   r4   r5   r6   r7   Ústrr8   r   r:   r;   Útupler/   r<   r=   r>   s   @r"   rQ   rQ   `   s�   ø† ññs cð s°cð sÐPTð sÐ`d÷ sð sð $(ñgà—|‘|ðgð �\‰\˜DÑ ðgð 
ˆu�|‰|˜SÐ Ñ	!÷	gó gr$   rQ   c                   ó`  ^ • \ rS rSrSrSS\S\S-  4U 4S jjjr    SS\R                  S\R                  S-  S	\R                  S-  S
\R                  S-  S\R                  S-  S\\R                  \R                  \R                  \R                  \R                  4   4S jjrSrU =r$ )ÚAdaLayerNormZeroé‚   úÁ
Norm layer adaptive layer norm zero (adaLN-Zero).

Parameters:
    embedding_dim (`int`): The size of each embedding vector.
    num_embeddings (`int`): The size of the embeddings dictionary.
Nr   r   c                 óV  >• [         TU ]  5         Ub  [        X!5      U l        OS U l        [        R
                  " 5       U l        [        R                  " USU-  US9U l        US:X  a  [        R                  " USSS9U l
        g US:X  a  [        USSS9U l
        g [        S	U S
35      e)Né   rV   rG   FrW   rX   Úfp32_layer_norm)rY   rJ   rZ   ú@) provided. Supported ones are: 'layer_norm', 'fp32_layer_norm'.)r   r   r   r   r   r   r   r   r   r   r   r@   r\   )r    r   r   rS   rJ   r!   s        €r"   r   ÚAdaLayerNormZero.__init__‹   sž   ø€ Ü‰ÑÔØÑ%Ü6°~ÓUˆD�HàˆDŒHä—G’G“IˆŒ	Ü—i’i ¨q°=Ñ/@ÀtÑLˆŒØ˜Ó$ÜŸš ]ÀuÐRVÑWˆD�IØÐ+Ó+Ü% mÈÐTYÑZˆD�IäØ+¨I¨;Ð6vÐwóð r$   r%   r&   Úclass_labelsÚhidden_dtyper   r(   c                 óþ   • U R                   b  U R                  X#US9nU R                  U R                  U5      5      nUR                  SSS9u  pgp‰p«U R	                  U5      SUS S 2S 4   -   -  US S 2S 4   -   nXXšU4$ )N)r|   rw   r   r*   )r   r   r   r,   r   )r    r%   r&   r{   r|   r   rb   rc   rd   re   rf   rg   s               r"   r/   ÚAdaLayerNormZero.forward�   sŒ   € ð �8‰8ÑØ—(‘(˜8À�(ÐMˆCØ�k‰k˜$Ÿ)™) C›.Ó)ˆØILÏÉÐSTÐZ[ÈÐI\ÑFˆ	˜h°9Ø�I‰I�a‹L˜A 	ª!¨T¨'Ñ 2Ñ2Ñ3°iÂÀ4ÀÑ6HÑHˆØ˜I°(Ð:Ð:r$   )r   r   r   r   )NrG   T)NNNN)r2   r3   r4   r5   r6   r7   r   r:   r;   Ú
LongTensorrE   rq   r/   r<   r=   r>   s   @r"   rs   rs   ‚   sÄ   ø† ññ cð ¸3À¹:÷ ð ð* )-Ø04Ø+/Ø#'ñ;à�<‰<ð;ð —,‘, Ñ%ð;ð ×&Ñ&¨Ñ-ð	;ð
 —k‘k DÑ(ð;ð �\‰\˜DÑ ð;ð 
ˆu�|‰|˜UŸ\™\¨5¯<©<¸¿¹ÀuÇ|Á|ÐSÑ	T÷;ó ;r$   rs   c                   óö   ^ • \ rS rSrSrSS\4U 4S jjjr SS\R                  S\R                  S-  S\	\R                  \R                  \R                  \R                  \R                  4   4S	 jjr
S
rU =r$ )ÚAdaLayerNormZeroSingleé­   ru   r   c                 óò   >• [         TU ]  5         [        R                  " 5       U l        [        R
                  " USU-  US9U l        US:X  a  [        R                  " USSS9U l        g [        SU S35      e)	Né   rV   rG   FrW   rX   rZ   ry   r[   r]   s       €r"   r   ÚAdaLayerNormZeroSingle.__init__¶   sk   ø€ Ü‰ÑÔä—G’G“IˆŒ	Ü—i’i ¨q°=Ñ/@ÀtÑLˆŒØ˜Ó$ÜŸš ]ÀuÐRVÑWˆD�IäØ+¨I¨;Ð6vÐwóð r$   Nr%   r   r(   c                 ó¾   • U R                  U R                  U5      5      nUR                  SSS9u  p4nU R                  U5      SUS S 2S 4   -   -  US S 2S 4   -   nX4$ )Nr„   r   r*   ra   )r    r%   r   rb   rc   rd   s         r"   r/   ÚAdaLayerNormZeroSingle.forwardÂ   sg   € ð
 �k‰k˜$Ÿ)™) C›.Ó)ˆØ),¯©°1¸!¨Ð)<Ñ&ˆ	˜hØ�I‰I�a‹L˜A 	ª!¨T¨'Ñ 2Ñ2Ñ3°iÂÀ4ÀÑ6HÑHˆØˆ{Ðr$   rn   ro   rD   ©r2   r3   r4   r5   r6   r7   r   r:   r;   rq   r/   r<   r=   r>   s   @r"   r�   r�   ­   su   ø† ññ
 c÷ 
ð 
ð $(ñà�<‰<ðð �\‰\˜DÑ ðð 
ˆu�|‰|˜UŸ\™\¨5¯<©<¸¿¹ÀuÇ|Á|ÐSÑ	T÷	ó r$   r�   c                   óä   ^ • \ rS rSrSrS\S\S\4U 4S jjr SS\	R                  S	\	R                  S-  S
\\	R                  \	R                  \	R                  \	R                  4   4S jjrSrU =r$ )ÚLuminaRMSNormZeroéÍ   zx
Norm layer adaptive RMS normalization zero.

Parameters:
    embedding_dim (`int`): The size of each embedding vector.
r   r   r   c                 óÂ   >• [         TU ]  5         [        R                  " 5       U l        [        R
                  " [        US5      SU-  SS9U l        [        XS9U l	        g )Ni   é   TrV   ©rK   )
r   r   r   r   r   r   Úminr   ÚRMSNormr   )r    r   r   r   r!   s       €r"   r   ÚLuminaRMSNormZero.__init__Õ   sN   ø€ Ü‰ÑÔÜ—G’G“IˆŒ	Ü—i’iÜ�˜tÓ$Ø�ÑØñ
ˆŒô
 ˜MÑ8ˆ�	r$   Nr%   r   r(   c                 ó¬   • U R                  U R                  U5      5      nUR                  SSS9u  p4pVU R                  U5      SUS S 2S 4   -   -  nXXV4$ )Nr�   r   r*   ra   )r    r%   r   rc   rd   rf   rg   s          r"   r/   ÚLuminaRMSNormZero.forwardß   s\   € ð
 �k‰k˜$Ÿ)™) C›.Ó)ˆØ36·9±9¸QÀA°9Ð3FÑ0ˆ	˜YØ�I‰I�a‹L˜A 	ª!¨T¨'Ñ 2Ñ2Ñ3ˆà˜IÐ/Ð/r$   rn   rD   )r2   r3   r4   r5   r6   r7   r9   r8   r   r:   r;   rq   r/   r<   r=   r>   s   @r"   rŠ   rŠ   Í   sy   ø† ñð9 cð 9°Uð 9ÐUY÷ 9ð $(ñ	0à�<‰<ð	0ð �\‰\˜DÑ ð	0ð 
ˆu�|‰|˜UŸ\™\¨5¯<©<¸¿¹ÐEÑ	F÷		0ó 	0r$   rŠ   c                   ó0  ^ • \ rS rSrSrSS\S\4U 4S jjjr   SS\R                  S\
\\R                  4   S-  S	\S-  S
\R                  S-  S\\R                  \R                  \R                  \R                  \R                  4   4
S jjrSrU =r$ )ÚAdaLayerNormSingleéë   a<  
Norm layer adaptive layer norm single (adaLN-single).

As proposed in PixArt-Alpha (see: https://huggingface.co/papers/2310.00426; Section 2.3).

Parameters:
    embedding_dim (`int`): The size of each embedding vector.
    use_additional_conditions (`bool`): To use additional conditions for normalization or not.
r   Úuse_additional_conditionsc                 ó¶   >• [         TU ]  5         [        XS-  US9U l        [        R
                  " 5       U l        [        R                  " USU-  SS9U l        g )Nr„   )Úsize_emb_dimr—   rw   TrV   )	r   r   r	   r   r   r   r   r   r   )r    r   r—   r!   s      €r"   r   ÚAdaLayerNormSingle.__init__ö   sM   ø€ Ü‰ÑÔä<Ø¸Ñ(:ÐVoñ
ˆŒô —G’G“IˆŒ	Ü—i’i ¨q°=Ñ/@ÀtÑLˆ�r$   Nr&   Úadded_cond_kwargsÚ
batch_sizer|   r(   c                 ó�   • U=(       d    S S S.nU R                   " U40 UDX4S.D6nU R                  U R                  U5      5      U4$ )N)Ú
resolutionÚaspect_ratio)rœ   r|   ©r   r   r   )r    r&   r›   rœ   r|   Úembedded_timesteps         r"   r/   ÚAdaLayerNormSingle.forward   sN   € ð .×[ÀÐVZÑ1[ÐØ ŸHšH XÑuÐ1BÐuÈzÓuÐØ�{‰{˜4Ÿ9™9Ð%6Ó7Ó8Ð:KÐKÐKr$   r    )F)NNN)r2   r3   r4   r5   r6   r7   r8   r   r:   r;   Údictrp   rE   rq   r/   r<   r=   r>   s   @r"   r•   r•   ë   s¹   ø† ññM cð MÀd÷ Mð Mð =AØ!%Ø+/ñ
Là—,‘,ð
Lð    U§\¡\Ð 1Ñ2°TÑ9ð
Lð ˜$‘Jð	
Lð
 —k‘k DÑ(ð
Lð 
ˆu�|‰|˜UŸ\™\¨5¯<©<¸¿¹ÀuÇ|Á|ÐSÑ	T÷
Ló 
Lr$   r•   c                   ó¢   ^ • \ rS rSrSr SS\S\S\S\S-  S\4
U 4S	 jjjrS
\	R                  S\	R                  S\	R                  4S jrSrU =r$ )ÚAdaGroupNormi  aÑ  
GroupNorm layer modified to incorporate timestep embeddings.

Parameters:
    embedding_dim (`int`): The size of each embedding vector.
    num_embeddings (`int`): The size of the embeddings dictionary.
    num_groups (`int`): The number of groups to separate the channels into.
    act_fn (`str`, *optional*, defaults to `None`): The activation function to use.
    eps (`float`, *optional*, defaults to `1e-5`): The epsilon value to use for numerical stability.
Nr   Úout_dimÚ
num_groupsÚact_fnrK   c                 ó¬   >• [         TU ]  5         X0l        XPl        Uc  S U l        O[        U5      U l        [        R                  " XS-  5      U l        g r   )	r   r   r§   rK   Úactr   r   r   r   )r    r   r¦   r§   r¨   rK   r!   s         €r"   r   ÚAdaGroupNorm.__init__  sF   ø€ ô 	‰ÑÔØ$ŒØŒà‰>ØˆD�Hä% fÓ-ˆDŒHä—i’i ¸©{Ó;ˆ�r$   r%   r   r(   c                 ó  • U R                   (       a  U R                  U5      nU R                  U5      nUS S 2S S 2S S 4   nUR                  SSS9u  p4[        R                  " XR
                  U R                  S9nUSU-   -  U-   nU$ )Nr   r   r*   rŽ   )rª   r   r,   rF   Ú
group_normr§   rK   )r    r%   r   r.   r-   s        r"   r/   ÚAdaGroupNorm.forward'  sz   € Ø�8�8Ø—(‘(˜3“-ˆCØ�k‰k˜#ÓˆØ’!’Q˜˜dÐ"Ñ#ˆØ—y‘y ¨�yÐ*‰ˆä�LŠL˜ŸO™O°·±Ñ:ˆØ��U‘‰O˜eÑ#ˆØˆr$   )rª   rK   r   r§   )Nr1   )r2   r3   r4   r5   r6   r7   rp   r9   r   r:   r;   r/   r<   r=   r>   s   @r"   r¥   r¥     sp   ø† ñ	ð jnñ<Ø ð<Ø+.ð<Ø<?ð<ØILÈtÉð<Øaf÷<ð <ð	˜Ÿ™ð 	¨E¯L©Lð 	¸U¿\¹\÷ 	ò 	r$   r¥   c                   ó–   ^ • \ rS rSrSr    SS\S\4U 4S jjjrS\R                  S\R                  S\R                  4S	 jr	S
r
U =r$ )ÚAdaLayerNormContinuousi3  aq  
Adaptive normalization layer with a norm layer (layer_norm or rms_norm).

Args:
    embedding_dim (`int`): Embedding dimension to use during projection.
    conditioning_embedding_dim (`int`): Dimension of the input condition.
    elementwise_affine (`bool`, defaults to `True`):
        Boolean flag to denote if affine transformation should be applied.
    eps (`float`, defaults to 1e-5): Epsilon factor.
    bias (`bias`, defaults to `True`): Boolean flag to denote if bias should be use.
    norm_type (`str`, defaults to `"layer_norm"`):
        Normalization layer to use. Values supported: "layer_norm", "rms_norm".
r   Úconditioning_embedding_dimc                 ó
  >• [         TU ]  5         [        R                  " 5       U l        [        R
                  " X!S-  US9U l        US:X  a  [        XX55      U l        g US:X  a  [        XU5      U l        g [        SU 35      e)Nr   rV   rG   Úrms_normúunknown norm_type )r   r   r   r   r   r   r   r   r   r�   r\   )r    r   r±   rY   rK   rJ   rS   r!   s          €r"   r   ÚAdaLayerNormContinuous.__init__B  sw   ø€ ô 	‰ÑÔÜ—G’G“IˆŒ	Ü—i’iÐ :ÈAÑ<MÐTXÑYˆŒØ˜Ó$Ü! -Ð6HÓOˆD�IØ˜*Ó$Ü Ð4FÓGˆD�IäÐ1°)°Ð=Ó>Ð>r$   r%   Úconditioning_embeddingr(   c                 ó  • U R                  U R                  U5      R                  UR                  5      5      n[        R
                  " USSS9u  pEU R                  U5      SU-   S S 2S S S 24   -  US S 2S S S 24   -   nU$ )Nr   r   r*   )r   r   rL   rE   r:   r,   r   )r    r%   r¶   r   r.   r-   s         r"   r/   ÚAdaLayerNormContinuous.forwardZ  ss   € à�k‰k˜$Ÿ)™)Ð$:Ó;×>Ñ>¸q¿w¹wÓGÓHˆÜ—{’{ 3¨¨qÑ1‰ˆØ�I‰I�a‹L˜A ™I¢q¨$² zÑ2Ñ2°Uº1¸dÂA¸:Ñ5FÑFˆØˆr$   rn   )Tr1   TrG   )r2   r3   r4   r5   r6   r7   r   r:   r;   r/   r<   r=   r>   s   @r"   r°   r°   3  s`   ø† ñð.  ØØØñ?àð?ð %(÷?ð ?ð0˜Ÿ™ð ¸u¿|¹|ð ÐPU×P\ÑP\÷ ò r$   r°   c                   óž   ^ • \ rS rSr     SS\S\S\S-  4U 4S jjjrS\R                  S\R                  S	\R                  4S
 jrSr	U =r
$ )ÚLuminaLayerNormContinuousib  Nr   r±   r¦   c                 óL  >• [         TU ]  5         [        R                  " 5       U l        [        R
                  " X!US9U l        US:X  a  [        XX55      U l        O$US:X  a  [        XUS9U l        O[        SU 35      eS U l        Ub  [        R
                  " XUS9U l        g g )NrV   rG   r³   ©rK   rY   r´   )r   r   r   r   r   r   Úlinear_1r   r   r�   r\   Úlinear_2)	r    r   r±   rY   rK   rJ   rS   r¦   r!   s	           €r"   r   Ú"LuminaLayerNormContinuous.__init__c  s˜   ø€ ô 	‰ÑÔô —G’G“IˆŒ	ÜŸ	š	Ð"<ÐRVÑWˆŒà˜Ó$Ü! -Ð6HÓOˆD�IØ˜*Ó$Ü ÐK]Ñ^ˆD�IäÐ1°)°Ð=Ó>Ð>àˆŒØÑÜŸIšI mÀ4ÑHˆD�Mð r$   r%   r¶   r(   c                 óú   • U R                  U R                  U5      R                  UR                  5      5      nUnU R	                  U5      SU-   S S 2S S S 24   -  nU R
                  b  U R                  U5      nU$ ©Nr   )r½   r   rL   rE   r   r¾   )r    r%   r¶   r   r.   s        r"   r/   Ú!LuminaLayerNormContinuous.forwardƒ  sn   € ð �m‰m˜DŸI™IÐ&<Ó=×@Ñ@ÀÇÁÓIÓJˆØˆØ�I‰I�a‹L˜A ™I¢q¨$² zÑ2Ñ2ˆà�=‰=Ñ$Ø—‘˜aÓ ˆAàˆr$   )r½   r¾   r   r   )Tr1   TrG   N)r2   r3   r4   r5   r7   r   r:   r;   r/   r<   r=   r>   s   @r"   rº   rº   b  sv   ø† ð  ØØØØ"ñIàðIð %(ðIð �t‘÷Ið Ið@à�<‰<ðð !&§¡ðð 
�‰÷	ò r$   rº   c                   ó  ^ • \ rS rSrSrS\S\4U 4S jjr SS\R                  S\R                  S	\R                  S-  S
\	\R                  \R                  \R                  \R                  \R                  4   4S jjr
SrU =r$ )Ú%CogView3PlusAdaLayerNormZeroTextImagei“  ru   r   r+   c                 óþ   >• [         TU ]  5         [        R                  " 5       U l        [        R
                  " USU-  SS9U l        [        R                  " USSS9U l        [        R                  " USSS9U l	        g )Né   TrV   Fr1   rX   )
r   r   r   r   r   r   r   r   Únorm_xÚnorm_c)r    r   r+   r!   s      €r"   r   Ú.CogView3PlusAdaLayerNormZeroTextImage.__init__œ  s[   ø€ Ü‰ÑÔä—G’G“IˆŒ	Ü—i’i ¨r°C©x¸dÑCˆŒÜ—l’l 3¸5ÀdÑKˆŒÜ—l’l 3¸5ÀdÑKˆ�r$   Nr%   Úcontextr   r(   c                 ó2  • U R                  U R                  U5      5      nUR                  SSS9u  nnnnnn	n
nnnnnU R                  U5      nU R	                  U5      nUSUS S 2S 4   -   -  US S 2S 4   -   nUSUS S 2S 4   -   -  U
S S 2S 4   -   nXXxX’XÍXï4
$ )NrÆ   r   r*   )r   r   r,   rÇ   rÈ   )r    r%   rÊ   r   rb   rc   rd   re   rf   rg   Úc_shift_msaÚc_scale_msaÚ
c_gate_msaÚc_shift_mlpÚc_scale_mlpÚ
c_gate_mlpÚnormed_xÚnormed_contexts                     r"   r/   Ú-CogView3PlusAdaLayerNormZeroTextImage.forward¤  sÎ   € ð �k‰k˜$Ÿ)™) C›.Ó)ˆð �I‰I�b˜aˆIÐ ñ	
ØØØØØØØØØØØØà—;‘;˜q“>ˆØŸ™ WÓ-ˆØ˜˜I¢a¨ gÑ.Ñ.Ñ/°)ºA¸t¸GÑ2DÑDˆØ  A¨²A°t°GÑ(<Ñ$<Ñ=ÀÊAÈtÈGÑ@TÑTˆØ˜I°(ÀZÐ^iÐuÐur$   )r   rÈ   rÇ   r   rD   rˆ   r>   s   @r"   rÄ   rÄ   “  sŽ   ø† ñðL cð L°÷ Lð $(ñ	và�<‰<ðvð —‘ðvð �\‰\˜DÑ ð	vð
 
ˆu�|‰|˜UŸ\™\¨5¯<©<¸¿¹ÀuÇ|Á|ÐSÑ	T÷vó vr$   rÄ   c                   óÖ   ^ • \ rS rSr   SS\S\S\S\S\SS4U 4S	 jjjrS
\R                  S\R                  S\R                  S\
\R                  \R                  4   4S jrSrU =r$ )ÚCogVideoXLayerNormZeroiÀ  Úconditioning_dimr   rY   rK   rJ   r(   Nc                 óÆ   >• [         TU ]  5         [        R                  " 5       U l        [        R
                  " USU-  US9U l        [        R                  " X$US9U l        g )Nrw   rV   r¼   )	r   r   r   r   r   r   r   r   r   )r    r×   r   rY   rK   rJ   r!   s         €r"   r   ÚCogVideoXLayerNormZero.__init__Á  sJ   ø€ ô 	‰ÑÔä—G’G“IˆŒ	Ü—i’iÐ 0°!°mÑ2CÈ$ÑOˆŒÜ—L’L ÐL^Ñ_ˆ�	r$   r_   Úencoder_hidden_statesr'   c                 óP  • U R                  U R                  U5      5      R                  SSS9u  pEpgp‰U R                  U5      SU-   S S 2S S S 24   -  US S 2S S S 24   -   nU R                  U5      SU-   S S 2S S S 24   -  US S 2S S S 24   -   nXUS S 2S S S 24   U	S S 2S S S 24   4$ )Nrw   r   r*   ra   )
r    r_   rÚ   r'   r-   r.   ÚgateÚ	enc_shiftÚ	enc_scaleÚenc_gates
             r"   r/   ÚCogVideoXLayerNormZero.forwardÏ  sÏ   € ð >B¿[¹[ÈÏÉÐSWËÓ=Y×=_Ñ=_Ð`aÐghÐ=_Ð=iÑ:ˆ�d yØŸ	™	 -Ó0°A¸±IºqÀ$Ê¸zÑ3JÑJÈUÒSTÐVZÒ\]ÐS]ÑM^Ñ^ˆØ $§	¡	Ð*?Ó @ÀAÈ	ÁMÒSTÐVZÒ\]ÐS]ÑC^Ñ ^ÐajÒklÐnrÒtuÐkuÑavÑ vÐØ°Tº!¸TÂ1¸*Ñ5EÀxÒPQÐSWÒYZÐPZÑG[Ð[Ð[r$   rn   )Tr1   T)r2   r3   r4   r5   r7   r8   r9   r   r:   r;   rq   r/   r<   r=   r>   s   @r"   rÖ   rÖ   À  sª   ø† ð
 $(ØØñ`àð`ð ð`ð !ð	`ð
 ð`ð ð`ð 
÷`ð `ð\Ø"Ÿ\™\ð\ØBGÇ,Á,ð\ØV[×VbÑVbð\à	ˆu�|‰|˜UŸ\™\Ð)Ñ	*÷\ò \r$   rÖ   z>=z2.1.0c                   óF   ^ • \ rS rSrSrS	S\S\S\4U 4S jjjrS rSr	U =r
$ )
r   iÝ  ap  
LayerNorm with the bias parameter.

Args:
    dim (`int`): Dimensionality to use for the parameters.
    eps (`float`, defaults to 1e-5): Epsilon factor.
    elementwise_affine (`bool`, defaults to `True`):
        Boolean flag to denote if affine transformation should be applied.
    bias (`bias`, defaults to `True`): Boolean flag to denote if bias should be use.
rK   rY   rJ   c                 ó¢  >• [         TU ]  5         X l        [        U[        R
                  5      (       a  U4n[        R                  " U5      U l        U(       ah  [        R                  " [        R                  " U5      5      U l        U(       a*  [        R                  " [        R                  " U5      5      OS U l        g S U l        S U l        g rD   )r   r   rK   Ú
isinstanceÚnumbersÚIntegralr:   ÚSizer+   r   Ú	ParameterÚonesrI   ÚzerosrJ   ©r    r+   rK   rY   rJ   r!   s        €r"   r   ÚLayerNorm.__init__é  s‚   ø€ Ü‰GÑÔàŒHä˜#œw×/Ñ/×0Ñ0Ø�f�ä—z’z #“ˆDŒHæ!Ü Ÿlšl¬5¯:ª:°c«?Ó;�”Þ>BœBŸLšL¬¯ª°SÓ)9Ô:È�•	à"�”Ø �•	r$   c                 ó„   • [         R                  " XR                  U R                  U R                  U R
                  5      $ rD   )rF   rG   r+   rI   rJ   rK   )r    Úinputs     r"   r/   ÚLayerNorm.forwardú  s'   € Ü—<’< §x¡x°·±¸d¿i¹iÈÏÉÓRÐRr$   )rJ   r+   rK   rI   )r1   TT©r2   r3   r4   r5   r6   r9   r8   r   r/   r<   r=   r>   s   @r"   r   r   Ý  s4   ø† ñ		ñ	! Uð 	!Àtð 	!ÐZ^÷ 	!ð 	!÷"	Sð 	Sr$   r   c                   óF   ^ • \ rS rSrSrS	S\S\S\4U 4S jjjrS rSr	U =r
$ )
r�   iþ  aì  
RMS Norm as introduced in https://huggingface.co/papers/1910.07467 by Zhang et al.

Args:
    dim (`int`): Number of dimensions to use for `weights`. Only effective when `elementwise_affine` is True.
    eps (`float`): Small value to use when calculating the reciprocal of the square-root.
    elementwise_affine (`bool`, defaults to `True`):
        Boolean flag to denote if affine transformation should be applied.
    bias (`bool`, defaults to False): If also training the `bias` param.
rK   rY   rJ   c                 ó¬  >• [         TU ]  5         X l        X0l        [	        U[
        R                  5      (       a  U4n[        R                  " U5      U l	        S U l
        S U l        U(       ag  [        R                  " [        R                  " U5      5      U l
        U(       a0  [        R                  " [        R                  " U5      5      U l        g g g rD   )r   r   rK   rY   rã   rä   rå   r:   ræ   r+   rI   rJ   r   rç   rè   ré   rê   s        €r"   r   ÚRMSNorm.__init__
  s‘   ø€ Ü‰ÑÔàŒØ"4Ôä�cœ7×+Ñ+×,Ñ,Ø�&ˆCä—:’:˜c“?ˆŒàˆŒØˆŒ	æÜŸ,š,¤u§z¢z°#£Ó7ˆDŒKÞÜŸLšL¬¯ª°SÓ)9Ó:�•	ð ð r$   c                 ó®  • [        5       (       a²  SS KnU R                  b]  U R                  R                  [        R
                  [        R                  4;   a%  UR                  U R                  R                  5      nUR                  XR                  U R                  S9S   nU R                  b  XR                  -   nU$ UR                  nUR                  [        R                  5      R                  S5      R                  SSS9nU[        R                  " X@R                  -   5      -  nU R                  bˆ  U R                  R                  [        R
                  [        R                  4;   a%  UR                  U R                  R                  5      nXR                  -  nU R                  b  XR                  -   nU$ UR                  U5      nU$ )Nr   )Úepsilonr   éÿÿÿÿT©Úkeepdim)r   Ú	torch_npurI   rE   r:   Úfloat16Úbfloat16rL   Únpu_rms_normrK   rJ   Úfloat32ÚpowÚmeanÚrsqrt)r    r_   rø   Úinput_dtypeÚvariances        r"   r/   ÚRMSNorm.forward  sp  € Ü!×#Ñ#Ûà�{‰{Ñ&à—;‘;×$Ñ$¬¯©¼¿¹Ð(GÓGØ$1×$4Ñ$4°T·[±[×5FÑ5FÓ$G�MØ%×2Ñ2°=Ç+Á+ÐW[×W_ÑW_Ð2Ð`ÐabÑcˆMØ�y‰yÑ$Ø -·	±	Ñ 9�ð  Ðð (×-Ñ-ˆKØ$×'Ñ'¬¯©Ó6×:Ñ:¸1Ó=×BÑBÀ2ÈtÐBÐTˆHØ)¬E¯KªK¸Ç8Á8Ñ8KÓ,LÑLˆMà�{‰{Ñ&à—;‘;×$Ñ$¬¯©¼¿¹Ð(GÓGØ$1×$4Ñ$4°T·[±[×5FÑ5FÓ$G�MØ -·±Ñ ;�Ø—9‘9Ñ(Ø$1·I±IÑ$=�Mð Ðð !.× 0Ñ 0°Ó =�àÐr$   )rJ   r+   rY   rK   rI   )TFrï   r>   s   @r"   r�   r�   þ  s1   ø† ñ	ñ; ð ;¸Dð ;Èt÷ ;ð ;÷&ð r$   r�   c                   ó>   ^ • \ rS rSrSS\S\4U 4S jjjrS rSrU =r	$ )ÚMochiRMSNormi<  rK   rY   c                 ó$  >• [         TU ]  5         X l        [        U[        R
                  5      (       a  U4n[        R                  " U5      U l        U(       a0  [        R                  " [        R                  " U5      5      U l        g S U l        g rD   )r   r   rK   rã   rä   rå   r:   ræ   r+   r   rç   rè   rI   )r    r+   rK   rY   r!   s       €r"   r   ÚMochiRMSNorm.__init__=  s^   ø€ Ü‰ÑÔàŒä�cœ7×+Ñ+×,Ñ,Ø�&ˆCä—:’:˜c“?ˆŒæÜŸ,š,¤u§z¢z°#£Ó7ˆD�KàˆD�Kr$   c                 ó8  • UR                   nUR                  [        R                  5      R	                  S5      R                  SSS9nU[        R                  " X0R                  -   5      -  nU R                  b  XR                  -  nUR                  U5      nU$ )Nr   rõ   Trö   )	rE   rL   r:   rü   rý   rþ   rÿ   rK   rI   )r    r_   r   r  s       r"   r/   ÚMochiRMSNorm.forwardL  s‚   € Ø#×)Ñ)ˆØ ×#Ñ#¤E§M¡MÓ2×6Ñ6°qÓ9×>Ñ>¸rÈ4Ð>ÐPˆØ%¬¯ª°H¿x¹xÑ4GÓ(HÑHˆà�;‰;Ñ"Ø)¯K©KÑ7ˆMØ%×(Ñ(¨Ó5ˆàÐr$   )r+   rK   rI   )T)
r2   r3   r4   r5   r9   r8   r   r/   r<   r=   r>   s   @r"   r  r  <  s%   ø† ñ ð ¸D÷ ð ÷	ð 	r$   r  c                   ó2   ^ • \ rS rSrSrU 4S jrS rSrU =r$ )ÚGlobalResponseNormiX  z¸
Global response normalization as introduced in ConvNeXt-v2 (https://huggingface.co/papers/2301.00808).

Args:
    dim (`int`): Number of dimensions to use for the `gamma` and `beta`.
c                 óê   >• [         TU ]  5         [        R                  " [        R
                  " SSSU5      5      U l        [        R                  " [        R
                  " SSSU5      5      U l        g rÁ   )r   r   r   rç   r:   ré   ÚgammaÚbeta)r    r+   r!   s     €r"   r   ÚGlobalResponseNorm.__init__a  sL   ø€ Ü‰ÑÔÜ—\’\¤%§+¢+¨a°°A°sÓ";Ó<ˆŒ
Ü—L’L¤§¢¨Q°°1°cÓ!:Ó;ˆ�	r$   c                 óœ   • [         R                  " USSSS9nX"R                  SSS9S-   -  nU R                  X-  -  U R                  -   U-   $ )Nr   )r   r   T)Úpr+   r÷   rõ   )r+   r÷   rW   )r:   r   rþ   r  r  )r    r%   ÚgxÚnxs       r"   r/   ÚGlobalResponseNorm.forwardf  sO   € Ü�ZŠZ˜˜Q F°DÑ9ˆØ—7‘7˜r¨4�7Ð0°4Ñ7Ñ8ˆØ�z‰z˜Q™VÑ$ t§y¡yÑ0°1Ñ4Ð4r$   )r  r  )	r2   r3   r4   r5   r6   r   r/   r<   r=   r>   s   @r"   r
  r
  X  s   ø† ñõ<÷
5ð 5r$   r
  c                   óv   ^ • \ rS rSrS
S\S\S\4U 4S jjjrS\R                  S\R                  4S jr	S	r
U =r$ )ÚLpNormil  r  r+   rK   c                 óF   >• [         TU ]  5         Xl        X l        X0l        g rD   )r   r   r  r+   rK   )r    r  r+   rK   r!   s       €r"   r   ÚLpNorm.__init__m  s   ø€ Ü‰ÑÔàŒØŒØ�r$   r_   r(   c                 ój   • [         R                  " XR                  U R                  U R                  S9$ )N)r  r+   rK   )rF   Ú	normalizer  r+   rK   )r    r_   s     r"   r/   ÚLpNorm.forwardt  s!   € Ü�{Š{˜=¯F©F¸¿¹ÀdÇhÁhÑOÐOr$   )r+   rK   r  )r   rõ   gê-�™—q=)r2   r3   r4   r5   r7   r9   r   r:   r;   r/   r<   r=   r>   s   @r"   r  r  l  sF   ø† ñ˜#ð ¨ð °u÷ ð ðP U§\¡\ð P°e·l±l÷ Pò Pr$   r  rS   Únum_featuresrK   rY   rJ   r(   c                 óº   • U S:X  a  [        XX4S9nU$ U S:X  a  [        R                  " XX4S9nU$ U S:X  a  [        R                  " XUS9nU$ [	        SU < S35      e)Nr³   )rK   rY   rJ   rG   Ú
batch_norm)rK   Úaffinez
norm_type=z is not supported.)r�   r   r   ÚBatchNorm2dr\   )rS   r  rK   rY   rJ   r   s         r"   Úget_normalizationr   x  s{   € ð �JÓÜ�|ÐASÑ_ˆð €Kð 
�lÓ	"Ü�|Š|˜LÐFXÑdˆð
 €Kð	 
�lÓ	"Ü�~Š~˜lÐ<NÑOˆð €Kô ˜J˜I™<Ð'9Ð:Ó;Ð;r$   )r  Nr1   TT)&rä   r:   Útorch.nnr   Útorch.nn.functionalÚ
functionalrF   Úutilsr   r   Úactivationsr   Ú
embeddingsr   r	   ÚModuler   r   r@   rQ   rs   r�   rŠ   r•   r¥   r°   rº   rÄ   rÖ   r�   r  r
  r  rp   r7   r9   r8   r   rO   r$   r"   Ú<module>r(     s´  ðó  ã Ý ß Ð ç <Ý 'ß bô6�2—9‘9ô 6ôr	�B—L‘Lô 	ôg˜BŸI™Iô gôD(;�r—y‘yô (;ôV˜RŸY™Yô ô@0˜Ÿ	™	ô 0ô<L˜Ÿ™ô LôD#�2—9‘9ô #ôL,˜RŸY™Yô ,ô^. §	¡	ô .ôb*v¨B¯I©Iô *vôZ\˜RŸY™Yô \ñ0 �D˜'×"Ñ"Ø—‘�IôS�B—I‘Iô SôB9ˆb�i‰iô 9ô|�2—9‘9ô ô85˜Ÿ™ô 5ô(	PˆR�Y‰Yô 	Pð "Ø#ØØ#ØñØðà˜‘*ðð 
ðð ð	ð
 ðð ‡Y�Yör$   