ó
    oM‡j�.  ã            	       ó¼  • S SK r S SKJrJr  S SKJr  S SKrSSKJr  SSK	J
r
  SSKJrJrJrJr  SS	KJrJr  SS
KJrJr  \" \5      rSr\ " S S5      5       r " S S\R4                  R6                  5      r " S S\5      r " S S\5      r " S S\5      rS\R@                  RB                  S\SS4S jr"SS\R@                  RB                  S\S\#S-  SS4S jjr$g)é    N)ÚasdictÚ	dataclass)ÚCallableé   )Ú
get_logger)Úunwrap_moduleé   )Ú"_ALL_TRANSFORMER_BLOCK_IDENTIFIERSÚ_ATTENTION_CLASSESÚ_FEEDFORWARD_CLASSESÚ_get_submodule_from_fqn)ÚAttentionProcessorRegistryÚTransformerBlockRegistry)ÚHookRegistryÚ	ModelHookÚlayer_skip_hookc                   ó˜   • \ rS rSr% Sr\\   \S'   Sr\	\S'   Sr
\\S'   Sr\\S	'   Sr\\S
'   Sr\\S'   S rS r\S\SS 4S j5       rSrg)ÚLayerSkipConfigé(   ay  
Configuration for skipping internal transformer blocks when executing a transformer model.

Args:
    indices (`list[int]`):
        The indices of the layer to skip. This is typically the first layer in the transformer block.
    fqn (`str`, defaults to `"auto"`):
        The fully qualified name identifying the stack of transformer blocks. Typically, this is
        `transformer_blocks`, `single_transformer_blocks`, `blocks`, `layers`, or `temporal_transformer_blocks`.
        For automatic detection, set this to `"auto"`. "auto" only works on DiT models. For UNet models, you must
        provide the correct fqn.
    skip_attention (`bool`, defaults to `True`):
        Whether to skip attention blocks.
    skip_ff (`bool`, defaults to `True`):
        Whether to skip feed-forward blocks.
    skip_attention_scores (`bool`, defaults to `False`):
        Whether to skip attention score computation in the attention blocks. This is equivalent to using `value`
        projections as the output of scaled dot product attention.
    dropout (`float`, defaults to `1.0`):
        The dropout probability for dropping the outputs of the skipped layers. By default, this is set to `1.0`,
        meaning that the outputs of the skipped layers are completely ignored. If set to `0.0`, the outputs of the
        skipped layers are fully retained, which is equivalent to not skipping any layers.
ÚindicesÚautoÚfqnTÚskip_attentionFÚskip_attention_scoresÚskip_ffç      ð?Údropoutc                 óî   • SU R                   s=::  a  S::  d  O  [        SU R                    S35      e[        R                  " U R                   S5      (       d  U R                  (       a  [        S5      eg g )Nr   r	   z6Expected `dropout` to be between 0.0 and 1.0, but got Ú.r   úbCannot set `skip_attention_scores` to True when `dropout` is not 1.0. Please set `dropout` to 1.0.)r   Ú
ValueErrorÚmathÚiscloser   ©Úselfs    Ú¨C:\Users\johnn\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.13_qbz5n2kfra8p0\LocalCache\local-packages\Python313\site-packages\diffusers/hooks/layer_skip.pyÚ__post_init__ÚLayerSkipConfig.__post_init__I   sg   € Ø�T—\‘\Õ& QÕ&ÜÐUÐVZ×VbÑVbÐUcÐcdÐeÓfÐfÜ�|Š|˜DŸL™L¨#×.Ñ.°4×3M×3MÜØtóð ð 4NÐ.ó    c                 ó   • [        U 5      $ ©N)r   r$   s    r&   Úto_dictÚLayerSkipConfig.to_dictQ   s   € Ü�d‹|Ðr)   ÚdataÚreturnc                 ó   • [        S0 U D6$ )N© )r   )r.   s    r&   Ú	from_dictÚLayerSkipConfig.from_dictT   s   € äÑ& Ñ&Ð&r)   r1   N)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__ÚlistÚintÚ__annotations__r   Ústrr   Úboolr   r   r   Úfloatr'   r,   ÚstaticmethodÚdictr2   Ú__static_attributes__r1   r)   r&   r   r   (   ss   ‡ ñð0 �#‰YÓØ€CˆÓØ€N�DÓØ"'Ð˜4Ó'Ø€GˆTÓØ€GˆUÓòòð ð'˜ð 'Ð!2ó 'ó ó'r)   r   c                   ó   • \ rS rSrSS jrSrg)ÚAttentionScoreSkipFunctionModeéY   r1   Nc                 óV  • Uc  0 nU[         R                  R                  R                  L av  UR	                  SS 5      nUR	                  SS 5      nUR	                  SS 5      nUb  UOUS   nUb  UOUS   nUb  UOUS   nUR
                  S   UR
                  S   :X  a  U$ U" U0 UD6$ )NÚqueryÚkeyÚvaluer   r	   r   )ÚtorchÚnnÚ
functionalÚscaled_dot_product_attentionÚgetÚshape)r%   ÚfuncÚtypesÚargsÚkwargsrF   rG   rH   s           r&   Ú__torch_function__Ú1AttentionScoreSkipFunctionMode.__torch_function__Z   s²   € Ø‰>ØˆFØ”5—8‘8×&Ñ&×CÑCÒCØ—J‘J˜w¨Ó-ˆEØ—*‘*˜U DÓ)ˆCØ—J‘J˜w¨Ó-ˆEØ"Ñ.‘E°D¸±GˆEØ™‘#¨d°1©gˆCØ"Ñ.‘E°D¸±GˆEð �{‰{˜1‰~ §¡¨Q¡Ó/Ø�Ù�TÐ$˜VÑ$Ð$r)   )r1   N)r4   r5   r6   r7   rS   rA   r1   r)   r&   rC   rC   Y   s   † ÷%r)   rC   c                   ód   • \ rS rSrS
S\S\S\4S jjrS\R                  R                  4S jrSrg	)ÚAttentionProcessorSkipHookén   Úskip_processor_output_fnr   r   c                 ó(   • Xl         X l        X0l        g r+   )rX   r   r   )r%   rX   r   r   s       r&   Ú__init__Ú#AttentionProcessorSkipHook.__init__o   s   € Ø(@Ô%Ø%:Ô"Ø�r)   Úmodulec                 ó&  • U R                   (       ab  [        R                  " U R                  S5      (       d  [	        S5      e[        5          U R                  R                  " U0 UD6nS S S 5        U$ [        R                  " U R                  S5      (       a  U R                  " U/UQ70 UD6nU$ U R                  R                  " U0 UD6n[        R                  R                  R                  X@R                  S9nU$ ! , (       d  f       W$ = f)Nr   r    ©Úp)r   r"   r#   r   r!   rC   Úfn_refÚoriginal_forwardrX   rI   rJ   rK   ©r%   r\   rQ   rR   Úoutputs        r&   Únew_forwardÚ&AttentionProcessorSkipHook.new_forwardt   sæ   € Ø×%×%Ü—<’< §¡¨c×2Ñ2Ü Øxóð ô 0Õ1ØŸ™×5Ò5°tÐF¸vÑF�÷ 2ð ˆô �|Š|˜DŸL™L¨#×.Ñ.Ø×6Ò6°vÐOÀÒOÈÑO�ð ˆð Ÿ™×5Ò5°tÐF¸vÑF�ÜŸ™×,Ñ,×4Ñ4°V¿|¹|Ð4ÐL�Øˆ÷ 2Ô1ð ˆús   ÁDÄ
D)r   r   rX   N)Fr   )r4   r5   r6   r7   r   r=   r>   rZ   rI   rJ   ÚModulerd   rA   r1   r)   r&   rV   rV   n   s2   † ñ°ð ÐRVð Ðinõ ð
 %§(¡(§/¡/÷ r)   rV   c                   óf   ^ • \ rS rSrS\4U 4S jjrS\R                  R                  4S jr	Sr
U =r$ )ÚFeedForwardSkipHooké…   r   c                 ó.   >• [         TU ]  5         Xl        g r+   ©ÚsuperrZ   r   ©r%   r   Ú	__class__s     €r&   rZ   ÚFeedForwardSkipHook.__init__†   ó   ø€ Ü‰ÑÔØ�r)   r\   c                 ól  • [         R                  " U R                  S5      (       a@  UR                  SS 5      nUc  UR                  SS 5      nUc  [	        U5      S:”  a  US   nU$ U R
                  R                  " U0 UD6n[        R                  R                  R                  X@R                  S9nU$ )Nr   Úhidden_statesÚxr   r^   )
r"   r#   r   rM   Úlenr`   ra   rI   rJ   rK   rb   s        r&   rd   ÚFeedForwardSkipHook.new_forwardŠ   sœ   € Ü�<Š<˜Ÿ™ c×*Ñ*Ø—Z‘Z °Ó6ˆFØ‰~ØŸ™ C¨Ó.�Ø‰~¤# d£)¨a£-Ø˜a™�ð ˆð —[‘[×1Ò1°4ÐB¸6ÑBˆFÜ—X‘X×(Ñ(×0Ñ0°¿<¹<Ð0ÐHˆFØˆr)   )r   )r4   r5   r6   r7   r>   rZ   rI   rJ   rf   rd   rA   Ú__classcell__©rn   s   @r&   rh   rh   …   s(   ø† ð ÷ ð
 %§(¡(§/¡/÷ 
ò 
r)   rh   c                   ól   ^ • \ rS rSrS\4U 4S jjrS rS\R                  R                  4S jr
SrU =r$ )ÚTransformerBlockSkipHooké—   r   c                 ó.   >• [         TU ]  5         Xl        g r+   rk   rm   s     €r&   rZ   Ú!TransformerBlockSkipHook.__init__˜   rp   r)   c                 ób   • [         R                  " [        U5      R                  5      U l        U$ r+   )r   rM   r   rn   Ú	_metadata)r%   r\   s     r&   Úinitialize_hookÚ(TransformerBlockSkipHook.initialize_hookœ   s$   € Ü1×5Ò5´mÀFÓ6K×6UÑ6UÓVˆŒØˆr)   r\   c                 óž  • [         R                  " U R                  S5      (       aY  U R                  R	                  SX#5      nU R                  R
                  c  UnU$ U R                  R	                  SX#5      nXF4n U$ U R                  R                  " U0 UD6n[        R                  R                  R                  XPR                  S9nU$ )Nr   rr   Úencoder_hidden_statesr^   )r"   r#   r   r~   Ú_get_parameter_from_args_kwargsÚ"return_encoder_hidden_states_indexr`   ra   rI   rJ   rK   )r%   r\   rQ   rR   Úoriginal_hidden_statesrc   Úoriginal_encoder_hidden_statess          r&   rd   Ú$TransformerBlockSkipHook.new_forward    sº   € Ü�<Š<˜Ÿ™ c×*Ñ*Ø%)§^¡^×%SÑ%SÐTcÐeiÓ%rÐ"Ø�~‰~×@Ñ@ÑHØ/�ð ˆð 26·±×1_Ñ1_Ø+¨Tó2Ð.ð 1ÐQ‘ð ˆð —[‘[×1Ò1°4ÐB¸6ÑBˆFÜ—X‘X×(Ñ(×0Ñ0°¿<¹<Ð0ÐHˆFØˆr)   )r~   r   )r4   r5   r6   r7   r>   rZ   r   rI   rJ   rf   rd   rA   rv   rw   s   @r&   ry   ry   —   s-   ø† ð ÷ òð %§(¡(§/¡/÷ ò r)   ry   r\   Úconfigr/   c                 ó   • [        X5        g)ag  
Apply layer skipping to internal layers of a transformer.

Args:
    module (`torch.nn.Module`):
        The transformer model to which the layer skip hook should be applied.
    config (`LayerSkipConfig`):
        The configuration for the layer skip hook.

Example:

```python
>>> from diffusers import apply_layer_skip_hook, CogVideoXTransformer3DModel, LayerSkipConfig

>>> transformer = CogVideoXTransformer3DModel.from_pretrained("THUDM/CogVideoX-5b", torch_dtype=torch.bfloat16)
>>> config = LayerSkipConfig(layer_index=[10, 20], fqn="transformer_blocks")
>>> apply_layer_skip_hook(transformer, config)
```
N)Ú_apply_layer_skip_hook)r\   rˆ   s     r&   Úapply_layer_skipr‹   °   s   € ô( ˜6Õ*r)   Únamec                 óœ  • U=(       d    [         nUR                  (       a  UR                  (       a  [        S5      e[        R
                  " UR                  S5      (       d  UR                  (       a  [        S5      eUR                  S:X  a0  [         H  n[        X5      (       d  M  X1l          O   [        S5      e[        XR                  5      nUb)  [        U[        R                  R                  5      (       d  [        SUR                   S35      e[        UR                   5      S:X  a  [        S	5      eS
n[#        U5       GHQ  u  pgXaR                   ;  a  M  SnUR                  (       au  UR$                  (       ad  [&        R)                  SUR                   SU S35        [*        R,                  " U5      n[/        UR                  5      n	UR1                  X’5        GO	UR                  (       d  UR                  (       aç  UR3                  5        HÓ  u  p«[        U[4        5      (       d  M  UR6                  (       a  M/  [&        R)                  SUR                   SU SU
 S35        [8        R:                  " UR<                  R>                  5      R@                  n[*        R,                  " U5      n[C        XÁR                  UR                  5      n	UR1                  X’5        MÕ     UR$                  (       d  GM¼  UR3                  5        H�  u  p«[        U[D        5      (       d  M  [&        R)                  SUR                   SU SU
 S35        [*        R,                  " U5      n[G        UR                  5      n	UR1                  X’5        Mƒ     GMT     U(       d&  [        SUR                    SUR                   S35      eg )NzXCannot set both `skip_attention` and `skip_attention_scores` to True. Please choose one.r   r    r   z±Could not find a suitable identifier for the transformer blocks automatically. Please provide a valid `fqn` (fully qualified name) that identifies a stack of transformer blocks.zCould not find zº in the provided module, or configured `fqn` (fully qualified name) does not identify a `torch.nn.ModuleList`. Please provide a valid `fqn` that identifies a stack of transformer blocks.r   zTLayer index list is empty. Please provide a non-empty list of layer indices to skip.FTz&Applying TransformerBlockSkipHook to 'r   Ú'z(Applying AttentionProcessorSkipHook to 'z!Applying FeedForwardSkipHook to 'zDCould not find any transformer blocks matching the provided indices z and fully qualified name 'z4'. Please check the indices and fqn for correctness.)$Ú_LAYER_SKIP_HOOKr   r   r!   r"   r#   r   r   r
   Úhasattrr   Ú
isinstancerI   rJ   Ú
ModuleListrt   r   Ú	enumerater   ÚloggerÚdebugr   Úcheck_if_exists_or_initializery   Úregister_hookÚnamed_modulesr   Úis_cross_attentionr   rM   Ú	processorrn   rX   rV   r   rh   )r\   rˆ   rŒ   Ú
identifierÚtransformer_blocksÚblocks_foundÚiÚblockÚregistryÚhookÚsubmodule_nameÚ	submoduleÚ	output_fns                r&   rŠ   rŠ   Ç   s  € Ø×#Ô#€Dà×× ×!=×!=ÜÐsÓtÐtÜ�<Š<˜Ÿ™¨×,Ñ,°×1M×1MÜØpó
ð 	
ð ‡z�z�VÓß<ˆJÜ�v×*Ó*Ø'”
Ùñ =ô
 ð^óð ô
 1°¿¹ÓDÐØÑ!¬Ð4FÌÏÉ×H[ÑH[×)\Ñ)\ÜØ˜fŸj™j˜\ð *sð tó
ð 	
ô ˆ6�>‰>Ó˜aÓÜÐoÓpÐpà€LÜÐ0×1‰ˆØ—N‘NÓ"Ùàˆà× ×  V§^§^Ü�L‰LÐAÀ&Ç*Á*ÀÈQÈqÈcÐQRÐSÔTÜ#×AÒAÀ%ÓHˆHÜ+¨F¯N©NÓ;ˆDØ×"Ñ" 4Ö.à×"×" f×&B×&BØ-2×-@Ñ-@Ö-BÑ)�Ü˜iÔ);×<Ó<ÀY×Ea×EaÑEaÜ—L‘LÐ#KÈFÏJÉJÈ<ÐWXÐYZÐX[Ð[\Ð]kÐ\lÐlmÐ!nÔoÜ :× >Ò >¸y×?RÑ?R×?\Ñ?\Ó ]× vÑ v�IÜ+×IÒIÈ)ÓT�HÜ5°i×A]ÑA]Ð_e×_mÑ_mÓn�DØ×*Ñ*¨4Ö6ñ .Cð �>�>Š>Ø-2×-@Ñ-@Ö-BÑ)�Ü˜iÔ)=×>Ó>Ü—L‘LÐ#DÀVÇZÁZÀLÐPQÐRSÐQTÐTUÐVdÐUeÐefÐ!gÔhÜ+×IÒIÈ)ÓT�HÜ.¨v¯~©~Ó>�DØ×*Ñ*¨4Ö6ô .Cñ- 2ö: ÜØRÐSY×SaÑSaÐRbð c%Ø%+§Z¡Z LÐ0dðfó
ð 	
ð r)   r+   )%r"   Údataclassesr   r   Útypingr   rI   Úutilsr   Úutils.torch_utilsr   Ú_commonr
   r   r   r   Ú_helpersr   r   Úhooksr   r   r4   r”   r�   r   Ú	overridesÚTorchFunctionModerC   rV   rh   ry   rJ   rf   r‹   r<   rŠ   r1   r)   r&   Ú<module>r®      sê   ðó ß )Ý ã å Ý -÷ó ÷ Kß *ñ 
�HÓ	€à$Ð ð
 ÷-'ð -'ó ð-'ô`% U§_¡_×%FÑ%Fô %ô* ô ô.˜)ô ô$˜yô ð2+˜UŸX™XŸ_™_ð +°oð +È$ô +ñ.@
 5§8¡8§?¡?ð @
¸Oð @
ÐSVÐY]ÑS]ð @
Ðimö @
r)   