ó
    ˆŽ¡jBx  ã                   óN  • S r SSKrSSKJr  SSKJr  SSKJr  SSKr	SSK
r
SSK
Jr  SSKJr  SS	KJr  SS
KJr  SSKJrJr  SSKJrJr  SSKJr  SSKJrJrJrJr  SSKJ r J!r!  SSK"J#r#J$r$  SSK%J&r&  SSK'J(r(  \RR                  " \*5      r+\" SS9\ " S S\5      5       5       r,\" SS9\ " S S\5      5       5       r-S r. " S S\R^                  5      r0 " S S\R^                  5      r1  SDS \R^                  S!\
Rd                  S"\
Rd                  S#\
Rd                  S$\
Rd                  S-  S%\3S-  S&\3S'\\   4S( jjr4 " S) S*\R^                  5      r5 " S+ S,\R^                  5      r6 " S- S.\R^                  5      r7 " S/ S0\R^                  5      r8 " S1 S2\R^                  5      r9 " S3 S4\5      r: " S5 S6\R^                  5      r;\ " S7 S8\5      5       r<\ " S9 S:\<5      5       r= " S; S<\R^                  5      r>\" S=S9 " S> S?\<5      5       r?\" S@S9 " SA SB\<5      5       r@/ SCQrAg)Ez,PyTorch VideoMAE (masked autoencoder) model.é    N)ÚCallable)Údeepcopy)Ú	dataclass)Únn)ÚMSELossé   )ÚACT2FN)ÚGradientCheckpointingLayer)ÚBaseModelOutputÚImageClassifierOutput)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚModelOutputÚTransformersKwargsÚauto_docstringÚlogging)ÚIMAGENET_DEFAULT_MEANÚIMAGENET_DEFAULT_STD)Úcan_return_tupleÚmerge_with_config_defaults)Úcapture_outputsé   )ÚVideoMAEConfigz[
    Class for VideoMAEDecoder's outputs, with potential hidden states and attentions.
    )Úcustom_introc                   óž   • \ rS rSr% SrSr\R                  S-  \S'   Sr	\
\R                     S-  \S'   Sr\
\R                     S-  \S'   Srg)ÚVideoMAEDecoderOutputé)   zx
logits (`torch.FloatTensor` of shape `(batch_size, patch_size ** 2 * num_channels)`):
    Pixel reconstruction logits.
NÚlogitsÚhidden_statesÚ
attentions© )Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__r   ÚtorchÚFloatTensorÚ__annotations__r    Útupler!   Ú__static_attributes__r"   ó    Ú¼C:\Users\johnn\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.13_qbz5n2kfra8p0\LocalCache\local-packages\Python313\site-packages\transformers/models/videomae/modeling_videomae.pyr   r   )   sR   ‡ ñð
 (,€FˆE×Ñ Ñ$Ó+Ø59€M�5˜×*Ñ*Ñ+¨dÑ2Ó9Ø26€J��e×'Ñ'Ñ(¨4Ñ/Ö6r-   r   zb
    Class for VideoMAEForPreTraining's outputs, with potential hidden states and attentions.
    c                   óÆ   • \ rS rSr% SrSr\R                  S-  \S'   Sr	\R                  S-  \S'   Sr
\\R                     S-  \S'   Sr\\R                     S-  \S'   Srg)	ÚVideoMAEForPreTrainingOutputé:   zÃ
loss (`torch.FloatTensor` of shape `(1,)`):
    Pixel reconstruction loss.
logits (`torch.FloatTensor` of shape `(batch_size, patch_size ** 2 * num_channels)`):
    Pixel reconstruction logits.
NÚlossr   r    r!   r"   )r#   r$   r%   r&   r'   r2   r(   r)   r*   r   r    r+   r!   r,   r"   r-   r.   r0   r0   :   sg   ‡ ñð &*€Dˆ%×
Ñ
˜dÑ
"Ó)Ø'+€FˆE×Ñ Ñ$Ó+Ø59€M�5˜×*Ñ*Ñ+¨dÑ2Ó9Ø26€J��e×'Ñ'Ñ(¨4Ñ/Ö6r-   r0   c                 óv  ^• U4S jn[         R                  " [        U 5       Vs/ s H
  o2" U5      PM     sn5      n[         R                  " USS2SSS24   5      USS2SSS24'   [         R                  " USS2SSS24   5      USS2SSS24'   [
        R                  " U5      R                  S5      $ s  snf )z Sinusoid position encoding tablec           
      ó†   >• [        T5       Vs/ s H%  o[        R                  " SSUS-  -  T-  5      -  PM'     sn$ s  snf )Ni'  é   )ÚrangeÚnpÚpower)ÚpositionÚhid_jÚd_hids     €r.   Úget_position_angle_vecÚ;get_sinusoid_encoding_table.<locals>.get_position_angle_vecT   s?   ø€ ÜRWÐX]ÔR^Ó_ÒR^Èœ2Ÿ8š8 E¨1°¸±
Ñ+;¸eÑ+CÓDÔDÑR^Ñ_Ð_ùÒ_s   �,>Nr   r5   r   )r7   Úarrayr6   ÚsinÚcosr(   r)   Ú	unsqueeze)Ú
n_positionr;   r<   Úpos_iÚsinusoid_tables    `   r.   Úget_sinusoid_encoding_tablerE   P   s°   ø€ õ`ô —X’XÌ%ÐPZÔJ[Ó\ÒJ[ÀÐ5°eÖ<ÑJ[Ñ\Ó]€NÜ Ÿfšf ^²A°q°t¸!°t°GÑ%<Ó=€N’1�a�d˜�d�7ÑÜ Ÿfšf ^²A°q°t¸!°t°GÑ%<Ó=€N’1�a�d˜�d�7Ñä×Ò˜^Ó,×6Ñ6°qÓ9Ð9ùò	 ]s   ¥B6c                   ó2   ^ • \ rS rSrSrU 4S jrS rSrU =r$ )ÚVideoMAEEmbeddingsé^   z/
Construct the patch and position embeddings.

c                 óÎ   >• [         TU ]  5         [        U5      U l        U R                  R                  U l        [        U R                  UR                  5      U l        Xl        g ©N)	ÚsuperÚ__init__ÚVideoMAEPatchEmbeddingsÚpatch_embeddingsÚnum_patchesrE   Úhidden_sizeÚposition_embeddingsÚconfig©ÚselfrR   Ú	__class__s     €r.   rL   ÚVideoMAEEmbeddings.__init__d   sP   ø€ Ü‰ÑÔä 7¸Ó ?ˆÔØ×0Ñ0×<Ñ<ˆÔä#>¸t×?OÑ?OÐQW×QcÑQcÓ#dˆÔ Ø�r-   c                 ó  • U R                  U5      nX0R                  R                  5       R                  U5      R	                  UR
                  SS9-   nUb'  UR                  u  pEnX2)    nUR                  USU5      nU$ )NT©ÚdeviceÚcopyéÿÿÿÿ)rN   rQ   ÚdetachÚtype_asÚtorY   ÚshapeÚreshape)rT   Úpixel_valuesÚbool_masked_posÚ
embeddingsÚ
batch_sizeÚ_Únum_channelss          r.   ÚforwardÚVideoMAEEmbeddings.forwardm   s‘   € à×*Ñ*¨<Ó8ˆ
ð  ×":Ñ":×"AÑ"AÓ"C×"KÑ"KÈJÓ"W×"ZÑ"ZØ×$Ñ$¨4ð #[ð #
ñ 
ˆ
ð
 Ñ&Ø*4×*:Ñ*:Ñ'ˆJ˜<Ø#Ð$4Ñ5ˆJØ#×+Ñ+¨J¸¸LÓIˆJàÐr-   )rR   rO   rN   rQ   ©	r#   r$   r%   r&   r'   rL   rg   r,   Ú__classcell__©rU   s   @r.   rG   rG   ^   s   ø† ñõ
÷ð r-   rG   c                   ó2   ^ • \ rS rSrSrU 4S jrS rSrU =r$ )rM   é   ac  
Video to Patch Embedding. This module turns a batch of videos of shape (batch_size, num_frames, num_channels,
height, width) into a tensor of shape (batch_size, seq_len, hidden_size) to be consumed by a Transformer encoder.

The seq_len (the number of patches) equals (number of frames // tubelet_size) * (height // patch_size) * (width //
patch_size).

c           	      óŠ  >• [         T	U ]  5         UR                  nUR                  nUR                  nUR
                  nUR                  nUR                  n[        U[        R                  R                  5      (       a  UOX"4n[        U[        R                  R                  5      (       a  UOX34nX l        X0l        [        U5      U l        US   US   -  US   US   -  -  X`R                  -  -  nX@l        X€l        [        R                  " UUU R                  US   US   4U R                  US   US   4S9U l        g )Nr   r   )Úin_channelsÚout_channelsÚkernel_sizeÚstride)rK   rL   Ú
image_sizeÚ
patch_sizerf   rP   Ú
num_framesÚtubelet_sizeÚ
isinstanceÚcollectionsÚabcÚIterableÚintrO   r   ÚConv3dÚ
projection)
rT   rR   rs   rt   rf   rP   ru   rv   rO   rU   s
            €r.   rL   Ú VideoMAEPatchEmbeddings.__init__‰   s3  ø€ Ü‰ÑÔà×&Ñ&ˆ
Ø×&Ñ&ˆ
Ø×*Ñ*ˆØ×(Ñ(ˆØ×&Ñ&ˆ
Ø×*Ñ*ˆä#-¨j¼+¿/¹/×:RÑ:R×#SÑ#S‘ZÐZdÐYqˆ
Ü#-¨j¼+¿/¹/×:RÑ:R×#SÑ#S‘ZÐZdÐYqˆ
Ø$ŒØ$ŒÜ Ó-ˆÔà˜‰]˜j¨™mÑ+°
¸1±ÀÈAÁÑ0NÑOÐS]×arÑarÑSrÑsð 	ð )ÔØ&ÔÜŸ)š)Ø$Ø$Ø×*Ñ*¨J°q©M¸:Àa¹=ÐIØ×%Ñ% z°!¡}°jÀ±mÐDñ	
ˆ�r-   c                 ó�  • UR                   u  p#pEnX@R                  :w  a  [        S5      eXPR                  S   :w  d  X`R                  S   :w  a2  [        SU SU SU R                  S    SU R                  S    S3	5      eUR	                  SSSS	S
5      nU R                  U5      R                  S5      R                  SS5      nU$ )NzeMake sure that the channel dimension of the pixel values match with the one set in the configuration.r   r   zInput image size (Ú*z) doesn't match model (z).r5   r   é   )r_   rf   Ú
ValueErrorrs   Úpermuter}   ÚflattenÚ	transpose)rT   ra   rd   ru   rf   ÚheightÚwidthrc   s           r.   rg   ÚVideoMAEPatchEmbeddings.forward¤   sÜ   € Ø>J×>PÑ>PÑ;ˆ
 °eØ×,Ñ,Ó,ÜØwóð ð —_‘_ QÑ'Ó'¨5·O±OÀAÑ4FÓ+FÜØ$ V H¨A¨e¨WÐ4KÈDÏOÉOÐ\]ÑL^ÐK_Ð_`Ðae×apÑapÐqrÑasÐ`tÐtvÐwóð ð $×+Ñ+¨A¨q°!°Q¸Ó:ˆØ—_‘_ \Ó2×:Ñ:¸1Ó=×GÑGÈÈ1ÓMˆ
ØÐr-   )rs   rf   rO   rt   r}   rv   ri   rk   s   @r.   rM   rM      s   ø† ñõ
÷6ð r-   rM   ÚmoduleÚqueryÚkeyÚvalueÚattention_maskÚscalingÚdropoutÚkwargsc                 óŽ  • Uc  UR                  S5      S-  n[        R                  " XR                  SS5      5      U-  nUb  X„-   n[        R
                  R                  USS9n[        R
                  R                  X†U R                  S9n[        R                  " Xƒ5      n	U	R                  SS5      R                  5       n	X˜4$ )Nr[   ç      à¿r5   r   ©Údim)ÚpÚtrainingr   )
Úsizer(   Úmatmulr…   r   Ú
functionalÚsoftmaxr�   r–   Ú
contiguous)
r‰   rŠ   r‹   rŒ   r�   rŽ   r�   r�   Úattn_weightsÚattn_outputs
             r.   Úeager_attention_forwardrž   µ   s±   € ð �Ø—*‘*˜R“. DÑ(ˆô —<’< §}¡}°Q¸Ó':Ó;¸gÑE€LàÑ!Ø#Ñ4ˆä—=‘=×(Ñ(¨¸2Ð(Ð>€LÜ—=‘=×(Ñ(¨È6Ï?É?Ð(Ð[€Lä—,’,˜|Ó3€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€KàÐ$Ð$r-   c                   ó˜   ^ • \ rS rSrS\SS4U 4S jjr S	S\R                  S-  S\\R                  \R                  4   4S jjr	Sr
U =r$ )
ÚVideoMAESelfAttentionéÑ   rR   ÚreturnNc                 ó0  >• [         TU ]  5         UR                  UR                  -  S:w  a7  [	        US5      (       d&  [        SUR                   SUR                   S35      eXl        UR                  U l        [        UR                  UR                  -  5      U l        U R                  U R                  -  U l	        UR                  U l        U R                  S-  U l        SU l        [        R                  " UR                  U R                  UR                   S9U l        [        R                  " UR                  U R                  UR                   S9U l        [        R                  " UR                  U R                  UR                   S9U l        g )	Nr   Úembedding_sizezThe hidden size z4 is not a multiple of the number of attention heads Ú.r’   F©Úbias)rK   rL   rP   Únum_attention_headsÚhasattrr‚   rR   r{   Úattention_head_sizeÚall_head_sizeÚattention_probs_dropout_probÚdropout_probrŽ   Ú	is_causalr   ÚLinearÚqkv_biasrŠ   r‹   rŒ   rS   s     €r.   rL   ÚVideoMAESelfAttention.__init__Ò   sG  ø€ Ü‰ÑÔØ×Ñ × :Ñ :Ñ:¸aÓ?ÌÐPVÐXh×HiÑHiÜØ" 6×#5Ñ#5Ð"6ð 7Ø×3Ñ3Ð4°Að7óð ð ŒØ#)×#=Ñ#=ˆÔ Ü#& v×'9Ñ'9¸F×<VÑ<VÑ'VÓ#WˆÔ Ø!×5Ñ5¸×8PÑ8PÑPˆÔØ"×?Ñ?ˆÔØ×/Ñ/°Ñ5ˆŒØˆŒä—Y’Y˜v×1Ñ1°4×3EÑ3EÈFÏOÉOÑ\ˆŒ
Ü—9’9˜V×/Ñ/°×1CÑ1CÈ&Ï/É/ÑZˆŒÜ—Y’Y˜v×1Ñ1°4×3EÑ3EÈFÏOÉOÑ\ˆ�
r-   r    c                 ó®  • UR                   S S n/ UQSPU R                  P7nU R                  U5      R                  U5      R	                  SS5      nU R                  U5      R                  U5      R	                  SS5      nU R                  U5      R                  U5      R	                  SS5      n[        R                  " U R                  R                  [        5      nU" U UUUS U R                  U R                  U R                  (       d  SOU R                  S9u  p‰UR!                  5       S S U R"                  4-   n
UR%                  U
5      nX‰4$ )Nr[   r   r5   ç        )r®   rŽ   r�   éþÿÿÿ)r_   rª   r‹   Úviewr…   rŒ   rŠ   r   Úget_interfacerR   Ú_attn_implementationrž   r®   rŽ   r–   r­   r—   r«   r`   )rT   r    Úinput_shapeÚhidden_shapeÚkeysÚvaluesÚqueriesÚattention_interfaceÚcontext_layerÚattention_probsÚnew_context_layer_shapes              r.   rg   ÚVideoMAESelfAttention.forwardå   s@  € ð $×)Ñ)¨#¨2Ð.ˆàC˜ÐC bÐC¨$×*BÑ*BÑCˆØ�x‰x˜Ó&×+Ñ+¨LÓ9×CÑCÀAÀqÓIˆØ—‘˜MÓ*×/Ñ/°Ó=×GÑGÈÈ1ÓMˆØ—*‘*˜]Ó+×0Ñ0°Ó>×HÑHÈÈAÓNˆä(?×(MÒ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ *=ØØØØØØ—n‘nØ—L‘LØ#Ÿ}Ÿ}‘C°$×2CÑ2Cñ	*
Ñ&ˆð #0×"4Ñ"4Ó"6°s¸Ð";¸t×?QÑ?QÐ>SÑ"SÐØ%×-Ñ-Ð.EÓFˆàÐ-Ð-r-   )
r«   rª   rR   r­   r®   r‹   r¨   rŠ   rŽ   rŒ   rJ   )r#   r$   r%   r&   r   rL   r(   ÚTensorr+   rg   r,   rj   rk   s   @r.   r    r    Ñ   sS   ø† ð]˜~ð ]°$÷ ]ð( 48ñ.Ø"Ÿ\™\¨DÑ0ð.à	ˆu�|‰|˜UŸ\™\Ð)Ñ	*÷.ó .r-   r    c                   ó†   ^ • \ rS rSrSrS\4U 4S jjrS\R                  S\R                  S\R                  4S jr	S	r
U =r$ )
ÚVideoMAESelfOutputi  z™
The residual connection is defined in VideoMAELayer instead of here (as is the case with other models), due to the
layernorm applied before each block.
rR   c                 óÌ   >• [         TU ]  5         [        R                  " UR                  UR                  5      U l        [        R                  " UR                  5      U l        g rJ   )	rK   rL   r   r¯   rP   ÚdenseÚDropoutÚhidden_dropout_probr�   rS   s     €r.   rL   ÚVideoMAESelfOutput.__init__  sB   ø€ Ü‰ÑÔÜ—Y’Y˜v×1Ñ1°6×3EÑ3EÓFˆŒ
Ü—z’z &×"<Ñ"<Ó=ˆ�r-   r    Úinput_tensorr¢   c                 óJ   • U R                  U5      nU R                  U5      nU$ rJ   ©rÆ   r�   ©rT   r    rÊ   s      r.   rg   ÚVideoMAESelfOutput.forward  s$   € ØŸ
™
 =Ó1ˆØŸ™ ]Ó3ˆØÐr-   rÌ   )r#   r$   r%   r&   r'   r   rL   r(   rÂ   rg   r,   rj   rk   s   @r.   rÄ   rÄ     sB   ø† ñð
>˜~÷ >ð
 U§\¡\ð ÀÇÁð ÐRW×R^ÑR^÷ ò r-   rÄ   c                   ót   ^ • \ rS rSrS\4U 4S jjrS\R                  S\\	   S\R                  4S jr
SrU =r$ )	ÚVideoMAEAttentioni  rR   c                 ób   >• [         TU ]  5         [        U5      U l        [	        U5      U l        g rJ   )rK   rL   r    Ú	attentionrÄ   ÚoutputrS   s     €r.   rL   ÚVideoMAEAttention.__init__  s&   ø€ Ü‰ÑÔÜ.¨vÓ6ˆŒÜ(¨Ó0ˆ�r-   r    r�   r¢   c                 óR   • U R                   " U40 UD6u  p4U R                  X15      nU$ rJ   ©rÒ   rÓ   )rT   r    r�   Úself_attn_outputre   rÓ   s         r.   rg   ÚVideoMAEAttention.forward  s/   € ð
 #Ÿnšn¨]ÑE¸fÑEÑÐØ—‘Ð-Ó=ˆØˆr-   rÖ   )r#   r$   r%   r&   r   rL   r(   rÂ   r   r   rg   r,   rj   rk   s   @r.   rÐ   rÐ     sC   ø† ð1˜~÷ 1ð
à—|‘|ðð Ð+Ñ,ðð 
�‰÷	ò r-   rÐ   c                   ój   ^ • \ rS rSrS\4U 4S jjrS\R                  S\R                  4S jrSr	U =r
$ )ÚVideoMAEIntermediatei(  rR   c                 ó  >• [         TU ]  5         [        R                  " UR                  UR
                  5      U l        [        UR                  [        5      (       a  [        UR                     U l        g UR                  U l        g rJ   )rK   rL   r   r¯   rP   Úintermediate_sizerÆ   rw   Ú
hidden_actÚstrr	   Úintermediate_act_fnrS   s     €r.   rL   ÚVideoMAEIntermediate.__init__)  s`   ø€ Ü‰ÑÔÜ—Y’Y˜v×1Ñ1°6×3KÑ3KÓLˆŒ
Ü�f×'Ñ'¬×-Ñ-Ü'-¨f×.?Ñ.?Ñ'@ˆDÕ$à'-×'8Ñ'8ˆDÕ$r-   r    r¢   c                 óJ   • U R                  U5      nU R                  U5      nU$ rJ   ©rÆ   rß   )rT   r    s     r.   rg   ÚVideoMAEIntermediate.forward1  s&   € ØŸ
™
 =Ó1ˆØ×0Ñ0°Ó?ˆØÐr-   râ   ©r#   r$   r%   r&   r   rL   r(   rÂ   rg   r,   rj   rk   s   @r.   rÚ   rÚ   (  s/   ø† ð9˜~÷ 9ð U§\¡\ð °e·l±l÷ ò r-   rÚ   c                   ó‚   ^ • \ rS rSrS\4U 4S jjrS\R                  S\R                  S\R                  4S jrSr	U =r
$ )	ÚVideoMAEOutputi8  rR   c                 óÌ   >• [         TU ]  5         [        R                  " UR                  UR
                  5      U l        [        R                  " UR                  5      U l	        g rJ   )
rK   rL   r   r¯   rÜ   rP   rÆ   rÇ   rÈ   r�   rS   s     €r.   rL   ÚVideoMAEOutput.__init__9  sB   ø€ Ü‰ÑÔÜ—Y’Y˜v×7Ñ7¸×9KÑ9KÓLˆŒ
Ü—z’z &×"<Ñ"<Ó=ˆ�r-   r    rÊ   r¢   c                 óR   • U R                  U5      nU R                  U5      nX-   nU$ rJ   rÌ   rÍ   s      r.   rg   ÚVideoMAEOutput.forward>  s,   € ØŸ
™
 =Ó1ˆØŸ™ ]Ó3ˆØ%Ñ4ˆØÐr-   rÌ   rä   rk   s   @r.   ræ   ræ   8  s=   ø† ð>˜~÷ >ð
 U§\¡\ð ÀÇÁð ÐRW×R^ÑR^÷ ò r-   ræ   c                   óx   ^ • \ rS rSrSrS\4U 4S jjrS\R                  S\	\
   S\R                  4S jrS	rU =r$ )
ÚVideoMAELayeriF  z?This corresponds to the Block class in the timm implementation.rR   c                 ój  >• [         TU ]  5         UR                  U l        SU l        [	        U5      U l        [        U5      U l        [        U5      U l	        [        R                  " UR                  UR                  S9U l        [        R                  " UR                  UR                  S9U l        g )Nr   ©Úeps)rK   rL   Úchunk_size_feed_forwardÚseq_len_dimrÐ   rÒ   rÚ   Úintermediateræ   rÓ   r   Ú	LayerNormrP   Úlayer_norm_epsÚlayernorm_beforeÚlayernorm_afterrS   s     €r.   rL   ÚVideoMAELayer.__init__I  s‡   ø€ Ü‰ÑÔØ'-×'EÑ'EˆÔ$ØˆÔÜ*¨6Ó2ˆŒÜ0°Ó8ˆÔÜ$ VÓ,ˆŒÜ "§¢¨V×-?Ñ-?ÀV×EZÑEZÑ [ˆÔÜ!Ÿ|š|¨F×,>Ñ,>ÀF×DYÑDYÑZˆÕr-   r    r�   r¢   c                 ó¼   • U R                  U5      nU R                  " U40 UD6nXA-   nU R                  U5      nU R                  U5      nU R	                  XQ5      nU$ rJ   )rõ   rÒ   rö   rò   rÓ   )rT   r    r�   Úhidden_states_normÚattention_outputÚlayer_outputs         r.   rg   ÚVideoMAELayer.forwardS  sl   € ð
 "×2Ñ2°=ÓAÐØŸ>š>Ð*<ÑGÀÑGÐð )Ñ8ˆð ×+Ñ+¨MÓ:ˆØ×(Ñ(¨Ó6ˆð —{‘{ <Ó?ˆàÐr-   )rÒ   rð   rò   rö   rõ   rÓ   rñ   )r#   r$   r%   r&   r'   r   rL   r(   rÂ   r   r   rg   r,   rj   rk   s   @r.   rì   rì   F  sH   ø† ÙIð[˜~÷ [ðà—|‘|ðð Ð+Ñ,ðð 
�‰÷	ò r-   rì   c                   ó`   ^ • \ rS rSrS\4U 4S jjrS\R                  S\\	   S\
4S jrSrU =r$ )	ÚVideoMAEEncoderii  rR   c                 óÖ   >• [         TU ]  5         Xl        [        R                  " [        UR                  5       Vs/ s H  n[        U5      PM     sn5      U l        SU l	        g s  snf )NF)
rK   rL   rR   r   Ú
ModuleListr6   Únum_hidden_layersrì   ÚlayerÚgradient_checkpointing)rT   rR   re   rU   s      €r.   rL   ÚVideoMAEEncoder.__init__j  sR   ø€ Ü‰ÑÔØŒÜ—]’]Ä5È×IaÑIaÔCbÓ#cÒCb¸a¤M°&Ö$9ÑCbÑ#cÓdˆŒ
Ø&+ˆÕ#ùò $ds   ½A&r    r�   r¢   c                 óL   • U R                    H  nU" U40 UD6nM     [        US9$ )N©Úlast_hidden_state)r  r   )rT   r    r�   Úlayer_modules       r.   rg   ÚVideoMAEEncoder.forwardp  s.   € ð
 !ŸJœJˆLÙ(¨ÑA¸&ÑAŠMñ 'ô °Ñ?Ð?r-   )rR   r  r  )r#   r$   r%   r&   r   rL   r(   rÂ   r   r   r   rg   r,   rj   rk   s   @r.   rþ   rþ   i  sD   ø† ð,˜~÷ ,ð@à—|‘|ð@ð Ð+Ñ,ð@ð 
÷	@ò @r-   rþ   c                   óR   • \ rS rSr% \\S'   SrSrSrSr	SS/r
SrSrSrSr\\S	.rS
rg)ÚVideoMAEPreTrainedModeli{  rR   Úvideomaera   ÚvideoTrG   rì   )r    r!   r"   N)r#   r$   r%   r&   r   r*   Úbase_model_prefixÚmain_input_nameÚinput_modalitiesÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_supports_sdpaÚ_supports_flash_attnÚ_supports_flex_attnÚ_supports_attention_backendrì   r    Ú_can_record_outputsr,   r"   r-   r.   r  r  {  sO   ‡ àÓØ"ÐØ$€OØÐØ&*Ð#Ø-¨Ð?ÐØ€NØÐØÐØ"&Ðà&Ø+ñÓr-   r  c                   ó¨   ^ • \ rS rSrU 4S jrS r\\" SS9\ SS\	R                  S\	R                  S-  S	\\   S
\4S jj5       5       5       rSrU =r$ )ÚVideoMAEModeli�  c                 ó  >• [         TU ]  U5        Xl        [        U5      U l        [        U5      U l        UR                  (       a  S U l        O.[        R                  " UR                  UR                  S9U l        U R                  5         g )Nrî   )rK   rL   rR   rG   rc   rþ   ÚencoderÚuse_mean_poolingÚ	layernormr   ró   rP   rô   Ú	post_initrS   s     €r.   rL   ÚVideoMAEModel.__init__�  sg   ø€ Ü‰Ñ˜Ô ØŒä,¨VÓ4ˆŒÜ& vÓ.ˆŒà×"×"Ø!ˆD�NäŸ\š\¨&×*<Ñ*<À&×BWÑBWÑXˆDŒNð 	�‰Õr-   c                 ó.   • U R                   R                  $ rJ   )rc   rN   )rT   s    r.   Úget_input_embeddingsÚ"VideoMAEModel.get_input_embeddingsž  s   € Ø�‰×/Ñ/Ð/r-   F)Útie_last_hidden_statesNra   rb   r�   r¢   c                 ó¬   • U R                  X5      nU R                  U5      nUR                  nU R                  b  U R                  U5      n[	        US9$ )a‚  
bool_masked_pos (`torch.BoolTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Boolean masked positions. Indicates which patches are masked (1) and which aren't (0). Each video in the
    batch must have the same number of masked patches. If `None`, then all patches are considered. Sequence
    length is `(num_frames // tubelet_size) * (image_size // patch_size) ** 2`.

Examples:

```python
>>> import torch
>>> from transformers import VideoMAEVideoProcessor, VideoMAEModel
>>> from huggingface_hub import hf_hub_download

>>> # replace this with your own video file
>>> video_path = hf_hub_download(
...     repo_id="nielsr/video-demo", filename="eating_spaghetti.mp4", repo_type="dataset"
... )

>>> video_processor = VideoMAEVideoProcessor.from_pretrained("MCG-NJU/videomae-base")
>>> model = VideoMAEModel.from_pretrained("MCG-NJU/videomae-base")

>>> # prepare video for the model
>>> inputs = video_processor(video_path, return_tensors="pt")

>>> # forward pass
>>> with torch.no_grad():
...     outputs = model(**inputs)

>>> last_hidden_states = outputs.last_hidden_state
>>> list(last_hidden_states.shape)
[1, 1568, 768]
```r  )rc   r  r  r  r   )rT   ra   rb   r�   Úembedding_outputÚencoder_outputsÚsequence_outputs          r.   rg   ÚVideoMAEModel.forward¡  sQ   € ðT  Ÿ?™?¨<ÓIÐà+/¯<©<Ð8HÓ+IˆØ)×;Ñ;ˆØ�>‰>Ñ%Ø"Ÿn™n¨_Ó=ˆOä°ÑAÐAr-   )rR   rc   r  r  rJ   )r#   r$   r%   r&   rL   r!  r   r   r   r(   r)   Ú
BoolTensorr   r   r   rg   r,   rj   rk   s   @r.   r  r  �  s~   ø† õò0ð  Ù¨EÑ2Øð 48ñ.Bà×'Ñ'ð.Bð ×)Ñ)¨DÑ0ð.Bð Ð+Ñ,ð	.Bð
 
ô.Bó ó 3ó  ö.Br-   r  c                   óV   ^ • \ rS rSrS\4U 4S jjrS\R                  S\4S jr	Sr
U =r$ )ÚVideoMAEDecoderiÕ  rR   c                 ó’  >• [         TU ]  5         UR                  UR                  -  UR                  S-  -  n[        U5      nUR                  Ul        UR                  Ul	        UR                  Ul        UR                  Ul        [        R                  " [!        UR                  5       Vs/ s H  n[#        U5      PM     sn5      U l        [        R&                  " UR                  5      U l        US:”  a!  [        R*                  " UR                  U5      O[        R,                  " 5       U l        SU l        X0l        g s  snf )Nr5   r   F)rK   rL   rf   rv   rt   r   Údecoder_hidden_sizerP   Údecoder_num_hidden_layersr  Údecoder_num_attention_headsr¨   Údecoder_intermediate_sizerÜ   r   r   r6   rì   Údecoder_layersró   Únormr¯   ÚIdentityÚheadr  rR   )rT   rR   Údecoder_num_labelsÚdecoder_configre   rU   s        €r.   rL   ÚVideoMAEDecoder.__init__Ö  s
  ø€ Ü‰ÑÔà#×0Ñ0°6×3FÑ3FÑFÈ×IZÑIZÐ\]ÑI]Ñ]Ðä! &Ó)ˆØ%+×%?Ñ%?ˆÔ"Ø+1×+KÑ+KˆÔ(Ø-3×-OÑ-OˆÔ*Ø+1×+KÑ+KˆÔ(Ü ŸmšmÜ49¸&×:ZÑ:ZÔ4[Ó\Ò4[¨qŒ]˜>Ö*Ñ4[Ñ\ó
ˆÔô —L’L ×!;Ñ!;Ó<ˆŒ	àI[Ð^_ÓI_ŒB�IŠI�f×0Ñ0Ð2DÔEÔeg×epÒepÓerð 	Œ	ð ',ˆÔ#Ø$�ùò ]s   Â/Er    Úreturn_token_numc                 ó¦   • U R                    H  nU" U5      nM     US S 2U* S 24   nU R                  U5      nU R                  U5      n[        US9$ )N)r   )r1  r2  r4  r   )rT   r    r8  r  r   s        r.   rg   ÚVideoMAEDecoder.forwardì  s]   € à ×/Ô/ˆLÙ(¨Ó7ŠMñ 0ð &¢aÐ*:Ð):Ñ);Ð&;Ñ<ˆð Ÿ	™	 -Ó0ˆØ—‘˜=Ó)ˆä$¨FÑ3Ð3r-   )rR   r1  r  r4  r2  )r#   r$   r%   r&   r   rL   r(   rÂ   r{   rg   r,   rj   rk   s   @r.   r+  r+  Õ  s+   ø† ð%˜~÷ %ð,4 U§\¡\ð 4ÀS÷ 4ò 4r-   r+  zb
    The VideoMAE Model transformer with the decoder on top for self-supervised pre-training.
    c            
       ó„   ^ • \ rS rSrU 4S jr\\S\R                  S\R                  S\
\   S\4S j5       5       rSrU =r$ )	ÚVideoMAEForPreTrainingiú  c                 óÖ  >• [         TU ]  U5        Xl        [        U5      U l        [
        R                  " UR                  UR                  SS9U l	        [
        R                  " [        R                  " SSUR                  5      5      U l        [        U R                  R                  R                   UR                  5      U l        [%        U5      U l        U R)                  5         g )NFr¦   r   )rK   rL   rR   r  r  r   r¯   rP   r-  Úencoder_to_decoderÚ	Parameterr(   ÚzerosÚ
mask_tokenrE   rc   rO   rQ   r+  Údecoderr  rS   s     €r.   rL   ÚVideoMAEForPreTraining.__init__   s¨   ø€ Ü‰Ñ˜Ô ØŒä% fÓ-ˆŒä"$§)¢)¨F×,>Ñ,>À×@ZÑ@ZÐafÑ"gˆÔÜŸ,š,¤u§{¢{°1°a¸×9SÑ9SÓ'TÓUˆŒÜ#>Ø�M‰M×$Ñ$×0Ñ0°&×2LÑ2Ló$
ˆÔ ô ' vÓ.ˆŒð 	�‰Õr-   ra   rb   r�   r¢   c                 ó  • U R                   " U4SU0UD6nUR                  nU R                  U5      nUR                  u  pgnUc  [	        S5      eU R
                  R                  USS5      R                  U5      n	U	R                  5       R                  UR                  SS9n	X’)    R                  USU5      n
X’   R                  USU5      n[        R                  " XZ-   U R                  U-   /SS9nU R                  XËR                  S   5      nUR                   nSn[        R"                  " 5          U R$                  R&                  S	:w  a  UnO„UR                  nUR(                  n[        R*                  " [,        5      R                  UUS
9SSSS2SS4   n[        R*                  " [.        5      R                  UUS
9SSSS2SS4   nUU-  U-   nUR                  u  nnnnnU R$                  R0                  U R$                  R2                  nnU R$                  R4                  (       aÒ  UR7                  UUU-  UUUU-  UUU-  U5      nUR9                  SSSSSSSS	5      R;                  5       nUR7                  UUU-  U-  U-  U-  U-  UU-  U-  U5      nUUR=                  SSS9-
  UR?                  SSSS9RA                  5       S-   -  nUR7                  UUU-  U-  U-  U-  U-  UU-  U-  U-  5      nO—U R$                  R&                  S	:w  a  [	        S5      eUR7                  UUU-  UUUU-  UUU-  U5      nUR9                  SSSSSSSS	5      R;                  5       nUR7                  UUU-  U-  U-  U-  U-  UU-  U-  U-  5      nUR                  u  pgnUU   R                  USU5      nSSS5        [C        5       nU" UW5      n[E        UUURF                  URH                  S9$ ! , (       d  f       NA= f)ae  
bool_masked_pos (`torch.BoolTensor` of shape `(batch_size, sequence_length)`):
    Boolean masked positions. Indicates which patches are masked (1) and which aren't (0). Each video in the
    batch must have the same number of masked patches. Sequence length is `(num_frames // tubelet_size) *
    (image_size // patch_size) ** 2`.

Examples:
```python
>>> from transformers import AutoImageProcessor, VideoMAEForPreTraining
>>> import numpy as np
>>> import torch

>>> num_frames = 16
>>> video = list(np.random.randint(0, 256, (num_frames, 3, 224, 224)))

>>> image_processor = AutoImageProcessor.from_pretrained("MCG-NJU/videomae-base")
>>> model = VideoMAEForPreTraining.from_pretrained("MCG-NJU/videomae-base")

>>> pixel_values = image_processor(video, return_tensors="pt").pixel_values

>>> num_patches_per_frame = (model.config.image_size // model.config.patch_size) ** 2
>>> seq_length = (num_frames // model.config.tubelet_size) * num_patches_per_frame
>>> bool_masked_pos = torch.randint(0, 2, (1, seq_length)).bool()

>>> outputs = model(pixel_values, bool_masked_pos=bool_masked_pos)
>>> loss = outputs.loss
```rb   Nz!One must provided a boolean mask r[   TrX   r   r“   r   )rY   Údtyper   r�   é   r5   é   é   r´   )r”   Úkeepdim)r”   ÚunbiasedrI  g�íµ ÷Æ°>zQCan't unnormalize non-RGB images. Consider setting config.norm_pix_loss to False.©r2   r   r    r!   )%r  r  r>  r_   r‚   rQ   Úexpandr]   r\   r^   rY   r`   r(   ÚcatrA  rB  r   Úno_gradrR   rf   rE  Ú	as_tensorr   r   rv   rt   Únorm_pix_lossrµ   rƒ   r›   ÚmeanÚvarÚsqrtr   r0   r    r!   )rT   ra   rb   r�   Úoutputsr'  rd   re   rf   Úexpanded_position_embeddingsÚpos_emb_visibleÚpos_emb_maskÚx_fullÚdecoder_outputsr   r2   ÚframesrY   rE  rQ  ÚstdÚtimer†   r‡   rv   rt   Úframes_normÚvideos_patchÚlabelsÚloss_fcts                                 r.   rg   ÚVideoMAEForPreTraining.forward  su  € ðF $(§=¢=°Ñ#iÈÐ#iÐbhÑ#iˆà!×3Ñ3ˆØ×1Ñ1°/ÓBˆð '6×&;Ñ&;Ñ#ˆ
�|ð Ñ"ÜÐ@ÓAÐAà'+×'?Ñ'?×'FÑ'FÀzÐSUÐWYÓ'Z×'bÑ'bÐcoÓ'pÐ$Ø'C×'JÑ'JÓ'L×'OÑ'OÐWc×WjÑWjÐquÐ'OÐ'vÐ$Ø6Ð7GÑH×PÑPÐQ[Ð]_ÐamÓnˆØ3ÑD×LÑLÈZÐY[Ð]iÓjˆô —’˜OÑ=¸t¿¹ÐQ]Ñ?]Ð^ÐdeÑfˆð 26·±¸f×FXÑFXÐYZÑF[Ó1\ˆØ ×'Ñ'ˆàˆÜ�]Š]�_à�{‰{×'Ñ'¨1Ó,à%‘ð &×,Ñ,�Ø$×*Ñ*�Ü—’Ô'<Ó=×@Ñ@ÈÐV[Ð@Ð\Ð]aÐcgÒijÐlpÐrvÐ]vÑw�Ü—o’oÔ&:Ó;×>Ñ>ÀfÐTYÐ>ÐZÐ[_ÐaeÒghÐjnÐptÐ[tÑu�Ø%¨Ñ+¨dÑ2�à<B¿L¹LÑ9ˆJ˜˜l¨F°EØ'+§{¡{×'?Ñ'?ÀÇÁ×AWÑAW˜*ˆLØ�{‰{×(×(àŸ™ØØ˜LÑ(Ø Ø Ø˜jÑ(ØØ˜ZÑ'Øó	�ð  Ÿ™¨¨1¨a°°A°q¸!¸QÓ?×JÑJÓL�àŸ™ØØ˜LÑ(¨6Ñ1°ZÑ?À%ÑGÈ:ÑUØ  :Ñ-°
Ñ:Ø ó	�ð  &¨¯©¸ÀD¨Ð(IÑIØ—J‘J 2°¸d�JÐC×HÑHÓJÈTÑQñ�ð  +×/Ñ/ØØ˜LÑ(¨6Ñ1°ZÑ?À%ÑGÈ:ÑUØ  :Ñ-°
Ñ:¸\ÑIó ‘ð —;‘;×+Ñ+¨qÓ0Ü$Økóð ð  Ÿ™ØØ˜LÑ(Ø Ø Ø˜jÑ(ØØ˜ZÑ'Øó	�ð  Ÿ™¨¨1¨a°°A°q¸!¸QÓ?×JÑJÓL�à%Ÿ{™{ØØ˜LÑ(¨6Ñ1°ZÑ?À%ÑGÈ:ÑUØ  :Ñ-°
Ñ:¸\ÑIó �ð +7×*<Ñ*<Ñ'ˆJ˜<Ø! /Ñ2×:Ñ:¸:ÀrÈ<ÓXˆF÷Q ôT “9ˆÙ˜ Ó'ˆä+ØØØ!×/Ñ/Ø×)Ñ)ñ	
ð 	
÷[ �_ús   Ä6J	O:Ï:
P)rR   rB  r>  rA  rQ   r  )r#   r$   r%   r&   rL   r   r   r(   r)   r)  r   r   r0   rg   r,   rj   rk   s   @r.   r<  r<  ú  sb   ø† õð" ØðL
à×'Ñ'ðL
ð ×)Ñ)ðL
ð Ð+Ñ,ð	L
ð
 
&óL
ó ó öL
r-   r<  z­
    VideoMAE Model transformer with a video classification head on top (a linear layer on top of the average pooled hidden
    states of all tokens) e.g. for ImageNet.
    c                   ó˜   ^ • \ rS rSrU 4S jr\\  S
S\R                  S-  S\R                  S-  S\	\
   S\4S jj5       5       rS	rU =r$ )ÚVideoMAEForVideoClassificationi¢  c                 ó   >• [         TU ]  U5        UR                  U l        [        U5      U l        UR
                  (       a   [        R                  " UR                  5      OS U l	        UR                  S:”  a+  [        R                  " UR                  UR                  5      O[        R                  " 5       U l        U R                  5         g )Nr   )rK   rL   Ú
num_labelsr  r  r  r   ró   rP   Úfc_normr¯   r3  Ú
classifierr  rS   s     €r.   rL   Ú'VideoMAEForVideoClassification.__init__©  s‘   ø€ Ü‰Ñ˜Ô à ×+Ñ+ˆŒÜ% fÓ-ˆŒð <B×;R×;R”r—|’| F×$6Ñ$6Ô7ÐX\ˆŒØNT×N_ÑN_ÐbcÓNcœ"Ÿ)š) F×$6Ñ$6¸×8IÑ8IÔJÔik×itÒitÓivˆŒð 	�‰Õr-   Nra   r_  r�   r¢   c                 óZ  • U R                   " U40 UD6nUR                  nU R                  b#  UR                  S5      nU R                  U5      nO	USS2S4   nU R	                  U5      nSnUb  U R
                  " X'U R                  40 UD6n[        UUUR                  UR                  S9$ )aØ  
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
    Labels for computing the image classification/regression loss. Indices should be in `[0, ...,
    config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
    `config.num_labels > 1` a classification loss is computed (Cross-Entropy).

Examples:

```python
>>> import torch
>>> from transformers import VideoMAEVideoProcessor, VideoMAEForVideoClassification
>>> from huggingface_hub import hf_hub_download

>>> # replace this with your own video file
>>> video_path = hf_hub_download(
...     repo_id="nielsr/video-demo", filename="eating_spaghetti.mp4", repo_type="dataset"
... )

>>> video_processor = VideoMAEVideoProcessor.from_pretrained("MCG-NJU/videomae-base-finetuned-kinetics")
>>> model = VideoMAEForVideoClassification.from_pretrained("MCG-NJU/videomae-base-finetuned-kinetics")

>>> inputs = video_processor(video_path, return_tensors="pt")

>>> with torch.no_grad():
...     outputs = model(**inputs)
...     logits = outputs.logits

>>> # model predicts one of the 400 Kinetics-400 classes
>>> predicted_label = logits.argmax(-1).item()
>>> print(model.config.id2label[predicted_label])
eating spaghetti
```Nr   r   rK  )
r  r  rf  rQ  rg  Úloss_functionrR   r   r    r!   )	rT   ra   r_  r�   rT  r'  rÓ   r   r2   s	            r.   rg   Ú&VideoMAEForVideoClassification.forward¶  s²   € ðR $(§=¢=°Ñ#HÀÑ#HˆØ!×3Ñ3ˆà�<‰<Ñ#Ø$×)Ñ)¨!Ó,ˆFØ—\‘\ &Ó)‰Fà$¢Q¨ TÑ*ˆFà—‘ Ó(ˆàˆØÑØ×%Ò% f°d·k±kÑLÀVÑLˆDä$ØØØ!×/Ñ/Ø×)Ñ)ñ	
ð 	
r-   )rg  rf  re  r  )NN)r#   r$   r%   r&   rL   r   r   r(   rÂ   r   r   r   rg   r,   rj   rk   s   @r.   rc  rc  ¢  sj   ø† õð Øð -1Ø&*ñ;
à—l‘l TÑ)ð;
ð —‘˜tÑ#ð;
ð Ð+Ñ,ð	;
ð
 
ô;
ó ó ö;
r-   rc  )r<  r  r  rc  )Nr³   )Br'   Úcollections.abcrx   r   rZ   r   Údataclassesr   Únumpyr7   r(   r   Útorch.nnr   Úactivationsr	   Úmodeling_layersr
   Úmodeling_outputsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   r   Úutils.constantsr   r   Úutils.genericr   r   Úutils.output_capturingr   Úconfiguration_videomaer   Ú
get_loggerr#   Úloggerr   r0   rE   ÚModulerG   rM   rÂ   Úfloatrž   r    rÄ   rÐ   rÚ   ræ   rì   rþ   r  r  r+  r<  rc  Ú__all__r"   r-   r.   Ú<module>r     sr  ðñ 3ã Ý $Ý Ý !ã Û Ý Ý å !Ý 9ß Fß FÝ &ß MÓ Mß Jß IÝ 5Ý 2ð 
×	Ò	˜HÓ	%€ñ ðñð
 ô7˜Kó 7ó óð7ñ ðñð
 ô7 ;ó 7ó óð7ò :ô˜Ÿ™ô ôB2˜bŸi™iô 2ðx !Øñ%Ø�I‰Ið%à�<‰<ð%ð 
�‰ð%ð �<‰<ð	%ð
 —L‘L 4Ñ'ð%ð �T‰\ð%ð ð%ð Ð'Ñ(õ%ô80.˜BŸI™Iô 0.ôh˜Ÿ™ô ô$˜Ÿ	™	ô ô"˜2Ÿ9™9ô ô 
�R—Y‘Yô 
ôÐ.ô ôF@�b—i‘iô @ð$ ô˜oó ó ðð" ôDBÐ+ó DBó ðDBôN"4�b—i‘iô "4ñJ ðñô
`
Ð4ó `
óð
`
ñF ðñôK
Ð%<ó K
óðK
ò\ s�r-   