ó
    ˆŽ¡jÚê  ã                   óÚ  • S SK Jr  S SKJr  S SKJr  S SKrS SKJr  SSKJ	r
  SSKJr  SS	KJr  SS
KJr  SSKJr  SSKJrJrJrJrJrJr  SSKJrJr  SSKJr  SSKJ r   SSK!J"r"J#r#J$r$J%r%J&r&J'r'  SSK(J)r)  SSK*J+r+J,r,  SSK-J.r.J/r/J0r0  SSK1J2r2J3r3J4r4  \&Rj                  " \65      r7 " S S\Rp                  5      r9 " S S\Rp                  5      r:\$ " S S\5      5       r; SLS\Rp                  S\Rx                  S\Rx                  S \Rx                  S!\Rx                  S-  S"\=S#\=4S$ jjr> " S% S&\Rp                  5      r? " S' S(\Rp                  5      r@ " S) S*\5      rA " S+ S,\Rp                  5      rB " S- S.\;5      rC " S/ S0\Rp                  5      rD " S1 S2\Rp                  5      rE " S3 S4\Rp                  5      rF " S5 S6\Rp                  5      rG " S7 S8\Rp                  5      rH " S9 S:\5      rI " S; S<\Rp                  5      rJ " S= S>\;5      rK\$" S?S@9\ " SA SB\"5      5       5       rL\$" SCS@9 " SD SE\;5      5       rM\$\ " SF SG\5      5       5       rN\$" SHS@9 " SI SJ\;\5      5       rO/ SKQrPg)Mé    )ÚCallable)Ú	dataclass)ÚAnyN)Únné   )Úinitialization)ÚACT2FN)ÚGenerationMixin)Úcreate_bidirectional_mask)ÚGradientCheckpointingLayer)ÚBaseModelOutputÚ)BaseModelOutputWithPastAndCrossAttentionsÚBaseModelOutputWithPoolingÚ,BaseModelOutputWithPoolingAndCrossAttentionsÚCausalLMOutputWithPastÚSeq2SeqLMOutput)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)Úapply_chunking_to_forward)ÚModelOutputÚTransformersKwargsÚauto_docstringÚcan_return_tupleÚloggingÚ	torch_int)Úmerge_with_config_defaults)ÚOutputRecorderÚcapture_outputsé   )Ú	AutoModelÚAutoModelForCausalLMÚAutoModelForSeq2SeqLMé   )ÚInstructBlipVideoConfigÚInstructBlipVideoQFormerConfigÚInstructBlipVideoVisionConfigc                   ó´   ^ • \ rS rSrS\4U 4S jjrS\R                  S\S\S\R                  4S jr	SS	\R                  S
\S\R                  4S jjrSrU =r$ )Ú!InstructBlipVideoVisionEmbeddingsé;   Úconfigc                 ór  >• [         TU ]  5         Xl        UR                  U l        UR
                  U l        UR                  U l        [        R                  " [        R                  " SSU R                  5      5      U l        [        R                  " SU R                  U R                  U R                  S9U l        U R
                  U R                  -  S-  U l        U R                  S-   U l        [        R                  " [        R                  " SU R                  U R                  5      5      U l        g )Nr$   r   )Úin_channelsÚout_channelsÚkernel_sizeÚstrider    )ÚsuperÚ__init__r+   Úhidden_sizeÚ	embed_dimÚ
image_sizeÚ
patch_sizer   Ú	ParameterÚtorchÚrandnÚclass_embeddingÚConv2dÚpatch_embeddingÚnum_patchesÚnum_positionsÚposition_embedding©Úselfr+   Ú	__class__s     €ÚÎC:\Users\johnn\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.13_qbz5n2kfra8p0\LocalCache\local-packages\Python313\site-packages\transformers/models/instructblipvideo/modeling_instructblipvideo.pyr2   Ú*InstructBlipVideoVisionEmbeddings.__init__<   sß   ø€ Ü‰ÑÔØŒØ×+Ñ+ˆŒØ ×+Ñ+ˆŒØ ×+Ñ+ˆŒä!Ÿ|š|¬E¯KªK¸¸1¸d¿n¹nÓ,MÓNˆÔä!ŸyšyØ¨¯©ÀDÇOÁOÐ\`×\kÑ\kñ 
ˆÔð !ŸO™O¨t¯©Ñ>À1ÑDˆÔØ!×-Ñ-°Ñ1ˆÔä"$§,¢,¬u¯{ª{¸1¸d×>PÑ>PÐRV×R`ÑR`Ó/aÓ"bˆÕó    Ú
embeddingsÚheightÚwidthÚreturnc                 óœ  • UR                   S   S-
  nU R                  R                   S   S-
  n[        R                  R	                  5       (       d  XE:X  a  X#:X  a  U R                  $ U R                  SS2SS24   nU R                  SS2SS24   nUR                   S   nX R
                  -  n	X0R
                  -  n
[        US-  5      nUR                  SX»U5      nUR                  SSSS5      n[        R                  R                  UXš4SS	S
9nUR                  SSSS5      R                  SSU5      n[        R                  " Xg4SS9$ )aÐ  
This method allows to interpolate the pre-trained position encodings, to be able to use the model on higher resolution
images. This method is also adapted to support torch.jit tracing.

Adapted from:
- https://github.com/facebookresearch/dino/blob/de9ee3df6cf39fac952ab558447af1fa1365362a/vision_transformer.py#L174-L194, and
- https://github.com/facebookresearch/dinov2/blob/e1277af2ba9496fbadf7aec6eba56e8d882d1e35/dinov2/models/vision_transformer.py#L179-L211
r$   Néÿÿÿÿg      à?r   r   r    ÚbicubicF)ÚsizeÚmodeÚalign_corners©Údim)Úshaper?   r8   ÚjitÚ
is_tracingr6   r   ÚreshapeÚpermuter   Ú
functionalÚinterpolateÚviewÚcat)rA   rF   rG   rH   r=   r>   Úclass_pos_embedÚpatch_pos_embedrQ   Ú
new_heightÚ	new_widthÚsqrt_num_positionss               rC   Úinterpolate_pos_encodingÚ:InstructBlipVideoVisionEmbeddings.interpolate_pos_encodingN   sS  € ð !×&Ñ& qÑ)¨AÑ-ˆØ×/Ñ/×5Ñ5°aÑ8¸1Ñ<ˆô �y‰y×#Ñ#×%Ñ%¨+Ó*FÈ6Ë?Ø×*Ñ*Ð*à×1Ñ1²!°R°a°R°%Ñ8ˆØ×1Ñ1²!°Q±R°%Ñ8ˆà×Ñ˜rÑ"ˆàŸ™Ñ.ˆ
ØŸ_™_Ñ,ˆ	ä& }°cÑ'9Ó:ÐØ)×1Ñ1°!Ð5GÐ]`ÓaˆØ)×1Ñ1°!°Q¸¸1Ó=ˆäŸ-™-×3Ñ3ØØÐ(ØØð	 4ð 
ˆð *×1Ñ1°!°Q¸¸1Ó=×BÑBÀ1ÀbÈ#ÓNˆä�yŠy˜/Ð;ÀÑCÐCrE   Úpixel_valuesr`   c                 ó  • UR                   u  p4pVU R                  R                  R                  nU R                  UR	                  US95      nUR                  S5      R                  SS5      nU R                  R                  USS5      R	                  U5      n	[        R                  " X˜/SS9n
U(       a  U R                  X¥U5      nOU R                  nX«S S 2S U
R                  S5      2S S 24   R	                  U5      -   n
U
$ )N)Údtyper    r$   rK   rP   )rR   r<   Úweightrd   ÚtoÚflattenÚ	transposer:   Úexpandr8   rZ   r`   r?   rM   )rA   rb   r`   Ú
batch_sizeÚ_rG   rH   Útarget_dtypeÚpatch_embedsÚclass_embedsrF   r?   s               rC   ÚforwardÚ)InstructBlipVideoVisionEmbeddings.forwardv   sñ   € Ø'3×'9Ñ'9Ñ$ˆ
�vØ×+Ñ+×2Ñ2×8Ñ8ˆØ×+Ñ+¨L¯O©OÀ,¨OÐ,OÓPˆØ#×+Ñ+¨AÓ.×8Ñ8¸¸AÓ>ˆØ×+Ñ+×2Ñ2°:¸qÀ"ÓE×HÑHÈÓVˆÜ—Y’Y Ð;ÀÑCˆ
Þ#Ø!%×!>Ñ!>¸zÐSXÓ!YÑà!%×!8Ñ!8ÐØ²QÐ8L¸*¿/¹/È!Ó:LÐ8LÊaÐ5OÑ"P×"SÑ"SÐT`Ó"aÑaˆ
ØÐrE   )	r:   r+   r4   r5   r=   r>   r<   r6   r?   ©F)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r'   r2   r8   ÚTensorÚintr`   ÚFloatTensorÚboolro   Ú__static_attributes__Ú__classcell__©rB   s   @rC   r)   r)   ;   sr   ø† ðcÐ<÷ cð$&D°5·<±<ð &DÈð &DÐUXð &DÐ]b×]iÑ]iô &DñP E×$5Ñ$5ð ÐQUð Ðbg×bnÑbn÷ ó rE   r)   c                   ó>   ^ • \ rS rSrSrU 4S jr    SS jrSrU =r$ )Ú"InstructBlipVideoQFormerEmbeddingsé…   z;Construct the embeddings from word and position embeddings.c                 ó4  >• [         TU ]  5         [        R                  " UR                  UR
                  UR                  S9U l        [        R                  " UR                  UR
                  5      U l	        [        R                  " UR
                  UR                  S9U l        [        R                  " UR                  5      U l        [        R                   " ["        R$                  " UR                  5      R'                  S5      SS9U l        Xl        g )N)Úpadding_idx©Úeps©r$   rK   F)Ú
persistent)r1   r2   r   Ú	EmbeddingÚ
vocab_sizer3   Úpad_token_idÚword_embeddingsÚmax_position_embeddingsÚposition_embeddingsÚ	LayerNormÚlayer_norm_epsÚ	layernormÚDropoutÚhidden_dropout_probÚdropoutÚBufferr8   Úarangeri   Úposition_idsr+   r@   s     €rC   r2   Ú+InstructBlipVideoQFormerEmbeddings.__init__ˆ   sÀ   ø€ Ü‰ÑÔÜ!Ÿ|š|¨F×,=Ñ,=¸v×?QÑ?QÐ_e×_rÑ_rÑsˆÔÜ#%§<¢<°×0NÑ0NÐPV×PbÑPbÓ#cˆÔ äŸš f×&8Ñ&8¸f×>SÑ>SÑTˆŒÜ—z’z &×"<Ñ"<Ó=ˆŒô ŸIšI¤e§l¢l°6×3QÑ3QÓ&R×&YÑ&YÐZaÓ&bÐotÑuˆÔà�rE   c                 óæ  • Ub  UR                  5       S   nOSnUc%  U R                  S S 2XEU-   24   R                  5       nUbY  U R                  U5      nU R	                  UR                  UR                  5      5      nXg-   nUb  [        R                  " X64SS9nOUnUR                  U R                  R                  R                  5      nU R                  U5      nU R                  U5      nU$ )Nr$   r   rP   )rM   r”   Úcloner‰   r‹   rf   Údevicer8   rZ   rŽ   re   rd   r‘   )rA   Ú	input_idsr”   Úquery_embedsÚpast_key_values_lengthÚ
seq_lengthrF   r‹   s           rC   ro   Ú*InstructBlipVideoQFormerEmbeddings.forward•   së   € ð Ñ Ø"Ÿ™Ó)¨!Ñ,‰JàˆJàÑØ×,Ñ,ªQÐ0FÐVlÑIlÐ0lÐ-lÑm×sÑsÓuˆLàÑ Ø×-Ñ-¨iÓ8ˆJà"&×":Ñ":¸<¿?¹?È:×K\ÑK\Ó;]Ó"^ÐØ#Ñ9ˆJàÑ'Ü"ŸYšY¨Ð'AÀqÑI�
øà%ˆJà—]‘] 4§>¡>×#8Ñ#8×#>Ñ#>Ó?ˆ
Ø—^‘^ JÓ/ˆ
Ø—\‘\ *Ó-ˆ
ØÐrE   )r+   r‘   rŽ   r‹   r”   r‰   ©NNNr   )	rr   rs   rt   ru   Ú__doc__r2   ro   rz   r{   r|   s   @rC   r~   r~   …   s#   ø† ÙEõð ØØØ ÷ò rE   r~   c                   ó„   ^ • \ rS rSr% \\S'   SrSrSrSr	Sr
SrSrSr/ SQr\R                   " 5       U 4S j5       rSrU =r$ )	Ú InstructBlipVideoPreTrainedModeléµ   r+   Úblip)ÚvideoÚtextT)r~   ÚInstructBlipVideoAttentionÚInstructBlipVideoEncoderLayerÚInstructBlipVideoQFormerLayerÚ*InstructBlipVideoQFormerMultiHeadAttentionÚ"InstructBlipVideoQFormerSelfOutputc                 óV  >• [         TU ]  U5        U R                  R                  n[	        U[
        5      (       aA  [        R                  " UR                  SUS9  [        R                  " UR                  SUS9  g[	        U[        [        45      (       a!  [        R                  " UR                  5        g[	        U[        5      (       a\  [        R                  " UR                   ["        R$                  " UR                   R&                  S   5      R)                  S5      5        gg)zInitialize the weightsç        )ÚmeanÚstdrK   r„   N)r1   Ú_init_weightsr+   Úinitializer_rangeÚ
isinstancer)   ÚinitÚtrunc_normal_r?   r:   Ú)InstructBlipVideoForConditionalGenerationÚInstructBlipVideoModelÚzeros_Úquery_tokensr~   Úcopy_r”   r8   r“   rR   ri   )rA   ÚmoduleÚfactorrB   s      €rC   r¯   Ú.InstructBlipVideoPreTrainedModel._init_weightsË   sÔ   ø€ ô 	‰Ñ˜fÔ%Ø—‘×.Ñ.ˆÜ�fÔ?×@Ñ@Ü×Ò˜v×8Ñ8¸sÈÒOÜ×Ò˜v×5Ñ5¸CÀVÓLÜ˜Ô!JÔLbÐ c×dÑdÜ�KŠK˜×+Ñ+Õ,Ü˜Ô B×CÑCÜ�JŠJ�v×*Ñ*¬E¯LªL¸×9LÑ9L×9RÑ9RÐSUÑ9VÓ,W×,^Ñ,^Ð_fÓ,gÕhð DrE   © )rr   rs   rt   ru   r%   Ú__annotations__Úbase_model_prefixÚinput_modalitiesÚsupports_gradient_checkpointingÚ_supports_attention_backendÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_can_compile_fullgraphÚ_no_split_modulesr8   Úno_gradr¯   rz   r{   r|   s   @rC   r¡   r¡   µ   s\   ø‡ à#Ó#ØÐØ(ÐØ&*Ð#Ø"&ÐØÐØ€NØÐà!ÐòÐð ‡]‚]ƒ_ô
ió ö
irE   r¡   r¹   ÚqueryÚkeyÚvalueÚattention_maskÚscalingr‘   c                 ó`  • [         R                  " XR                  SS5      5      U-  nUb  X„-   n[        R                  R                  USS9n[        R                  R                  X†U R                  S9n[         R                  " Xƒ5      n	U	R                  SS5      R                  5       n	X˜4$ )NrK   éþÿÿÿrP   )ÚpÚtrainingr$   r    )	r8   Úmatmulrh   r   rW   Úsoftmaxr‘   rÐ   Ú
contiguous)
r¹   rÈ   rÉ   rÊ   rË   rÌ   r‘   ÚkwargsÚattn_weightsÚattn_outputs
             rC   Úeager_attention_forwardr×   Ú   s˜   € ô —<’< §}¡}°R¸Ó'<Ó=ÀÑG€LØÑ!Ø#Ñ4ˆä—=‘=×(Ñ(¨¸2Ð(Ð>€LÜ—=‘=×(Ñ(¨È6Ï?É?Ð(Ð[€Lä—,’,˜|Ó3€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€KàÐ$Ð$rE   c                   óÖ   ^ • \ rS rSrSrU 4S jrS\R                  S\S\4S jr	S\R                  S	\
\R                  \R                  S
-  \
\R                     S
-  4   4S jrSrU =r$ )r¦   éñ   z=Multi-headed attention from 'Attention Is All You Need' paperc                 ó  >• [         TU ]  5         Xl        UR                  U l        UR
                  U l        U R                  U R                  -  U l        U R                  U R                  -  U R                  :w  a&  [        SU R                   SU R                   S35      eU R                  S-  U l	        SU l
        UR                  U l        [        R                  " U R                  SU R                  -  SS9U l        UR                  (       ai  [        R                   " ["        R$                  " U R                  5      5      n[        R                   " ["        R$                  " U R                  5      5      nOS nS nUbQ  ["        R&                  " U["        R(                  " USS9U45      n[        R                   " U5      U R                  l        [        R                  " U R                  U R                  5      U l        g )	Nz;embed_dim must be divisible by num_heads (got `embed_dim`: z and `num_heads`: z).ç      à¿Fr   )Úbias)Úrequires_grad)r1   r2   r+   r3   r4   Únum_attention_headsÚ	num_headsÚhead_dimÚ
ValueErrorÚscaleÚ	is_causalÚattention_dropoutr   ÚLinearÚqkvÚqkv_biasr7   r8   ÚzerosrZ   Ú
zeros_likerÜ   Ú
projection)rA   r+   Úq_biasÚv_biasrç   rB   s        €rC   r2   Ú#InstructBlipVideoAttention.__init__ô   ss  ø€ Ü‰ÑÔØŒØ×+Ñ+ˆŒØ×3Ñ3ˆŒØŸ™¨$¯.©.Ñ8ˆŒØ�=‰=˜4Ÿ>™>Ñ)¨T¯^©^Ó;ÜØMÈdÏnÉnÐM]ð ^Ø—N‘NÐ# 2ð'óð ð —]‘] DÑ(ˆŒ
ØˆŒØ!'×!9Ñ!9ˆÔô —9’9˜TŸ^™^¨Q°·±Ñ-?ÀeÑLˆŒà�?�?Ü—\’\¤%§+¢+¨d¯n©nÓ"=Ó>ˆFÜ—\’\¤%§+¢+¨d¯n©nÓ"=Ó>‰FàˆFØˆFàÑÜ—y’y &¬%×*:Ò*:¸6ÐQVÑ*WÐY_Ð!`ÓaˆHÜŸLšL¨Ó2ˆD�H‰HŒMäŸ)š) D§N¡N°D·N±NÓCˆ�rE   ÚtensorÚseq_lenÚbszc                 óŒ   • UR                  X2U R                  U R                  5      R                  SS5      R	                  5       $ )Nr$   r    )rY   rß   rà   rh   rÓ   )rA   rî   rï   rð   s       rC   Ú_shapeÚ!InstructBlipVideoAttention._shape  s5   € Ø�{‰{˜3¨¯©¸¿¹ÓG×QÑQÐRSÐUVÓW×bÑbÓdÐdrE   Úhidden_statesrI   Nc                 ó  • UR                  5       u  p4nU R                  U5      nUR                  X4SU R                  XPR                  -  5      R	                  SSSSS5      nUS   US   US   p˜n[
        R                  " U R                  R                  [        5      n
U
" U UUU	4SU R                  (       d  SOU R                  U R                  S.UD6u  p¼UR                  X4S	5      R                  5       nU R                  U5      nX¼4$ )
z#Input shape: Batch x Time x Channelr   r    r   r$   é   Nr¬   )rË   r‘   rÌ   rK   )rM   ræ   rU   rß   rV   r   Úget_interfacer+   Ú_attn_implementationr×   rÐ   rä   râ   rÓ   rê   )rA   rô   rÔ   rð   Útgt_lenr4   Ú	mixed_qkvÚquery_statesÚ
key_statesÚvalue_statesÚattention_interfacerÖ   rÕ   s                rC   ro   Ú"InstructBlipVideoAttention.forward  s  € ð #0×"4Ñ"4Ó"6Ñˆ�ià—H‘H˜]Ó+ˆ	à×%Ñ% c°A°t·~±~Ày×TbÑTbÑGbÓc×kÑkØˆq�!�Q˜ó
ˆ	ð 2;¸1±¸yÈ¹|ÈYÐWXÉ\ ,ˆä(?×(MÒ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØð		%
ð
  Ø#Ÿ}Ÿ}‘C°$×2HÑ2HØ—J‘Jñ	%
ð ñ	%
Ñ!ˆð "×)Ñ)¨#¸Ó;×FÑFÓHˆØ—o‘o kÓ2ˆàÐ(Ð(rE   )	rä   r+   r4   rà   rã   rß   rê   ræ   râ   )rr   rs   rt   ru   rŸ   r2   r8   rv   rw   rò   Útuplero   rz   r{   r|   s   @rC   r¦   r¦   ñ   su   ø† ÙGõDð>e˜UŸ\™\ð e°Cð e¸cô eð")à—|‘|ð")ð 
ˆu�|‰|˜UŸ\™\¨DÑ0°%¸¿¹Ñ2EÈÑ2LÐLÑ	M÷	")ò ")rE   r¦   c                   ób   ^ • \ rS rSrU 4S jrS\R                  S\R                  4S jrSrU =r	$ )ÚInstructBlipVideoMLPi;  c                 ó  >• [         TU ]  5         Xl        [        UR                     U l        [        R                  " UR                  UR                  5      U l
        [        R                  " UR                  UR                  5      U l        g ©N)r1   r2   r+   r	   Ú
hidden_actÚactivation_fnr   rå   r3   Úintermediate_sizeÚfc1Úfc2r@   s     €rC   r2   ÚInstructBlipVideoMLP.__init__<  sb   ø€ Ü‰ÑÔØŒÜ# F×$5Ñ$5Ñ6ˆÔÜ—9’9˜V×/Ñ/°×1IÑ1IÓJˆŒÜ—9’9˜V×5Ñ5°v×7IÑ7IÓJˆ�rE   rô   rI   c                 ól   • U R                  U5      nU R                  U5      nU R                  U5      nU$ r  )r  r  r	  ©rA   rô   s     rC   ro   ÚInstructBlipVideoMLP.forwardC  s4   € ØŸ™ Ó/ˆØ×*Ñ*¨=Ó9ˆØŸ™ Ó/ˆØÐrE   )r  r+   r  r	  ©
rr   rs   rt   ru   r2   r8   rv   ro   rz   r{   r|   s   @rC   r  r  ;  s)   ø† õKð U§\¡\ð °e·l±l÷ ò rE   r  c                   ó~   ^ • \ rS rSrS\4U 4S jjr\S\R                  S\	\
   S\R                  4S j5       rSrU =r$ )	r§   iJ  r+   c                 ó<  >• [         TU ]  5         UR                  U l        [	        U5      U l        [        R                  " U R                  UR                  S9U l	        [        U5      U l        [        R                  " U R                  UR                  S9U l        g ©Nr‚   )r1   r2   r3   r4   r¦   Ú	self_attnr   rŒ   r�   Úlayer_norm1r  ÚmlpÚlayer_norm2r@   s     €rC   r2   Ú&InstructBlipVideoEncoderLayer.__init__K  sm   ø€ Ü‰ÑÔØ×+Ñ+ˆŒÜ3°FÓ;ˆŒÜŸ<š<¨¯©¸F×<QÑ<QÑRˆÔÜ'¨Ó/ˆŒÜŸ<š<¨¯©¸F×<QÑ<QÑRˆÕrE   rô   rÔ   rI   c                 ó°   • UnU R                  U5      nU R                  " SSU0UD6u  pX-   nUnU R                  U5      nU R                  U5      nX-   nU$ )Nrô   r¼   )r  r  r  r  )rA   rô   rÔ   Úresidualrk   s        rC   ro   Ú%InstructBlipVideoEncoderLayer.forwardS  su   € ð !ˆà×(Ñ(¨Ó7ˆØŸ>š>ñ 
Ø'ð
àñ
Ñˆð &Ñ0ˆØ ˆØ×(Ñ(¨Ó7ˆØŸ™ Ó/ˆà%Ñ0ˆàÐrE   )r4   r  r  r  r  )rr   rs   rt   ru   r%   r2   r   r8   rv   r   r   rx   ro   rz   r{   r|   s   @rC   r§   r§   J  sR   ø† ðSÐ6÷ Sð ðà—|‘|ðð Ð+Ñ,ðð 
×	Ñ	ó	ó örE   r§   c                   ó\   ^ • \ rS rSrSrS\4U 4S jjr\S\\	   S\
\-  4S j5       rSrU =r$ )	ÚInstructBlipVideoEncoderij  a
  
Transformer encoder consisting of `config.num_hidden_layers` self attention layers. Each layer is a
[`InstructBlipVideoEncoderLayer`].

Args:
    config (`InstructBlipVideoConfig`):
        The corresponding vision configuration for the `InstructBlipVideoEncoder`.
r+   c                 óÖ   >• [         TU ]  5         Xl        [        R                  " [        UR                  5       Vs/ s H  n[        U5      PM     sn5      U l        SU l	        g s  snf ©NF)
r1   r2   r+   r   Ú
ModuleListÚrangeÚnum_hidden_layersr§   ÚlayersÚgradient_checkpointing)rA   r+   rk   rB   s      €rC   r2   Ú!InstructBlipVideoEncoder.__init__t  sU   ø€ Ü‰ÑÔØŒÜ—m’mÔTYÐZ`×ZrÑZrÔTsÓ$tÒTsÈqÔ%BÀ6Ö%JÑTsÑ$tÓuˆŒØ&+ˆÕ#ùò %uó   ½A&rÔ   rI   c                 óP   • UnU R                    H  nU" U40 UD6nM     [        US9$ )N©Úlast_hidden_state)r!  r   )rA   Úinputs_embedsrÔ   rô   Úencoder_layers        rC   ro   Ú InstructBlipVideoEncoder.forwardz  s9   € ð &ˆØ!Ÿ[œ[ˆMÙ)ØñàñŠMñ )ô °Ñ?Ð?rE   )r+   r"  r!  )rr   rs   rt   ru   rŸ   r%   r2   r   r   r   r   r   ro   rz   r{   r|   s   @rC   r  r  j  sL   ø† ñð,Ð6÷ ,ð ð@ð Ð+Ñ,ð@ð 
�Ñ	 ó	@ó ö@rE   r  c                   óÂ   ^ • \ rS rSr% SrSr\\S'   \\	S.r
S\4U 4S jjr\\" SS9\  SS\R                   S	-  S
\S\\   S\\-  4S jj5       5       5       rS rSrU =r$ )ÚInstructBlipVideoVisionModeliŠ  rb   r¤   r+   )rô   Ú
attentionsc                 óî   >• [         TU ]  U5        Xl        UR                  n[	        U5      U l        [        U5      U l        [        R                  " X!R                  S9U l        U R                  5         g r  )r1   r2   r+   r3   r)   rF   r  Úencoderr   rŒ   r�   Úpost_layernormÚ	post_init)rA   r+   r4   rB   s      €rC   r2   Ú%InstructBlipVideoVisionModel.__init__“  sY   ø€ Ü‰Ñ˜Ô ØŒØ×&Ñ&ˆ	ä;¸FÓCˆŒÜ/°Ó7ˆŒÜ Ÿlšl¨9×:OÑ:OÑPˆÔà�‰ÕrE   F)Útie_last_hidden_statesNr`   rÔ   rI   c                 óì   • Uc  [        S5      eU R                  XS9nU R                  " SSU0UD6nUR                  nU R	                  U5      nUS S 2SS S 24   nU R	                  U5      n[        UUS9$ )Nz You have to specify pixel_values)r`   r(  r   ©r'  Úpooler_outputr¼   )rá   rF   r/  r'  r0  r   )rA   rb   r`   rÔ   rô   Úencoder_outputsr'  Úpooled_outputs           rC   ro   Ú$InstructBlipVideoVisionModel.forwardž  s—   € ð ÑÜÐ?Ó@Ð@àŸ™¨˜Ðhˆà+/¯<ª<ñ ,
Ø'ð,
àñ,
ˆð
 ,×=Ñ=ÐØ ×/Ñ/Ð0AÓBÐà)ª!¨Q²¨'Ñ2ˆØ×+Ñ+¨MÓ:ˆä)Ø/Ø'ñ
ð 	
rE   c                 ó   • U R                   $ r  )rF   ©rA   s    rC   Úget_input_embeddingsÚ1InstructBlipVideoVisionModel.get_input_embeddings¼  s   € Ø�‰ÐrE   )r+   rF   r/  r0  r  )rr   rs   rt   ru   Úmain_input_namer¿   r'   r½   r§   r¦   Ú_can_record_outputsr2   r   r   r   r8   rx   ry   r   r   r   r   ro   r<  rz   r{   r|   s   @rC   r,  r,  Š  s§   ø‡ Ø$€OØÐØ)Ó)à6Ø0ñÐð
	Ð<÷ 	ð  Ù¨EÑ2Øð 26Ø).ñ
à×'Ñ'¨$Ñ.ð
ð #'ð
ð Ð+Ñ,ð	
ð
 
Ð+Ñ	+ô
ó ó 3ó  ð
÷6ð rE   r,  c                   óJ   ^ • \ rS rSrSU 4S jjr   SS\\   4S jjrSrU =r	$ )r©   iÀ  c                 óÆ  >• [         TU ]  5         Xl        UR                  UR                  -  S:w  a5  [        US5      (       d$  [        SUR                  UR                  4-  5      eUR                  U l        [        UR                  UR                  -  5      U l        U R                  U R                  -  U l	        U R                  S-  U l
        SU l        UR                  U l        [        R                  " UR                  U R                  5      U l        U(       aa  [        R                  " UR"                  U R                  5      U l        [        R                  " UR"                  U R                  5      U l        g [        R                  " UR                  U R                  5      U l        [        R                  " UR                  U R                  5      U l        g )Nr   Úembedding_sizezLThe hidden size (%d) is not a multiple of the number of attention heads (%d)rÛ   F)r1   r2   r+   r3   rÞ   Úhasattrrá   rw   Úattention_head_sizeÚall_head_sizerÌ   rã   Úattention_probs_dropout_probrä   r   rå   rÈ   Úencoder_hidden_sizerÉ   rÊ   ©rA   r+   Úis_cross_attentionrB   s      €rC   r2   Ú3InstructBlipVideoQFormerMultiHeadAttention.__init__Á  sk  ø€ Ü‰ÑÔØŒØ×Ñ × :Ñ :Ñ:¸aÓ?ÌÐPVÐXh×HiÑHiÜØ^Ø×%Ñ% v×'AÑ'AÐBñCóð ð
 $*×#=Ñ#=ˆÔ Ü#& v×'9Ñ'9¸F×<VÑ<VÑ'VÓ#WˆÔ Ø!×5Ñ5¸×8PÑ8PÑPˆÔØ×/Ñ/°Ñ5ˆŒØˆŒØ!'×!DÑ!DˆÔä—Y’Y˜v×1Ñ1°4×3EÑ3EÓFˆŒ
ÞÜ—y’y ×!;Ñ!;¸T×=OÑ=OÓPˆDŒHÜŸš 6×#=Ñ#=¸t×?QÑ?QÓRˆD�Jä—y’y ×!3Ñ!3°T×5GÑ5GÓHˆDŒHÜŸš 6×#5Ñ#5°t×7IÑ7IÓJˆD�JrE   rÔ   c                 ó  • US LnUR                   S S n/ UQSPU R                  P7nU(       a  Un	UnOUn	/ U	R                   S S QSPU R                  P7n
U R                  U5      R                  U5      R	                  SS5      nU R                  U	5      R                  U
5      R	                  SS5      nU R                  U	5      R                  U
5      R	                  SS5      n[        R                  " U R                  R                  [        5      nU" U UUUU4U R                  (       d  SOU R                  U R                  S.UD6u  nnUR                  " / UR                   S S QSP76 R!                  5       nUU4$ )NrK   r$   r    r¬   )r‘   rÌ   )rR   rD  rÈ   rY   rh   rÉ   rÊ   r   r÷   r+   rø   r×   rÐ   rä   rÌ   rU   rÓ   )rA   rô   rË   Úencoder_hidden_statesÚencoder_attention_maskrÔ   rI  Úinput_shapeÚhidden_shapeÚcurrent_statesÚkv_shapeÚquery_layerÚ	key_layerÚvalue_layerrþ   rÖ   rÕ   s                    rC   ro   Ú2InstructBlipVideoQFormerMultiHeadAttention.forwardÙ  sŒ  € ð 3¸$Ð>Ðà#×)Ñ)¨#¨2Ð.ˆØC˜ÐC bÐC¨$×*BÑ*BÑCˆæØ2ˆNØ3‰Nà*ˆNàM�^×)Ñ)¨#¨2Ð.ÐM°ÐM°D×4LÑ4LÑMˆØ—j‘j Ó/×4Ñ4°\ÓB×LÑLÈQÐPQÓRˆØ—H‘H˜^Ó,×1Ñ1°(Ó;×EÑEÀaÈÓKˆ	Ø—j‘j Ó0×5Ñ5°hÓ?×IÑIÈ!ÈQÓOˆä(?×(MÒ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØð	%
ð  $Ÿ}Ÿ}‘C°$×2HÑ2HØ—L‘Lñ	%
ð ñ	%
Ñ!ˆ�\ð "×)Ò)ÐE¨;×+<Ñ+<¸R¸aÐ+@ÐEÀ"ÒE×PÑPÓRˆØ˜LÐ(Ð(rE   )
rE  rä   rD  r+   rã   rÉ   rÞ   rÈ   rÌ   rÊ   rq   ©NNN)
rr   rs   rt   ru   r2   r   r   ro   rz   r{   r|   s   @rC   r©   r©   À  s.   ø† ÷Kð6 Ø"Ø#ñ,)ð Ð+Ñ,÷,)ó ,)rE   r©   c                   óz   ^ • \ rS rSrU 4S jrS\R                  S\R                  S\R                  4S jrSrU =r	$ )rª   i  c                 ó(  >• [         TU ]  5         [        R                  " UR                  UR                  5      U l        [        R                  " UR                  UR                  S9U l        [        R                  " UR                  5      U l
        g r  )r1   r2   r   rå   r3   ÚdenserŒ   r�   r�   r�   r‘   r@   s     €rC   r2   Ú+InstructBlipVideoQFormerSelfOutput.__init__	  s`   ø€ Ü‰ÑÔÜ—Y’Y˜v×1Ñ1°6×3EÑ3EÓFˆŒ
ÜŸš f×&8Ñ&8¸f×>SÑ>SÑTˆŒÜ—z’z &×"<Ñ"<Ó=ˆ�rE   rô   Úinput_tensorrI   c                 óp   • U R                  U5      nU R                  U5      nU R                  X-   5      nU$ r  ©rY  r‘   rŒ   ©rA   rô   r[  s      rC   ro   Ú*InstructBlipVideoQFormerSelfOutput.forward  ó5   € ØŸ
™
 =Ó1ˆØŸ™ ]Ó3ˆØŸ™ }Ñ'CÓDˆØÐrE   ©rŒ   rY  r‘   r  r|   s   @rC   rª   rª     ó6   ø† õ>ð U§\¡\ð ÀÇÁð ÐRW×R^ÑR^÷ ò rE   rª   c                   óÔ   ^ • \ rS rSrSU 4S jjr   SS\R                  S\R                  S-  S\R                  S-  S\R                  S-  S\\	   S	\R                  4S
 jjr
SrU =r$ )Ú!InstructBlipVideoQFormerAttentioni  c                 ób   >• [         TU ]  5         [        X5      U l        [	        U5      U l        g r  )r1   r2   r©   Ú	attentionrª   ÚoutputrH  s      €rC   r2   Ú*InstructBlipVideoQFormerAttention.__init__  s&   ø€ Ü‰ÑÔÜCÀFÓ_ˆŒÜ8¸Ó@ˆ�rE   Nrô   rË   rL  rM  rÔ   rI   c                 óZ   • U R                   " SUUUUS.UD6u  pgU R                  Xa5      nU$ )N)rô   rË   rL  rM  r¼   ©rf  rg  )	rA   rô   rË   rL  rM  rÔ   rÖ   rk   Úattention_outputs	            rC   ro   Ú)InstructBlipVideoQFormerAttention.forward  sF   € ð Ÿšð 
Ø'Ø)Ø"7Ø#9ñ	
ð
 ñ
‰ˆð  Ÿ;™; {ÓBÐØÐrE   rj  rq   rV  )rr   rs   rt   ru   r2   r8   rv   rx   r   r   ro   rz   r{   r|   s   @rC   rd  rd    s‡   ø† ÷Að 48Ø:>Ø;?ñ à—|‘|ð ð ×)Ñ)¨DÑ0ð ð  %×0Ñ0°4Ñ7ð	 ð
 !&× 1Ñ 1°DÑ 8ð ð Ð+Ñ,ð ð 
�‰÷ ó  rE   rd  c                   ób   ^ • \ rS rSrU 4S jrS\R                  S\R                  4S jrSrU =r	$ )Ú$InstructBlipVideoQFormerIntermediatei/  c                 ó  >• [         TU ]  5         [        R                  " UR                  UR
                  5      U l        [        UR                  [        5      (       a  [        UR                     U l        g UR                  U l        g r  )r1   r2   r   rå   r3   r  rY  r±   r  Ústrr	   Úintermediate_act_fnr@   s     €rC   r2   Ú-InstructBlipVideoQFormerIntermediate.__init__0  s`   ø€ Ü‰ÑÔÜ—Y’Y˜v×1Ñ1°6×3KÑ3KÓLˆŒ
Ü�f×'Ñ'¬×-Ñ-Ü'-¨f×.?Ñ.?Ñ'@ˆDÕ$à'-×'8Ñ'8ˆDÕ$rE   rô   rI   c                 óJ   • U R                  U5      nU R                  U5      nU$ r  ©rY  rq  r  s     rC   ro   Ú,InstructBlipVideoQFormerIntermediate.forward8  s&   € ØŸ
™
 =Ó1ˆØ×0Ñ0°Ó?ˆØÐrE   rt  r  r|   s   @rC   rn  rn  /  s(   ø† õ9ð U§\¡\ð °e·l±l÷ ò rE   rn  c                   óz   ^ • \ rS rSrU 4S jrS\R                  S\R                  S\R                  4S jrSrU =r	$ )ÚInstructBlipVideoQFormerOutputi>  c                 ó(  >• [         TU ]  5         [        R                  " UR                  UR
                  5      U l        [        R                  " UR
                  UR                  S9U l        [        R                  " UR                  5      U l        g r  )r1   r2   r   rå   r  r3   rY  rŒ   r�   r�   r�   r‘   r@   s     €rC   r2   Ú'InstructBlipVideoQFormerOutput.__init__?  s`   ø€ Ü‰ÑÔÜ—Y’Y˜v×7Ñ7¸×9KÑ9KÓLˆŒ
ÜŸš f×&8Ñ&8¸f×>SÑ>SÑTˆŒÜ—z’z &×"<Ñ"<Ó=ˆ�rE   rô   r[  rI   c                 óp   • U R                  U5      nU R                  U5      nU R                  X-   5      nU$ r  r]  r^  s      rC   ro   Ú&InstructBlipVideoQFormerOutput.forwardE  r`  rE   ra  r  r|   s   @rC   rw  rw  >  rb  rE   rw  c                   óT   ^ • \ rS rSrU 4S jr    SS\\   4S jjrS rS r	Sr
U =r$ )	r¨   iL  c                 ó^  >• [         TU ]  5         UR                  U l        SU l        [	        U5      U l        X l        X!R                  -  S:X  a  [	        USS9U l        SU l	        OSU l	        [        U5      U l        [        U5      U l        [        U5      U l        [        U5      U l        g )Nr$   r   T)rI  F)r1   r2   Úchunk_size_feed_forwardÚseq_len_dimrd  rf  Ú	layer_idxÚcross_attention_frequencyÚcrossattentionÚhas_cross_attentionrn  Úintermediaterw  rg  Úintermediate_queryÚoutput_query©rA   r+   r€  rB   s      €rC   r2   Ú&InstructBlipVideoQFormerLayer.__init__M  s™   ø€ Ü‰ÑÔØ'-×'EÑ'EˆÔ$ØˆÔÜ:¸6ÓBˆŒà"Œà×7Ñ7Ñ7¸1Ó<Ü"CÀFÐ_cÑ"dˆDÔØ'+ˆDÕ$à',ˆDÔ$ä@ÀÓHˆÔÜ4°VÓ<ˆŒä"FÀvÓ"NˆÔÜ:¸6ÓBˆÕrE   rÔ   c           
      ól  • U R                   " U4SU0UD6nUS:”  aì  US S 2S U2S S 24   nU R                  (       a%  Uc  [        S5      eU R                  " U4UUUS.UD6n[	        U R
                  U R                  U R                  U5      n	UR                  S   U:”  ag  [	        U R                  U R                  U R                  US S 2US 2S S 24   5      R                  U	R                  5      n
[        R                  " Xš/SS9n	U	$ [	        U R                  U R                  U R                  U5      n	U	$ )NrË   r   z>encoder_hidden_states must be given for cross-attention layers)rË   rL  rM  r$   rP   )rf  rƒ  rá   r‚  r   Úfeed_forward_chunk_queryr~  r  rR   Úfeed_forward_chunkrf   r˜   r8   rZ   )rA   rô   rË   rL  rM  Úquery_lengthrÔ   rk  Úquery_attention_outputÚlayer_outputÚlayer_output_texts              rC   ro   Ú%InstructBlipVideoQFormerLayer.forwarda  si  € ð  Ÿ>š>Øñ
à)ð
ð ñ
Ðð ˜!ÓØ%5²a¸¸,¸ÊÐ6IÑ%JÐ"à×'×'Ø(Ñ0Ü$Ð%eÓfÐfØ)-×)<Ò)<Ø*ð*à#1Ø*?Ø+Añ	*ð
 ñ*Ð&ô 5Ø×-Ñ-Ø×,Ñ,Ø× Ñ Ø&ó	ˆLð  ×%Ñ% aÑ(¨<Ó7Ü$=Ø×+Ñ+Ø×0Ñ0Ø×$Ñ$Ø$¢Q¨©²qÐ%8Ñ9ó	%÷
 ‘"�\×(Ñ(Ó)ð "ô  %Ÿyšy¨,Ð)JÐPQÑR�ð Ðô 5Ø×'Ñ'Ø×,Ñ,Ø× Ñ Ø ó	ˆLð ÐrE   c                 óJ   • U R                  U5      nU R                  X!5      nU$ r  )r„  rg  ©rA   rk  Úintermediate_outputrŽ  s       rC   r‹  Ú0InstructBlipVideoQFormerLayer.feed_forward_chunk–  s)   € Ø"×/Ñ/Ð0@ÓAÐØ—{‘{Ð#6ÓIˆØÐrE   c                 óJ   • U R                  U5      nU R                  X!5      nU$ r  )r…  r†  r’  s       rC   rŠ  Ú6InstructBlipVideoQFormerLayer.feed_forward_chunk_query›  s+   € Ø"×5Ñ5Ð6FÓGÐØ×(Ñ(Ð)<ÓOˆØÐrE   )
rf  r~  r‚  rƒ  r„  r…  r€  rg  r†  r  rž   )rr   rs   rt   ru   r2   r   r   ro   r‹  rŠ  rz   r{   r|   s   @rC   r¨   r¨   L  s<   ø† õCð. Ø"Ø#Øñ3ð Ð+Ñ,õ3òj÷
ð rE   r¨   c                   óR   ^ • \ rS rSrU 4S jr\    SS\\   4S jj5       rSr	U =r
$ )ÚInstructBlipVideoQFormerEncoderi¡  c           	      óÖ   >• [         TU ]  5         Xl        [        R                  " [        UR                  5       Vs/ s H  n[        X5      PM     sn5      U l        SU l	        g s  snf r  )
r1   r2   r+   r   r  r  r   r¨   Úlayerr"  r‡  s      €rC   r2   Ú(InstructBlipVideoQFormerEncoder.__init__¢  sY   ø€ Ü‰ÑÔØŒÜ—]’]ÜOTÐU[×UmÑUmÔOnÓoÒOnÀ)Ô*¨6Ö=ÑOnÑoó
ˆŒ
ð ',ˆÕ#ùò pr$  rÔ   c                 óš   • [        U R                  R                  5       H   nU R                  U   nU" UUU4UUS.UD6nM"     [	        US9$ )N)rM  rŒ  r&  )r  r+   r   rš  r   )	rA   rô   rË   rL  rM  rŒ  rÔ   ÚiÚlayer_modules	            rC   ro   Ú'InstructBlipVideoQFormerEncoder.forwardª  sf   € ô �t—{‘{×4Ñ4Ö5ˆAØŸ:™: a™=ˆLá(ØØØ%ðð (>Ø)ñð ñŠMñ 6ô 9Ø+ñ
ð 	
rE   )r+   r"  rš  rž   )rr   rs   rt   ru   r2   r   r   r   ro   rz   r{   r|   s   @rC   r˜  r˜  ¡  s:   ø† õ,ð ð Ø"Ø#Øñ
ð Ð+Ñ,ô
ó ö
rE   r˜  c                   óŠ  ^ • \ rS rSrSrSrSrSrSr\	\
" \SSS9/\
" \SSS9/S.rS	\4U 4S
 jjrS rS r\\\     SS\R*                  S\R,                  S-  S\R*                  S-  S\R.                  S-  S\R,                  S-  S\R,                  S-  S\\   S\\R,                     \-  4S jj5       5       5       rSrU =r$ )ÚInstructBlipVideoQFormerModeliÅ  z†
Querying Transformer (Q-Former), used in InstructBlipVideo. Slightly modified from BLIP-2 as it also takes the
instruction as input.
Tr$   z
.attention)ÚindexÚ
layer_namez.crossattention)rô   r-  Úcross_attentionsr+   c                 ó�   >• [         TU ]  U5        Xl        [        U5      U l        [        U5      U l        U R                  5         g r  )r1   r2   r+   r~   rF   r˜  r/  r1  r@   s     €rC   r2   Ú&InstructBlipVideoQFormerModel.__init__Ú  s7   ø€ Ü‰Ñ˜Ô ØŒä<¸VÓDˆŒä6°vÓ>ˆŒà�‰ÕrE   c                 ó.   • U R                   R                  $ r  ©rF   r‰   r;  s    rC   r<  Ú2InstructBlipVideoQFormerModel.get_input_embeddingsä  s   € Ø�‰×.Ñ.Ð.rE   c                 ó$   • XR                   l        g r  r¨  )rA   rÊ   s     rC   Úset_input_embeddingsÚ2InstructBlipVideoQFormerModel.set_input_embeddingsç  s   € Ø*/�‰Õ'rE   Nr™   rË   r”   rš   rL  rM  rÔ   rI   c                 ój  • Uc  Uc  [        S5      eUb  UR                  S   OSnU R                  UUUS9n	[        U R                  U	US9nUb,  US:”  a  U	SS2SU2SS24   OU	n
[        U R                  U
UUS9nU R
                  " U	4UUUUS.UD6nUR                  nUSS2SSS24   n[        UUS	9$ )
a  
query_embeds (`torch.FloatTensor`  of shape `(batch_size, sequence_length, hidden_size)`):
    Hidden states to be used in the attention computation. If cross-attention,
    will be used for the query (i.e., key and value will use the encoder_hidden_states).
Nz7You have to specify query_embeds when input_ids is Noner$   r   )r™   r”   rš   )r+   r(  rË   )r+   r(  rË   rL  )rË   rL  rM  rŒ  r5  )rá   rR   rF   r   r+   r/  r'  r   )rA   r™   rË   r”   rš   rL  rM  rÔ   rŒ  Úembedding_outputÚquery_embedding_outputr7  Úsequence_outputr8  s                 rC   ro   Ú%InstructBlipVideoQFormerModel.forwardê  s  € ð$ Ñ Ñ!5ÜÐVÓWÐWà0<Ñ0H�|×)Ñ)¨!Ò,ÈaˆàŸ?™?ØØ%Ø%ð +ð 
Ðô 3Ø—;‘;Ø*Ø)ñ
ˆð "Ñ-ð O[Ð]^ÓN^Ð%5²a¸¸,¸ÊÐ6IÒ%JÐdtÐ"Ü%>Ø—{‘{Ø4Ø5Ø&;ñ	&Ð"ð ,0¯<ª<Øð,
à)Ø"7Ø#9Ø%ñ,
ð ñ,
ˆð *×;Ñ;ˆØ'ª¨1ªa¨Ñ0ˆä;Ø-Ø'ñ
ð 	
rE   )r+   rF   r/  )NNNNN)rr   rs   rt   ru   rŸ   rÁ   rÃ   rÂ   rÄ   r¨   r   r©   r?  r&   r2   r<  r«  r   r   r   r8   Ú
LongTensorrx   rv   r   r   r   r   ro   rz   r{   r|   s   @rC   r¡  r¡  Å  sB  ø† ñð
 #'ÐØ€NØÐØÐð 7áÐEÈQÐ[gÑhð
ñ ÐEÈQÐ[lÑmð
ñÐðÐ=÷ ò/ò0ð  ØØð 48Ø04Ø,0Ø:>Ø;?ñ9
à×#Ñ#ð9
ð ×)Ñ)¨DÑ0ð9
ð ×&Ñ&¨Ñ-ð	9
ð
 —l‘l TÑ)ð9
ð  %×0Ñ0°4Ñ7ð9
ð !&× 1Ñ 1°DÑ 8ð9
ð Ð+Ñ,ð9
ð 
ˆu× Ñ Ñ	!Ð$PÑ	Pô9
ó ó ó  ö9
rE   r¡  zV
    Class defining the outputs of [`InstructBlipVideoForConditionalGeneration`].
    )Úcustom_introc                   óÌ   • \ rS rSr% SrSr\\R                     S-  \	S'   Sr
\\R                     S-  \	S'   Sr\S-  \	S'   Sr\S-  \	S'   Sr\\-  S-  \	S'   S	\\   4S
 jrSrg)Ú4InstructBlipVideoForConditionalGenerationModelOutputi)  a~  
loss (`torch.FloatTensor`, *optional*, returned when `labels` is provided, `torch.FloatTensor` of shape `(1,)`):
    Language modeling loss from the language model.
logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`):
    Prediction scores of the language modeling head of the language model.
vision_outputs (`BaseModelOutputWithPooling`):
    Outputs of the vision encoder.
qformer_outputs (`BaseModelOutputWithPoolingAndCrossAttentions`):
    Outputs of the Q-Former (Querying Transformer).
language_model_outputs (`CausalLMOutputWithPast` or `Seq2SeqLMOutput`):
    Outputs of the language model.
NÚlossÚlogitsÚvision_outputsÚqformer_outputsÚlanguage_model_outputsrI   c                 óJ   ^ • [        U 4S jT R                  5        5       5      $ )Nc              3   ón   >#   • U  H*  nUS ;  a  TU   O[        TU5      R                  5       v •  M,     g7f)©r¸  r¹  rº  N)ÚgetattrÚto_tuple)Ú.0ÚkrA   s     €rC   Ú	<genexpr>ÚPInstructBlipVideoForConditionalGenerationModelOutput.to_tuple.<locals>.<genexpr>D  sC   øé € ð 
ò !�ð ÐWÓWð �ŠGä˜˜qÓ!×*Ñ*Ó,ô-ò !ùs   ƒ25)r   Úkeysr;  s   `rC   r¿  Ú=InstructBlipVideoForConditionalGenerationModelOutput.to_tupleC  s%   ø€ Üô 
ð —Y‘Y”[ó	
ó 
ð 	
rE   r¼   )rr   rs   rt   ru   rŸ   r¶  r   r8   rx   r½   r·  r¸  r   r¹  r   rº  r   r   r   r¿  rz   r¼   rE   rC   rµ  rµ  )  sƒ   ‡ ñð -1€Dˆ%�×!Ñ!Ñ
" TÑ
)Ó0Ø.2€FˆE�%×#Ñ#Ñ$ tÑ+Ó2Ø8<€NÐ.°Ñ5Ó<ØKO€OÐAÀDÑHÓOØNRÐÐ2°_ÑDÀtÑKÓRð
˜% ™*÷ 
rE   rµ  z`
    InstructBlipVideo base Model consisting of language model, qformer and vision encoder.
    c                   ó²  ^ • \ rS rSrSrS/rS\4U 4S jjrS rS\	R                  S\	R                  4S	 jr\\        SS\	R                  S\	R                  S\	R                  S
-  S\	R                  S
-  S\	R                  S
-  S\	R                  S
-  S\	R                  S
-  S\	R                  S
-  S\S\S
-  S\\   S\\-  4S jj5       5       rSrU =r$ )rµ   iL  rb   r·   r+   c                 ó  >• [         TU ]  U5        [        UR                  5      U l        [
        R                  " [        R                  " SUR                  UR                  R                  5      5      U l        [        UR                  5      U l        [
        R                  " UR                  R                  UR                   R                  5      U l        [$        R&                  " UR                   5      U l        U R+                  5         g ©Nr$   )r1   r2   r,  Úvision_configÚvision_modelr   r7   r8   rè   Únum_query_tokensÚqformer_configr3   r·   r¡  Úqformerrå   Útext_configÚlanguage_projectionr!   Úfrom_configÚlanguage_modelr1  r@   s     €rC   r2   ÚInstructBlipVideoModel.__init__U  s·   ø€ Ü‰Ñ˜Ô ä8¸×9MÑ9MÓNˆÔÜŸLšL¬¯ª°Q¸×8OÑ8OÐQW×QfÑQf×QrÑQrÓ)sÓtˆÔÜ4°V×5JÑ5JÓKˆŒä#%§9¢9¨V×-BÑ-B×-NÑ-NÐPV×PbÑPb×PnÑPnÓ#oˆÔ Ü'×3Ò3°F×4FÑ4FÓGˆÔð 	�‰ÕrE   c                 ó"  • U R                   n[        U5      S:”  a=  SU;  a7  [        R                  R	                  5       S:”  a  [
        R                  S5        [        U R                  S5      (       a  SU R                  R                  l
        gg©z•
Some pre-processing hacks to make the model `accelerate` compatible. Check
https://github.com/huggingface/transformers/pull/21707 for more details.
r$   rÑ  a   The `language_model` is not in the `hf_device_map` dictionary and you are running your script in a multi-GPU environment. this may lead to unexpected behavior when using `accelerate`. Please pass a `device_map` that contains `language_model` to remove this warning. Please refer to https://github.com/huggingface/blog/blob/main/accelerate-large-models.md for more details on creating a `device_map` for large models.Ú_hf_hookTN©Úhf_device_mapÚlenr8   ÚcudaÚdevice_countÚloggerÚwarningrC  rÑ  rÕ  Úio_same_device©rA   r×  s     rC   Ú_preprocess_accelerateÚ-InstructBlipVideoModel._preprocess_accelerateb  ó|   € ð
 ×*Ñ*ˆäˆ}Ó Ó!Ð&6¸mÓ&KÔPU×PZÑPZ×PgÑPgÓPiÐlmÓPmä�N‰NðMôô �4×&Ñ&¨
×3Ñ3Ø:>ˆD×Ñ×(Ñ(Õ7ð 4rE   r™   r(  c           
      óf  • Uck  X R                  5       " [        R                  " SU R                  R                  [        R
                  UR                  S95      :H  nUR                  S5      nOXR                  R                  :H  nUR                  S5      R                  UR                  5      nU$ ©zJ
Obtains multimodal placeholder mask from `input_ids` or `inputs_embeds`.
r¼   ©rd   r˜   rK   )
r<  r8   Úfullr+   Úimage_token_idÚlongr˜   ÚallÚ	unsqueezerf   ©rA   r™   r(  Úspecial_image_masks       rC   Úget_placeholder_maskÚ+InstructBlipVideoModel.get_placeholder_maskv  ó˜   € ð ÑØ!.×2KÑ2KÔ2MÜ—
’
˜2˜tŸ{™{×9Ñ9ÄÇÁÐTa×ThÑThÑió3ñ "Ðð "4×!7Ñ!7¸Ó!;Ñà!*¯k©k×.HÑ.HÑ!HÐà/×9Ñ9¸"Ó=×@Ñ@À×AUÑAUÓVÐØ!Ð!rE   NÚqformer_input_idsÚqformer_attention_maskrË   Údecoder_input_idsÚdecoder_attention_maskr`   Ú	use_cacherÔ   rI   c           
      óv  • UR                   u  pÍpïnUR                  XÍ-  XïU5      nU R                  " S	UU	S.UD6nUS   n[        R                  " UR                  5       SS [        R                  UR                  S9nU R                  R                  UR                   S   SS5      n[        R                  " UR                  5       SS [        R                  UR                  S9nUc  [        R                  " U5      nUR                  USS9nUR                  USS9nUR                  UR                  5      n[        R                  " UU/SS9nU R                  " S	UUUUUS.UD6nUS   SS2SUR                  S5      2SS24   nU R                  U5      nUR                  XÀR                   R"                  U-  S5      nUcR  U R$                  R'                  5       " U5      nX@R                   R(                  :H  nUc  [        R                  " U5      nOjX€R'                  5       " [        R*                  " S	U R                   R(                  [        R                  UR                  S95      :H  nUR-                  S5      nUR/                  S5      R                  UR                  5      nUR                  UR                  UR0                  5      nUR3                  UU5      nU R                   R4                  (       a  U R$                  " S	UUU
S
.UD6nOU R$                  " S	UUUUU
S.UD6n[7        UUUS9$ )aU  
qformer_input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Indices of input sequence tokens in the vocabulary of the Q-Former. Input tokens can optionally be provided
    to serve as text prompt, which the Q-Former model will encode.

    Indices can be obtained using [`InstructBlipVideoProcessor`]. See [`InstructBlipVideoProcessor.__call__`] for
    details.

    [What are input IDs?](../glossary#input-ids)
qformer_attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
    Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:

    - 1 for tokens that are **not masked**,
    - 0 for tokens that are **masked**.

    [What are attention masks?](../glossary#attention-mask)
decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
    Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also
    be used by default.

    Only relevant in case an encoder-decoder language model (like T5) is used.
©rb   r`   r   NrK   rä  rP   r$   ©r™   rË   rš   rL  rM  r¼   ©r(  rË   ró  )r(  rË   rñ  rò  ró  r½  )rR   rU   rÊ  r8   ÚonesrM   rç  r˜   r·   ri   Ú	ones_likeÚrepeat_interleaverf   rZ   rÍ  rÏ  r+   rË  rÑ  r<  Úvideo_token_idrå  rè  ré  rd   Úmasked_scatterÚuse_decoder_only_language_modelrµ  )rA   rb   rï  rð  r™   rË   rñ  rò  r(  r`   ró  rÔ   rj   ÚframesÚchannelrG   rH   r¸  Úimage_embedsÚimage_attention_maskr·   Úquery_attention_maskÚquery_outputsÚquery_outputÚlanguage_model_inputsrë  Úoutputss                              rC   ro   ÚInstructBlipVideoModel.forward…  sU  € ðP 6B×5GÑ5GÑ2ˆ
˜G¨UØ#×+Ñ+¨JÑ,?ÀÐRWÓXˆà×*Ò*ð 
Ø%Ø%=ñ
ð ñ
ˆð
 & aÑ(ˆô  %Ÿzšz¨,×*;Ñ*;Ó*=¸c¸rÐ*BÌ%Ï*É*Ð]i×]pÑ]pÑqÐð ×(Ñ(×/Ñ/°×0BÑ0BÀ1Ñ0EÀrÈ2ÓNˆÜ$Ÿzšz¨,×*;Ñ*;Ó*=¸c¸rÐ*BÌ%Ï*É*Ð]i×]pÑ]pÑqÐà!Ñ)Ü%*§_¢_Ð5FÓ%GÐ"à-×?Ñ?ÀÈAÐ?ÐNÐØ!7×!IÑ!IÈ&ÐVWÐ!IÐ!XÐØ!7×!:Ñ!:Ð;O×;VÑ;VÓ!WÐÜ!&§¢Ð,@ÐBXÐ+YÐ_`Ñ!aÐØŸšð 
Ø'Ø1Ø%Ø".Ø#7ñ
ð ñ
ˆð % QÑ'ªÐ+A¨\×->Ñ->¸qÓ-AÐ+AÂ1Ð(DÑEˆð !%× 8Ñ 8¸Ó FÐð !6× =Ñ =¸jÏ+É+×JfÑJfÐioÑJoÐqsÓ tÐØÑ Ø ×/Ñ/×DÑDÔFÀyÓQˆMØ!*¯k©k×.HÑ.HÑ!HÐØÑ%Ü!&§¢°Ó!;�øà!.×2KÑ2KÔ2MÜ—
’
˜2˜tŸ{™{×9Ñ9ÄÇÁÐTa×ThÑThÑió3ñ "Ðð "4×!7Ñ!7¸Ó!;Ðà/×9Ñ9¸"Ó=×@Ñ@À×AUÑAUÓVÐØ 5× 8Ñ 8¸×9MÑ9MÈ}×ObÑObÓ cÐØ%×4Ñ4Ð5GÐI^Ó_ˆà�;‰;×6×6Ø×)Ò)ð Ø+Ø-Ø#ñð ñ	‰Gð ×)Ò)ð Ø+Ø-Ø"3Ø'=Ø#ñð ñˆGô DØ)Ø)Ø#*ñ
ð 	
rE   ©rÑ  rÏ  rÍ  r·   rÊ  )NNNNNNFN)rr   rs   rt   ru   r>  Ú_keep_in_fp32_modulesr%   r2   rß  r8   r²  rx   rì  r   r   rv   ry   r   r   r   rµ  ro   rz   r{   r|   s   @rC   rµ   rµ   L  sZ  ø† ð %€OØ+Ð,ÐðÐ6÷ ò?ð("¨e×.>Ñ.>ð "Èu×O`ÑO`ô "ð Øð
 ;?Ø.2Ø26Ø59Ø:>Ø-1Ø).Ø!%ñq
à×'Ñ'ðq
ð !×,Ñ,ðq
ð !&× 0Ñ 0°4Ñ 7ð	q
ð
 ×$Ñ$ tÑ+ðq
ð ×(Ñ(¨4Ñ/ðq
ð !×+Ñ+¨dÑ2ðq
ð !&× 0Ñ 0°4Ñ 7ðq
ð —|‘| dÑ*ðq
ð #'ðq
ð ˜$‘;ðq
ð Ð+Ñ,ðq
ð 
ÐEÑ	Eôq
ó ó öq
rE   rµ   c                   óB   • \ rS rSr% SrSr\S-  \S'   Sr\	S-  \S'   Sr
g)Ú'BaseModelOutputWithVisionQformerOutputsiû  zÉ
vision_outputs (`BaseModelOutputWithPooling`):
    Outputs of the vision encoder.
qformer_outputs (`BaseModelOutputWithPoolingAndCrossAttentions`):
    Outputs of the Q-Former (Querying Transformer).
Nr¸  r¹  r¼   )rr   rs   rt   ru   rŸ   r¸  r   r½   r¹  r   rz   r¼   rE   rC   r  r  û  s)   ‡ ñð 9=€NÐ.°Ñ5Ó<ØKO€OÐAÀDÑHÖOrE   r  a¯  
    InstructBlipVideo Model for generating text given an image and an optional text prompt. The model consists of a vision
    encoder, Querying Transformer (Q-Former) and a language model.

    One can optionally pass `input_ids` to the model, which serve as a text prompt, to make the language model continue
    the prompt. Otherwise, the language model starts generating text from the [BOS] (beginning-of-sequence) token.
    c                   ó¾  ^ • \ rS rSr% \\S'   SrSrS/rS\4U 4S jjr	S r
S\R                  4S	 jrSU 4S jjrS rS rS\R$                  S\R&                  4S jr\\         SS\R&                  S\R&                  S\R$                  S
-  S\R&                  S
-  S\R$                  S
-  S\R$                  S
-  S\R$                  S
-  S\R&                  S
-  S\R$                  S
-  S\S\S
-  S\\   S\\-  4S jj5       5       r\R:                  " 5             S S\R&                  S\R$                  S
-  S\R$                  S
-  S\R$                  S
-  S\R$                  S
-  S\R&                  S
-  S\S\R$                  4S jj5       r\\  S!S\R&                  S\R$                  S\R$                  S
-  S\S
-  S\\   S\\-  4S jj5       5       r Sr!U =r"$ )"r´   i	  r+   rb   Tr·   c                 óª  >• [         TU ]  U5        [        R                  UR                  5      U l        [        R                  " [        R                  " SUR                  UR                  R                  5      5      U l        [        R                  UR                  5      U l        [        R                   " UR                  R                  UR"                  R                  5      U l        UR&                  (       a!  [(        R*                  " UR"                  5      nO [,        R*                  " UR"                  5      nX l        U R1                  5         g rÈ  )r1   r2   r,  Ú_from_configrÉ  rÊ  r   r7   r8   rè   rË  rÌ  r3   r·   r¡  rÍ  rå   rÎ  rÏ  rý  r"   rÐ  r#   rÑ  r1  )rA   r+   rÑ  rB   s      €rC   r2   Ú2InstructBlipVideoForConditionalGeneration.__init__  så   ø€ Ü‰Ñ˜Ô ä8×EÑEÀf×FZÑFZÓ[ˆÔäŸLšL¬¯ª°Q¸×8OÑ8OÐQW×QfÑQf×QrÑQrÓ)sÓtˆÔÜ4×AÑAÀ&×BWÑBWÓXˆŒä#%§9¢9¨V×-BÑ-B×-NÑ-NÐPV×PbÑPb×PnÑPnÓ#oˆÔ à×1×1Ü1×=Ò=¸f×>PÑ>PÓQ‰Nä2×>Ò>¸v×?QÑ?QÓRˆNà,Ôð 	�‰ÕrE   c                 ó:   • U R                   R                  U5        g r  )rÑ  Úset_output_embeddings)rA   Únew_embeddingss     rC   r  Ú?InstructBlipVideoForConditionalGeneration.set_output_embeddings-  s   € Ø×Ñ×1Ñ1°.ÕArE   rI   c                 ó6   • U R                   R                  5       $ r  )rÑ  Úget_output_embeddingsr;  s    rC   r  Ú?InstructBlipVideoForConditionalGeneration.get_output_embeddings0  s   € Ø×"Ñ"×8Ñ8Ó:Ð:rE   Nc                 óX   >• Uc  U R                   R                  5       $ [        TU ]  US9$ )N)Úmodality)rÑ  Úget_encoderr1   )rA   r  rB   s     €rC   r  Ú5InstructBlipVideoForConditionalGeneration.get_encoder3  s1   ø€ ØÑØ×&Ñ&×2Ñ2Ó4Ð4ä‘7Ñ&°Ð&Ð9Ð9rE   c                 ó6   • U R                   R                  5       $ r  )rÑ  Úget_decoderr;  s    rC   r  Ú5InstructBlipVideoForConditionalGeneration.get_decoder9  s   € Ø×"Ñ"×.Ñ.Ó0Ð0rE   c                 ó"  • U R                   n[        U5      S:”  a=  SU;  a7  [        R                  R	                  5       S:”  a  [
        R                  S5        [        U R                  S5      (       a  SU R                  R                  l
        ggrÔ  rÖ  rÞ  s     rC   rß  Ú@InstructBlipVideoForConditionalGeneration._preprocess_accelerate<  rá  rE   r™   r(  c           
      óf  • Uck  X R                  5       " [        R                  " SU R                  R                  [        R
                  UR                  S95      :H  nUR                  S5      nOXR                  R                  :H  nUR                  S5      R                  UR                  5      nU$ rã  )
r<  r8   rå  r+   rû  rç  r˜   rè  ré  rf   rê  s       rC   rì  Ú>InstructBlipVideoForConditionalGeneration.get_placeholder_maskP  rî  rE   rï  rð  rË   rñ  rò  Úlabelsr`   ró  rÔ   c           
      óÄ  • U R                   " U4UUU
S.UD6nUR                  nUR                  nUR                  nUc  U R	                  5       " U5      nUc  [
        R                  " U5      nUR                  UR                  UR                  5      nU R                  XHS9nUR                  UU5      nU R                  R                  (       aT  U R                  " S	UUUS.UD6nUS   nSnU	b3  U R                  " S	UX�R                  R                   R"                  S.UD6nO1U R                  " S	UUUUU	US.UD6nUR$                  nUR&                  n[)        UUUUUS9$ )
aÐ	  
qformer_input_ids (`torch.LongTensor` of shape (batch_size, sequence_length)):
    The sequence used as a prompt to be fed to the Q-Former module.
qformer_attention_mask (`torch.LongTensor` of shape (batch_size, sequence_length), *optional*):
    Mask to avoid performing attention on padding token indices.

Examples:

```python
>>> from transformers import InstructBlipVideoProcessor, InstructBlipVideoForConditionalGeneration
>>> import torch
>>> from huggingface_hub import hf_hub_download
>>> import av
>>> import numpy as np

>>> def read_video_pyav(container, indices):
...     '''
...     Decode the video with PyAV decoder.
...     Args:
...         container (`av.container.input.InputContainer`): PyAV container.
...         indices (`list[int]`): List of frame indices to decode.
...     Returns:
...         result (np.ndarray): np array of decoded frames of shape (num_frames, height, width, 3).
...     '''
...     frames = []
...     container.seek(0)
...     start_index = indices[0]
...     end_index = indices[-1]
...     for i, frame in enumerate(container.decode(video=0)):
...         if i > end_index:
...             break
...         if i >= start_index and i in indices:
...             frames.append(frame)
...     return np.stack([x.to_ndarray(format="rgb24") for x in frames])

>>> model = InstructBlipVideoForConditionalGeneration.from_pretrained("Salesforce/instructblip-vicuna-7b", device_map="auto")
>>> processor = InstructBlipVideoProcessor.from_pretrained("Salesforce/instructblip-vicuna-7b")

>>> file_path = hf_hub_download(
...       repo_id="nielsr/video-demo", filename="eating_spaghetti.mp4", repo_type="dataset"
... )
>>> container = av.open(file_path)

>>> # sample uniformly 4 frames from the video
>>> total_frames = container.streams.video[0].frames
>>> indices = np.arange(0, total_frames, total_frames / 4).astype(int)
>>> clip = read_video_pyav(container, indices)

>>> prompt = "What is happening in the video?"
>>> inputs = processor(text=prompt, images=clip, return_tensors="pt").to(model.device)

>>> outputs = model.generate(
...     **inputs,
...     do_sample=False,
...     num_beams=5,
...     max_length=256,
...     repetition_penalty=1.5,
...     length_penalty=1.0,
... )
>>> generated_text = processor.batch_decode(outputs, skip_special_tokens=True)[0].strip()
>>> print(generated_text)
"A person is eating a bowl of pasta, and they are using a fork to eat it. The person is sitting at a table, and the plate of pasta is on the table in front"
```©rï  rð  r`   N©r(  r÷  r   )r·  r"  r‡   )r(  rË   rñ  rò  r"  ró  )r¶  r·  r¸  r¹  rº  r¼   )Úget_video_featuresr6  r¹  r¸  r<  r8   rù  rf   r˜   rd   rì  rü  r+   rý  rÑ  Úloss_functionrÎ  r‡   r¶  r·  rµ  )rA   rb   rï  rð  r™   rË   rñ  rò  r(  r"  r`   ró  rÔ   Úvideo_featuresr  r¹  r¸  rë  r  r·  r¶  s                        rC   ro   Ú1InstructBlipVideoForConditionalGeneration.forward_  s®  € ð` CG×BYÒBYØðC
à/Ø#9Ø%=ñ	C
ð
 ñC
ˆð !/× <Ñ <ÐØ(×8Ñ8ˆØ'×6Ñ6ˆàÑ Ø ×5Ñ5Ô7¸	ÓBˆMàÑ!Ü"Ÿ_š_¨YÓ7ˆNà 5× 8Ñ 8¸×9MÑ9MÈ}×ObÑObÓ cÐØ!×6Ñ6°yÐ6Ð^ÐØ%×4Ñ4Ð5GÐI^Ó_ˆà�;‰;×6×6Ø×)Ò)ð Ø+Ø-Ø#ñð ñ	ˆGð ˜Q‘ZˆFØˆDØÑ!Ø×)Ò)ð Ø!¨&¿[¹[×=TÑ=T×=_Ñ=_ñØciñ�øð
 ×)Ò)ð Ø+Ø-Ø"3Ø'=ØØ#ñð ñˆGð —<‘<ˆDØ—^‘^ˆFäCØØØ)Ø+Ø#*ñ
ð 	
rE   c                 óX  • [        U S5      (       a  U R                  5         UR                  S   n	U R                  UUUUS9n
U
R                  nUc®  Uc•  U R
                  R                  /U R
                  R                  -  S-  nXÀR
                  R                  R                  /-   n[        R                  " U/[        R                  UR                  S9nUR                  U	S5      nU R                  5       " U5      nUc  [        R                   " U5      nUR#                  UR                  UR$                  5      nU R'                  XFS9nUR)                  Xë5      nXeS.nU R*                  R
                  R,                  (       d  XOS	'   U R*                  R.                  " S
0 UDUD6nU$ )aA  
Overrides `generate` function to be able to use the model as a conditional generator.

Args:
    pixel_values (`torch.FloatTensor` of shape (batch_size, num_channels, height, width) or
        (batch_size, num_frames, num_channels, height, width)): Input images or videos to be processed.
    qformer_input_ids (`torch.LongTensor` of shape (batch_size, sequence_length), *optional*):
        The sequence used as a prompt to be fed to the Q-Former module.
    qformer_attention_mask (`torch.LongTensor` of shape (batch_size, sequence_length), *optional*):
        Mask to avoid performing attention on padding token indices.
    input_ids (`torch.LongTensor` of shape (batch_size, sequence_length), *optional*):
        The sequence used as a prompt for the generation.
    attention_mask (`torch.LongTensor` of shape (batch_size, sequence_length), *optional*):
        Mask to avoid performing attention on padding token indices.
    inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
        Embedded representation of the inputs. Should be float, not int tokens.
    interpolate_pos_encoding (`bool`, *optional*, defaults to `False`):
        Whether to interpolate the positional encoding of the image embeddings.

Returns:
    captions (list): A list of strings of length batch_size * num_captions.
r×  r   r$  rö   rä  r$   r%  )r(  rË   r™   r¼   )rC  rß  rR   r&  r6  r+   Úvideo_token_indexrË  rÎ  Úbos_token_idr8   rî   rç  r˜   Úrepeatr<  rù  rf   rd   rì  rü  rÑ  Úis_encoder_decoderÚgenerate)rA   rb   rï  rð  r™   rË   r(  r`   Úgenerate_kwargsrj   r(  r  Úvideo_tokensÚstart_tokensrë  Úinputsr  s                    rC   r/  Ú2InstructBlipVideoForConditionalGeneration.generateç  s’  € ôD �4˜×)Ñ)à×'Ñ'Ô)à!×'Ñ'¨Ñ*ˆ
ØBF×BYÑBYØØ/Ø#9Ø%=ð	 CZð C
ˆð !/× <Ñ <ÐàÑ ØÑ Ø $§¡× =Ñ =Ð>ÀÇÁ×A]ÑA]Ñ]Ð`aÑa�Ø+¯{©{×/FÑ/F×/SÑ/SÐ.TÑT�Ü!ŸLšL¨,¨¼u¿z¹zÐR^×ReÑReÑf�	Ø%×,Ñ,¨Z¸Ó;�	Ø ×5Ñ5Ô7¸	ÓBˆMàÑ!Ü"Ÿ_š_¨YÓ7ˆNà 5× 8Ñ 8¸×9MÑ9MÈ}×ObÑObÓ cÐØ!×6Ñ6°yÐ6Ð^ÐØ%×4Ñ4Ð5GÓ_ˆà#0ÑSˆØ×"Ñ"×)Ñ)×<×<Ø"+�;Ñà×%Ñ%×.Ò.ÑK°ÐK¸?ÑKˆàˆrE   c           	      ó  • UR                   u  pgp‰n
UR                  Xg-  X‰U
5      nU R                  " S
UUS.UD6n[        UR                  UR
                  UR                  UR                  USS9nUS   n[        R                  " UR                  5       SS [        R                  UR                  S9nU R                  R                  UR                   S   SS5      n[        R                  " UR                  5       SS [        R                  UR                  S9nUc  [        R                  " U5      nUR!                  USS9nUR!                  USS9nUR#                  UR                  5      n[        R$                  " Xó/SS9nU R&                  " S
UUUUUS	.UD6nUUl        US   SS2SUR                  S5      2SS24   nU R+                  U5      nUR                  X`R,                  R.                  U-  S5      nUUl        U$ )aÊ  
pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`):
    The tensors corresponding to the input images.
qformer_input_ids (`torch.LongTensor` of shape (batch_size, sequence_length)):
    The sequence used as a prompt to be fed to the Q-Former module.
qformer_attention_mask (`torch.LongTensor` of shape (batch_size, sequence_length), *optional*):
    Mask to avoid performing attention on padding token indices.
rõ  N)r'  r6  rô   r-  r¸  r¹  r   rK   rä  rP   r$   rö  r¼   )rR   rU   rÊ  r  r'  r6  rô   r-  r8   rø  rM   rç  r˜   r·   ri   rù  rú  rf   rZ   rÍ  r¹  rÏ  r+   rË  )rA   rb   rï  rð  r`   rÔ   rj   rþ  rÿ  rG   rH   r¸  r   r  r·   r  r¹  r  r(  s                      rC   r&  Ú<InstructBlipVideoForConditionalGeneration.get_video_features-  s/  € ð( 6B×5GÑ5GÑ2ˆ
˜G¨UØ#×+Ñ+¨JÑ,?ÀÐRWÓXˆà59×5FÒ5Fð 6
Ø%Ø%=ñ6
ð ñ6
ˆô
 AØ,×>Ñ>Ø(×6Ñ6Ø(×6Ñ6Ø%×0Ñ0Ø)Ø ñ
ˆð & aÑ(ˆô  %Ÿzšz¨,×*;Ñ*;Ó*=¸c¸rÐ*BÌ%Ï*É*Ð]i×]pÑ]pÑqÐð ×(Ñ(×/Ñ/°×0BÑ0BÀ1Ñ0EÀrÈ2ÓNˆÜ$Ÿzšz¨,×*;Ñ*;Ó*=¸c¸rÐ*BÌ%Ï*É*Ð]i×]pÑ]pÑqÐà!Ñ)Ü%*§_¢_Ð5FÓ%GÐ"à-×?Ñ?ÀÈAÐ?ÐNÐØ!7×!IÑ!IÈ&ÐVWÐ!IÐ!XÐØ!7×!:Ñ!:Ð;O×;VÑ;VÓ!WÐÜ!&§¢Ð,@Ð+YÐ_`Ñ!aÐØŸ,š,ð 
Ø'Ø1Ø%Ø".Ø#7ñ
ð ñ
ˆð *9ˆÔ&Ø& qÑ)ª!Ð-C¨|×/@Ñ/@ÀÓ/CÐ-CÂQÐ*FÑGˆð ×1Ñ1°,Ó?ˆð (×/Ñ/°
¿K¹K×<XÑ<XÐ[aÑ<aÐceÓfˆØ'5ˆÔ$àÐrE   r  r  )	NNNNNNNFN)NNNNNFr  )#rr   rs   rt   ru   r%   r½   r>  rÅ   r	  r2   r  r   ÚModuler  r  r  rß  r8   r²  rx   rì  r   r   ry   r   r   r   rµ  ro   rÇ   r/  r  r&  rz   r{   r|   s   @rC   r´   r´   	  sù  ø‡ ð $Ó#Ø$€Oà!ÐØ+Ð,ÐðÐ6÷ ò(Bð; r§y¡yô ;÷:ò1ò?ð("¨e×.>Ñ.>ð "Èu×O`ÑO`ô "ð Øð
 ;?Ø.2Ø26Ø59Ø:>Ø26Ø*.Ø).Ø!%ñD
à×'Ñ'ðD
ð !×,Ñ,ðD
ð !&× 0Ñ 0°4Ñ 7ð	D
ð
 ×$Ñ$ tÑ+ðD
ð ×(Ñ(¨4Ñ/ðD
ð !×+Ñ+¨dÑ2ðD
ð !&× 0Ñ 0°4Ñ 7ðD
ð ×(Ñ(¨4Ñ/ðD
ð × Ñ  4Ñ'ðD
ð #'ðD
ð ˜$‘;ðD
ð Ð+Ñ,ðD
ð 
ÐEÑ	EôD
ó ó ðD
ðL ‡]‚]ƒ_ð 6:Ø:>Ø-1Ø26Ø26Ø).ñCà×'Ñ'ðCð !×+Ñ+¨dÑ2ðCð !&× 0Ñ 0°4Ñ 7ð	Cð
 ×#Ñ# dÑ*ðCð ×(Ñ(¨4Ñ/ðCð ×(Ñ(¨4Ñ/ðCð #'ðCð 
×	Ñ	ôCó ðCðJ Øð
 ;?Ø05ñDà×'Ñ'ðDð !×+Ñ+ðDð !&× 0Ñ 0°4Ñ 7ð	Dð
 #'¨¡+ðDð Ð+Ñ,ðDð 
Ð8Ñ	8ôDó ó öDrE   r´   )r,  r¡   r¡  rµ   r´   )r¬   )QÚcollections.abcr   Údataclassesr   Útypingr   r8   r   Ú r   r²   Úactivationsr	   Ú
generationr
   Úmasking_utilsr   Úmodeling_layersr   Úmodeling_outputsr   r   r   r   r   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úpytorch_utilsr   Úutilsr   r   r   r   r   r   Úutils.genericr   Úutils.output_capturingr   r   Úautor!   r"   r#   Úconfiguration_instructblipvideor%   r&   r'   Ú
get_loggerrr   rÛ  r7  r)   r~   r¡   rv   Úfloatr×   r¦   r  r§   r  r,  r©   rª   rd  rn  rw  r¨   r˜  r¡  rµ  rµ   r  r´   Ú__all__r¼   rE   rC   Ú<module>rL     s›  ðõ, %Ý !Ý ã Ý å &Ý !Ý )Ý 6Ý 9÷÷ ÷ GÝ &Ý 6ß j× jÝ 7ß Eß IÑ I÷ñ ð 
×	Ò	˜HÓ	%€ôG¨¯	©	ô GôT-¨¯©ô -ð` ô i ó  ió ð iðV ñ%Ø�I‰Ið%à�<‰<ð%ð 
�‰ð%ð �<‰<ð	%ð
 —L‘L 4Ñ'ð%ð ð%ð õ%ô.G) §¡ô G)ôT˜2Ÿ9™9ô ôÐ$>ô ô@@˜rŸy™yô @ô@3Ð#Cô 3ôlE)°·±ô E)ôP¨¯©ô ô ¨¯	©	ô  ô2¨2¯9©9ô ô R§Y¡Yô ôRÐ$>ô Rôj!
 b§i¡iô !
ôHa
Ð$Dô a
ñH ðñð
 ô
¸;ó 
ó óð
ñ: ðñô
g
Ð=ó g
óð
g
ðT Ø
ô	PÐ.Hó 	Pó ó ð	Pñ ðñôaÐ0PÐRaó aóðaòH�rE   