ó
    ˆŽ¡jƒb  ã                   óº  • S SK rS SK Jr  S SKJr  S SKrS SKJr  SSKJ	r
  SSKJr  SSKJr  SSKJr  SS	KJrJr  SS
KJrJr  SSKJr  SSKJrJrJr  SSKJrJr  SSK J!r!  SSK"J#r#  SSK$J%r%  SSK&J'r'  SSK(J)r)J*r*J+r+J,r,J-r-  SSK.J/r/J0r0   SBS\Rb                  S\Rd                  S\Rd                  S\Rd                  S\Rd                  S-  S\3S\3\4-  4S jjr5 " S S\'5      r6 " S  S!\%5      r7\" S"S#9\ " S$ S%\5      5       5       r8 " S& S'\Rb                  5      r9 " S( S)\Rb                  5      r: " S* S+\#5      r;\Rx                  \6S,.r= " S- S.\5      r> " S/ S0\Rb                  5      r?\ " S1 S2\5      5       r@\ " S3 S4\@5      5       rA " S5 S6\-5      rBSrC " S7 S8\Rb                  5      rD " S9 S:\,5      rE " S; S<\+5      rF " S= S>\)5      rG " S? S@\*5      rH/ SAQrIg)Cé    N)ÚCallable)Ú	dataclassé   )Úinitialization)ÚACT2FN)ÚCache)ÚGradientCheckpointingLayer)ÚBaseModelOutputÚBaseModelOutputWithPooling)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚ	torch_int)Úcan_return_tupleÚmerge_with_config_defaults)Úcapture_outputsé   )ÚCLIPMLP)ÚJanusVisionAttention)ÚLlamaRMSNorm)ÚLlavaCausalLMOutputWithPastÚLlavaForConditionalGenerationÚ
LlavaModelÚLlavaModelOutputWithPastÚLlavaPreTrainedModelé   )ÚInternVLConfigÚInternVLVisionConfigÚmoduleÚqueryÚkeyÚvalueÚattention_maskÚscalingÚdropoutc                 óh  • UnUn	[         R                  " XR                  SS5      5      U-  n
Ub  X¤-   n
[        R                  R                  U
SS9n
[        R                  R                  X¦U R                  S9n
[         R                  " X©5      nUR                  SS5      R                  5       nXº4$ )Nr   r   éÿÿÿÿ©Údim)ÚpÚtrainingr   )	ÚtorchÚmatmulÚ	transposeÚnnÚ
functionalÚsoftmaxr'   r-   Ú
contiguous)r!   r"   r#   r$   r%   r&   r'   ÚkwargsÚ
key_statesÚvalue_statesÚattn_weightsÚattn_outputs               Ú»C:\Users\johnn\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.13_qbz5n2kfra8p0\LocalCache\local-packages\Python313\site-packages\transformers/models/internvl/modular_internvl.pyÚeager_attention_forwardr;   .   s¦   € ð €JØ€Lä—<’< ×';Ñ';¸A¸qÓ'AÓBÀWÑL€LØÑ!Ø#Ñ4ˆô —=‘=×(Ñ(¨¸2Ð(Ð>€LÜ—=‘=×(Ñ(¨È6Ï?É?Ð(Ð[€LÜ—,’,˜|Ó:€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€KàÐ$Ð$ó    c                   ó   • \ rS rSrSrg)ÚInternVLVisionRMSNorméH   © N©Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__static_attributes__r@   r<   r:   r>   r>   H   ó   † Úr<   r>   c                   ó€   ^ • \ rS rSrS\4U 4S jjr S
S\R                  S\R                  S-  S\\	   4S jjr
S	rU =r$ )ÚInternVLVisionAttentionéL   Úconfigc                 ó2  >• [         TU ]  U5        U ?SU l        UR                  nU(       a  [        U R                  5      O[        R                  " 5       U l	        U(       a  [        U R                  5      U l
        g [        R                  " 5       U l
        g ©NF)ÚsuperÚ__init__Únum_key_value_groupsÚ	is_causalÚuse_qk_normr>   Ú	embed_dimr1   ÚIdentityÚq_normÚk_norm)ÚselfrK   Úqk_normÚ	__class__s      €r:   rO   Ú InternVLVisionAttention.__init__M   sd   ø€ Ü‰Ñ˜Ô ØÐ%ð ˆŒØ×$Ñ$ˆæ?FÔ+¨D¯N©NÔ;ÌBÏKÊKËMˆŒÞ?FÔ+¨D¯N©NÓ;ˆ�ÌBÏKÊKËMˆ�r<   NÚhidden_statesr%   r5   c                 óp  • UR                  5       u  pEnU R                  U5      nU R                  U5      nU R                  U5      n	U R	                  U5      nU R                  U5      nUR                  XEU R                  U R                  5      R                  SS5      nUR                  XEU R                  U R                  5      R                  SS5      nU	R                  XEU R                  U R                  5      R                  SS5      n	[        R                  " U R                  R                  [        5      n
U
" U UUU	U4U R                   (       d  SOU R"                  U R$                  SS.UD6u  p¼UR                  XEU R&                  5      nU R)                  U5      nU R+                  U5      nXÜ4$ )Nr   r   ç        F)r'   r&   rQ   )ÚsizeÚq_projÚk_projÚv_projrU   rV   ÚreshapeÚ	num_headsÚhead_dimr0   Úviewr   Úget_interfacerK   Ú_attn_implementationr;   r-   Úattention_dropoutÚscalerS   Úprojection_layerÚprojection_dropout)rW   r[   r%   r5   Ú
batch_sizeÚseq_lenÚ_Úquery_statesr6   r7   Úattention_interfacer9   r8   Úoutputs                 r:   ÚforwardÚInternVLVisionAttention.forwardX   s�  € ð "/×!3Ñ!3Ó!5Ñˆ
˜Qà—{‘{ =Ó1ˆØ—[‘[ Ó/ˆ
Ø—{‘{ =Ó1ˆà—{‘{ <Ó0ˆØ—[‘[ Ó,ˆ
à#×+Ñ+¨JÀÇÁÐQU×Q^ÑQ^Ó_×iÑiÐjkÐmnÓoˆØ×'Ñ'¨
¸T¿^¹^ÈTÏ]É]Ó[×eÑeÐfgÐijÓkˆ
Ø#×(Ñ(¨¸d¿n¹nÈdÏmÉmÓ\×fÑfÐghÐjkÓlˆä(?×(MÒ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØð
%
ð  $Ÿ}Ÿ}‘C°$×2HÑ2HØ—J‘JØñ
%
ð ñ
%
Ñ!ˆð "×)Ñ)¨*¸t¿~¹~ÓNˆà×&Ñ& {Ó3ˆØ×(Ñ(¨Ó0ˆàÐ#Ð#r<   )rQ   rV   rU   ©N)rB   rC   rD   rE   r    rO   r.   ÚTensorr   r   rr   rF   Ú__classcell__©rY   s   @r:   rI   rI   L   sP   ø† ð	ZÐ3÷ 	Zð /3ñ'$à—|‘|ð'$ð Ÿ™ tÑ+ð'$ð Ð+Ñ,÷	'$ó '$r<   rI   z7
    Class for outputs of [`InternVLVisionModel`].
    ©Úcustom_introc                   ó   • \ rS rSrSrSrg)Ú$InternVLVisionModelOutputWithPoolingé‚   a2  
pooler_output (`torch.FloatTensor` of shape `(batch_size, hidden_size)`):
    Average of the last layer hidden states of the patch tokens (excluding the *[CLS]* token) if
    *config.use_mean_pooling* is set to True. If set to False, then the final hidden state of the *[CLS]* token
    will be returned.
r@   N)rB   rC   rD   rE   Ú__doc__rF   r@   r<   r:   r{   r{   ‚   s   † ôr<   r{   c                   óf   ^ • \ rS rSrSrU 4S jrS\R                  S\R                  4S jrSr	U =r
$ )ÚInternVLVisionPatchEmbeddingsé‘   zÜ
This class turns `pixel_values` of shape `(batch_size, num_channels, height, width)` into the initial
`hidden_states` (patch embeddings) of shape `(batch_size, seq_length, hidden_size)` to be consumed by a
Transformer.
c                 óH  >• [         TU ]  5         UR                  UR                  p2UR                  UR
                  pTUS   US   -  US   US   -  -  nUS   US   -  US   US   -  4nX l        X0l        X@l        X`l        Xpl        [        R                  " XEX3S9U l
        g )Nr   r   )Úkernel_sizeÚstride)rN   rO   Ú
image_sizeÚ
patch_sizeÚnum_channelsÚhidden_sizeÚnum_patchesÚpatch_shaper1   ÚConv2dÚ
projection)	rW   rK   r„   r…   r†   r‡   rˆ   r‰   rY   s	           €r:   rO   Ú&InternVLVisionPatchEmbeddings.__init__˜   s¬   ø€ Ü‰ÑÔØ!'×!2Ñ!2°F×4EÑ4E�JØ$*×$7Ñ$7¸×9KÑ9K�kà! !‘}¨
°1©Ñ5¸*ÀQ¹-È:ÐVWÉ=Ñ:XÑYˆØ! !‘}¨
°1©Ñ5°zÀ!±}È
ÐSTÉÑ7UÐVˆØ$ŒØ$ŒØ(ÔØ&ÔØ&ÔäŸ)š) LÈ:Ñiˆ�r<   Úpixel_valuesÚreturnc                 ó  • UR                   u  p#pEX0R                  :w  a  [        S5      eU R                  UR	                  U R                  R
                  R                  5      5      nUR                  S5      R                  SS5      nU$ )NzeMake sure that the channel dimension of the pixel values match with the one set in the configuration.r   r   )	Úshaper†   Ú
ValueErrorr‹   ÚtoÚweightÚdtypeÚflattenr0   )rW   r�   rl   r†   ÚheightÚwidthÚ
embeddingss          r:   rr   Ú%InternVLVisionPatchEmbeddings.forward§   sz   € Ø2>×2DÑ2DÑ/ˆ
 &Ø×,Ñ,Ó,ÜØwóð ð —_‘_ \§_¡_°T·_±_×5KÑ5K×5QÑ5QÓ%RÓSˆ
Ø×'Ñ'¨Ó*×4Ñ4°Q¸Ó:ˆ
àÐr<   )r„   r†   rˆ   r‰   r…   r‹   )rB   rC   rD   rE   r}   rO   r.   ru   rr   rF   rv   rw   s   @r:   r   r   ‘   s.   ø† ñõjð
 E§L¡Lð 
°U·\±\÷ 
ò 
r<   r   c                   óØ   ^ • \ rS rSrSrS\SS4U 4S jjrS\R                  S\	S	\	S\R                  4S
 jr
 SS\R                  S\R                  S-  S\R                  4S jjrSrU =r$ )ÚInternVLVisionEmbeddingsé¶   z[
Construct the CLS token, position and patch embeddings. Optionally, also the mask token.

rK   rŽ   Nc                 ó^  >• [         TU ]  5         [        R                  " [        R
                  " SSUR                  5      5      U l        UR                  (       a<  [        R                  " [        R
                  " SSUR                  5      5      U l	        OS U l	        [        U5      U l        UR                  U l        [        UR                  [        R                   R"                  5      (       a  UR                  OUR                  UR                  4U l        U R                  R$                  nUR&                  (       a?  [        R                  " [        R
                  " SUS-   UR                  5      5      U l        OS U l        [        R*                  " UR,                  5      U l        g )Nr   )rN   rO   r1   Ú	Parameterr.   Úzerosr‡   Ú	cls_tokenÚuse_mask_tokenÚ
mask_tokenr   Úpatch_embeddingsr…   Ú
isinstancer„   ÚcollectionsÚabcÚIterablerˆ   Ú use_absolute_position_embeddingsÚposition_embeddingsÚDropoutÚhidden_dropout_probr'   )rW   rK   rˆ   rY   s      €r:   rO   Ú!InternVLVisionEmbeddings.__init__¼   s'  ø€ Ü‰ÑÔäŸš¤e§k¢k°!°Q¸×8JÑ8JÓ&KÓLˆŒØ× × Ü Ÿlšl¬5¯;ª;°q¸!¸V×=OÑ=OÓ+PÓQˆD�Oà"ˆDŒOÜ =¸fÓ EˆÔØ ×+Ñ+ˆŒô ˜&×+Ñ+¬[¯_©_×-EÑ-E×FÑFð ×Òà×#Ñ# V×%6Ñ%6Ð7ð 	Œð
 ×+Ñ+×7Ñ7ˆØ×2×2Ü')§|¢|´E·K²KÀÀ;ÐQRÁ?ÐTZ×TfÑTfÓ4gÓ'hˆDÕ$à'+ˆDÔ$Ü—z’z &×"<Ñ"<Ó=ˆ�r<   r˜   r–   r—   c                 ó¨  • UR                   S   S-
  nU R                  R                   S   S-
  n[        R                  R	                  5       (       d  XE:X  a  X#:X  a  U R                  $ U R                  SS2SS24   nU R                  SS2SS24   nUR                   S   nX R
                  S   -  n	X0R
                  S   -  n
[        US-  5      nUR                  SX»U5      nUR                  SSSS5      n[        R                  R                  UXš4SS	S
9nUR                  SSSS5      R                  SSU5      n[        R                  " Xg4SS9$ )aÐ  
This method allows to interpolate the pre-trained position encodings, to be able to use the model on higher resolution
images. This method is also adapted to support torch.jit tracing.

Adapted from:
- https://github.com/facebookresearch/dino/blob/de9ee3df6cf39fac952ab558447af1fa1365362a/vision_transformer.py#L174-L194, and
- https://github.com/facebookresearch/dinov2/blob/e1277af2ba9496fbadf7aec6eba56e8d882d1e35/dinov2/models/vision_transformer.py#L179-L211
r   Nr)   r   ç      à?r   r   ÚbicubicF)r^   ÚmodeÚalign_cornersr*   )r�   r©   r.   ÚjitÚ
is_tracingr…   r   rb   Úpermuter1   r2   Úinterpolatere   Úcat)rW   r˜   r–   r—   rˆ   Únum_positionsÚclass_pos_embedÚpatch_pos_embedr+   Ú
new_heightÚ	new_widthÚsqrt_num_positionss               r:   Úinterpolate_pos_encodingÚ1InternVLVisionEmbeddings.interpolate_pos_encodingÒ   s]  € ð !×&Ñ& qÑ)¨AÑ-ˆØ×0Ñ0×6Ñ6°qÑ9¸AÑ=ˆô �y‰y×#Ñ#×%Ñ%¨+Ó*FÈ6Ë?Ø×+Ñ+Ð+à×2Ñ2²1°b°q°b°5Ñ9ˆØ×2Ñ2²1°a±b°5Ñ9ˆà×Ñ˜rÑ"ˆàŸ™¨qÑ1Ñ1ˆ
ØŸ_™_¨QÑ/Ñ/ˆ	ä& }°cÑ'9Ó:ÐØ)×1Ñ1°!Ð5GÐ]`ÓaˆØ)×1Ñ1°!°Q¸¸1Ó=ˆäŸ-™-×3Ñ3ØØÐ(ØØð	 4ð 
ˆð *×1Ñ1°!°Q¸¸1Ó=×BÑBÀ1ÀbÈ#ÓNˆä�yŠy˜/Ð;ÀÑCÐCr<   r�   Úbool_masked_posc                 óÐ  • UR                   u    p4nU R                  U5      nUR                  5       u  pxnUbI  U R                  R	                  XxS5      n	UR                  S5      R                  U	5      n
USU
-
  -  Xš-  -   nU R                  R	                  USS5      n[        R                  " X¶4SS9nU R                  b  X`R                  XdU5      -   nU R                  U5      nU$ )Nr)   r   r*   )r�   r£   r^   r¢   ÚexpandÚ	unsqueezeÚtype_asr    r.   r¶   r©   r½   r'   )rW   r�   r¿   rn   r–   r—   r˜   rl   rm   Úmask_tokensÚwÚ
cls_tokenss               r:   rr   Ú InternVLVisionEmbeddings.forwardú   sä   € ð
 +×0Ñ0Ñˆˆ1�eØ×*Ñ*¨<Ó8ˆ
Ø!+§¡Ó!2Ñˆ
˜QàÑ&ØŸ/™/×0Ñ0°ÀbÓIˆKà×)Ñ)¨"Ó-×5Ñ5°kÓBˆAØ# q¨1¡uÑ-°±Ñ?ˆJà—^‘^×*Ñ*¨:°r¸2Ó>ˆ
Ü—Y’Y 
Ð7¸QÑ?ˆ
à×#Ñ#Ñ/Ø#×&CÑ&CÀJÐX]Ó&^Ñ^ˆJà—\‘\ *Ó-ˆ
àÐr<   )r    r'   r„   r¢   r£   r…   r©   rt   )rB   rC   rD   rE   r}   r    rO   r.   ru   Úintr½   Ú
BoolTensorrr   rF   rv   rw   s   @r:   r›   r›   ¶   s�   ø† ñð
>Ð3ð >¸÷ >ð,&D°5·<±<ð &DÈð &DÐUXð &DÐ]b×]iÑ]iô &DðV 48ñà—l‘lðð ×)Ñ)¨DÑ0ðð 
�‰÷	ó r<   r›   c                   ó   • \ rS rSrSrg)ÚInternVLVisionMLPi  r@   NrA   r@   r<   r:   rË   rË     rG   r<   rË   )Ú
layer_normÚrms_normc                   ór   ^ • \ rS rSrSrS\SS4U 4S jjrS\R                  S\R                  4S jr	S	r
U =r$ )
ÚInternVLVisionLayeri  z?This corresponds to the Block class in the timm implementation.rK   rŽ   Nc                 ó¤  >• [         TU ]  5         UR                  U l        SU l        [	        U5      U l        [        U5      U l        [        UR                     " UR                  UR                  S9U l        [        UR                     " UR                  UR                  S9U l        UR                  n[        R                   " U["        R$                  " UR                  5      -  SS9U l        [        R                   " U["        R$                  " UR                  5      -  SS9U l        [        R*                  " UR,                  5      U l        g )Nr   ©ÚepsT)Úrequires_grad)rN   rO   Úchunk_size_feed_forwardÚseq_len_dimrI   Ú	attentionrË   ÚmlpÚNORM2FNÚ	norm_typer‡   Úlayer_norm_epsÚlayernorm_beforeÚlayernorm_afterÚlayer_scale_init_valuer1   rž   r.   ÚonesÚlambda_1Úlambda_2rª   r«   r'   )rW   rK   Úinit_valuesrY   s      €r:   rO   ÚInternVLVisionLayer.__init__  sù   ø€ Ü‰ÑÔØ'-×'EÑ'EˆÔ$ØˆÔÜ0°Ó8ˆŒÜ$ VÓ,ˆŒä '¨×(8Ñ(8Ò 9¸&×:LÑ:LÐRX×RgÑRgÑ hˆÔÜ& v×'7Ñ'7Ò8¸×9KÑ9KÐQW×QfÑQfÑgˆÔà×3Ñ3ˆÜŸš [´5·:²:¸f×>PÑ>PÓ3QÑ%QÐaeÑfˆŒÜŸš [´5·:²:¸f×>PÑ>PÓ3QÑ%QÐaeÑfˆŒÜ—z’z &×"<Ñ"<Ó=ˆ�r<   r[   c                 ó  • U R                  U R                  U5      5      u  p#U R                  U-  nX!-   nU R                  U5      nU R	                  U5      nU R                  U5      nU R                  b  U R                  U-  nXA-   nU$ rt   )rÖ   rÛ   rß   rÜ   r×   r'   rà   )rW   r[   Úattention_outputrn   Úlayer_outputs        r:   rr   ÚInternVLVisionLayer.forward-  s–   € ð #Ÿn™nØ×!Ñ! -Ó0ó
ÑÐð  Ÿ=™=Ð+;Ñ;Ðð )Ñ8ˆð ×+Ñ+¨MÓ:ˆà—x‘x Ó-ˆØ—|‘| LÓ1ˆà�=‰=Ñ$ØŸ=™=¨<Ñ7ˆLð $Ñ3ˆàÐr<   )	rÖ   rÔ   r'   rß   rà   rÜ   rÛ   r×   rÕ   )rB   rC   rD   rE   r}   r    rO   r.   ru   rr   rF   rv   rw   s   @r:   rÏ   rÏ     s>   ø† ÙIð>Ð3ð >¸÷ >ðà—|‘|ðð 
�‰÷ò r<   rÏ   c                   ó`   ^ • \ rS rSrS\SS4U 4S jjrS\R                  S\\	-  4S jr
SrU =r$ )	ÚInternVLVisionEncoderiI  rK   rŽ   Nc                 óÖ   >• [         TU ]  5         Xl        [        R                  " [        UR                  5       Vs/ s H  n[        U5      PM     sn5      U l        SU l	        g s  snf rM   )
rN   rO   rK   r1   Ú
ModuleListÚrangeÚnum_hidden_layersrÏ   ÚlayerÚgradient_checkpointing)rW   rK   ÚirY   s      €r:   rO   ÚInternVLVisionEncoder.__init__J  sS   ø€ Ü‰ÑÔØŒÜ—]’]ÌÈv×OgÑOgÔIhÓ#iÒIhÀAÔ$7¸Ö$?ÑIhÑ#iÓjˆŒ
Ø&+ˆÕ#ùò $js   ½A&r[   c                 óJ   • U R                    H  nU" U5      nM     [        US9$ )N)Úlast_hidden_state)rí   r
   )rW   r[   Úlayer_modules      r:   rr   ÚInternVLVisionEncoder.forwardP  s.   € ð !ŸJœJˆLÙ(¨Ó7ŠMñ 'ô Ø+ñ
ð 	
r<   )rK   rî   rí   )rB   rC   rD   rE   r    rO   r.   ru   Útupler
   rr   rF   rv   rw   s   @r:   rè   rè   I  s<   ø† ð,Ð3ð ,¸÷ ,ð	
à—|‘|ð	
ð 
�Ñ	 ÷	
ò 	
r<   rè   c                   óŒ   ^ • \ rS rSr% \\S'   SrSrSrSr	S/r
SrSrSrSr\\S.r\R&                  " 5       U 4S	 j5       rS
rU =r$ )ÚInternVLVisionPreTrainedModeli\  rK   Úinternvl_visionr�   )ÚimageÚvideoTrÏ   )r[   Ú
attentionsc                 óF  >• [         TU ]  U5        [        U[        5      (       a|  [        R
                  " UR                  5        UR                  b   [        R
                  " UR                  5        UR                  b!  [        R
                  " UR                  5        gg[        U[        5      (       ak  [        R                  " UR                  U R                  R                  5        [        R                  " UR                  U R                  R                  5        gg)zInitialize the weightsN)rN   Ú_init_weightsr¤   r›   ÚinitÚzeros_r    r¢   r©   rÏ   Ú	constant_rß   rK   rÝ   rà   )rW   r!   rY   s     €r:   rý   Ú+InternVLVisionPreTrainedModel._init_weightsn  s¾   ø€ ô 	‰Ñ˜fÔ%Ü�fÔ6×7Ñ7Ü�KŠK˜×(Ñ(Ô)Ø× Ñ Ñ,Ü—’˜F×-Ñ-Ô.Ø×)Ñ)Ñ5Ü—’˜F×6Ñ6Õ7ð 6ä˜Ô 3×4Ñ4Ü�NŠN˜6Ÿ?™?¨D¯K©K×,NÑ,NÔOÜ�NŠN˜6Ÿ?™?¨D¯K©K×,NÑ,NÕOð 5r<   r@   )rB   rC   rD   rE   r    Ú__annotations__Úbase_model_prefixÚmain_input_nameÚinput_modalitiesÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_supports_sdpaÚ_supports_flash_attnÚ_supports_flex_attnÚ_supports_attention_backendrÏ   rI   Ú_can_record_outputsr.   Úno_gradrý   rF   rv   rw   s   @r:   r÷   r÷   \  sl   ø‡ à Ó Ø)ÐØ$€OØ)ÐØ&*Ð#Ø.Ð/ÐØ€NØÐØÐØ"&Ðð -Ø-ñÐð
 ‡]‚]ƒ_ôPó öPr<   r÷   c                   ó°   ^ • \ rS rSrS\SS4U 4S jjrS r\\" SS9\	 SS	\
R                  S
\
R                  S-  S\\-  4S jj5       5       5       rSrU =r$ )ÚInternVLVisionModeli}  rK   rŽ   Nc                 ó8  >• [         TU ]  U5        Xl        [        U5      U l        [        U5      U l        UR                  (       a  [        R                  " 5       O([        R                  " UR                  UR                  S9U l        U R                  5         g )NrÑ   )rN   rO   rK   r›   r˜   rè   ÚencoderÚuse_mean_poolingr1   rT   Ú	LayerNormr‡   rÚ   Ú	layernormÚ	post_init©rW   rK   rY   s     €r:   rO   ÚInternVLVisionModel.__init__  sm   ø€ Ü‰Ñ˜Ô ØŒä2°6Ó:ˆŒÜ,¨VÓ4ˆŒð $×4×4ŒB�KŠKŒM¼"¿,º,Àv×GYÑGYÐ_e×_tÑ_tÑ:uð 	Œð
 	�‰Õr<   c                 ó.   • U R                   R                  $ rt   )r˜   r£   )rW   s    r:   Úget_input_embeddingsÚ(InternVLVisionModel.get_input_embeddings�  s   € Ø�‰×/Ñ/Ð/r<   F)Útie_last_hidden_statesr�   r¿   c                 ó¬   • U R                  XS9nU R                  U5      nUS   nU R                  U5      n[        UUR                  UR
                  S9$ )z³
bool_masked_pos (`torch.BoolTensor` of shape `(batch_size, num_patches)`, *optional*):
    Boolean masked positions. Indicates which patches are masked (1) and which aren't (0).
)r¿   r   )rò   r[   rû   )r˜   r  r  r{   r[   rû   )rW   r�   r¿   r5   Úembedding_outputÚencoder_outputsÚsequence_outputs          r:   rr   ÚInternVLVisionModel.forward�  s^   € ð  Ÿ?™?¨<˜?ÐYÐàŸ,™,Ð'7Ó8ˆØ)¨!Ñ,ˆØŸ.™.¨Ó9ˆä3Ø-Ø)×7Ñ7Ø&×1Ñ1ñ
ð 	
r<   )rK   r˜   r  r  rt   )rB   rC   rD   rE   r    rO   r  r   r   r   r.   ru   rÉ   rõ   r{   rr   rF   rv   rw   s   @r:   r  r  }  su   ø† ðÐ3ð ¸÷ ò0ð  Ù¨EÑ2ØàUYñ
Ø!ŸL™Lð
Ø;@×;KÑ;KÈdÑ;Rð
à	Ð5Ñ	5ô
ó ó 3ó  ö
r<   r  c                   ó   • \ rS rSrSrSrg)ÚInternVLPreTrainedModeli§  )rù   Útextrú   r@   N)rB   rC   rD   rE   r  rF   r@   r<   r:   r"  r"  §  s   † Ø1Ór<   r"  c                   ó6   ^ • \ rS rSrS\4U 4S jjrS rSrU =r$ )ÚInternVLMultiModalProjectori®  rK   c                 ó0  >• [         TU ]  5         [        R                  " UR                  R
                  [        SUR                  -  5      S-  -  5      U l        [        R                  " UR                  R
                  [        SUR                  -  5      S-  -  UR                  R
                  5      U l        [        UR                     U l        [        R                  " UR                  R
                  UR                  R
                  5      U l        g )Nr   r   )rN   rO   r1   r  Úvision_configr‡   rÈ   Údownsample_ratiorÌ   ÚLinearÚtext_configÚlinear_1r   Úprojector_hidden_actÚactÚlinear_2r  s     €r:   rO   Ú$InternVLMultiModalProjector.__init__¯  sÍ   ø€ Ü‰ÑÔÜŸ,š, v×';Ñ';×'GÑ'GÌ#ÈaÐRX×RiÑRiÑNiÓJjÐnoÑJoÑ'oÓpˆŒÜŸ	š	Ø× Ñ ×,Ñ,¬s°1°v×7NÑ7NÑ3NÓ/OÐSTÑ/TÑTÐV\×VhÑVh×VtÑVtó
ˆŒô ˜&×5Ñ5Ñ6ˆŒÜŸ	š	 &×"4Ñ"4×"@Ñ"@À&×BTÑBT×B`ÑB`Óaˆ�r<   c                 óŽ   • U R                  U5      nU R                  U5      nU R                  U5      nU R                  U5      nU$ rt   )rÌ   r+  r-  r.  )rW   Úimage_featuresr[   s      r:   rr   Ú#InternVLMultiModalProjector.forward¸  s@   € ØŸ™¨Ó7ˆØŸ™ mÓ4ˆØŸ™ Ó/ˆØŸ™ mÓ4ˆØÐr<   )r-  rÌ   r+  r.  )	rB   rC   rD   rE   r   rO   rr   rF   rv   rw   s   @r:   r%  r%  ®  s   ø† ðb˜~÷ b÷ð r<   r%  c                   ó   • \ rS rSrSrg)ÚInternVLModelOutputWithPastiÀ  r@   NrA   r@   r<   r:   r4  r4  À  rG   r<   r4  c                   óà  • \ rS rSrSS\R
                  S\4S jjr\\	\
" SS9  SS\R                  S	\\\   -  \\   -  S-  S
\S-  S\\   S\\-  4
S jj5       5       5       r\	\
        SS\R(                  S-  S\R                  S-  S\R
                  S-  S\R(                  S-  S\S-  S\R                  S-  S	\\\   -  \\   -  S-  S
\S-  S\\   S\\-  4S jj5       5       rSrg)ÚInternVLModeliÄ  Úvision_featuresÚscale_factorc           
      ó¬  • UR                  5       u  p4pVXR-  S:w  d  XB-  S:w  a  [        S5      eUR                  X4[        XR-  5      [        Xb-  5      5      nUR	                  SSSS5      R                  5       nUR                  U[        XR-  5      [        XB-  5      [        XbS-  -  5      5      nUR	                  SSSS5      R                  5       nU$ )aÞ  Perform pixel shuffle downsampling on vision features.

Args:
    vision_features (`torch.Tensor`):
        Input tensor of shape (batch_size, width, height, channels).
    scale_factor (`float`, *optional*, defaults to `0.5`):
        Factor by which to downsample. Default is 0.5, which halves the dimensions.

Returns:
    vision_features (`torch.Tensor`):
        Downsampled tensor of shape (batch_size, height*scale_factor, width*scale_factor, channels/(scale_factor^2)).
r   zKHeight and width must be divisible by scale_factor for proper downsampling.r   r   r   )r^   r‘   re   rÈ   r´   r4   )rW   r7  r8  rl   r—   r–   Úchannelss          r:   Úpixel_shuffleÚInternVLModel.pixel_shuffleÅ  sã   € ð />×.BÑ.BÓ.DÑ+ˆ
˜6àÑ  AÓ%¨Ñ)=ÀÓ)BÜÐjÓkÐkð *×.Ñ.Øœs 6Ñ#8Ó9¼3¸xÑ?VÓ;Wó
ˆð *×1Ñ1°!°Q¸¸1Ó=×HÑHÓJˆð *×.Ñ.Øœ˜FÑ1Ó2´C¸Ñ8LÓ4MÌsÐS[ÐmnÑ_nÑSoÓOpó
ˆð
 *×1Ñ1°!°Q¸¸1Ó=×HÑHÓJˆàÐr<   zWObtains image last hidden states from the vision tower and apply multimodal projection.rx   Nr�   Úvision_feature_layerÚvision_feature_select_strategyr5   rŽ   c                 ó  • UR                  U R                  S9nU R                  R                  nUS:w  a  SUS'   U R                  " SUSS.UD6nUS:X  a  UR
                  nOUR                  U   nUS:X  a  USS2SS2SS24   nUR                  S   n[        US	-  5      n	UR                  S
   n
UR                  X©U	S5      nU R                  XuS9nUR                  U
SUR                  S   5      nU R                  U5      nXvl        U$ )zø
pixel_values (`torch.FloatTensor` of shape `(batch_size, channels, height, width)`)
    The tensors corresponding to the input images.
vision_feature_layer (`int` or `list[int]`):
    Layer index or list of layer indices to extract features from.
)r”   r)   TÚoutput_hidden_states)r�   Úreturn_dictÚdefaultNr   r®   r   )r8  r@   )r’   r”   rK   r(  Úvision_towerrò   r[   r�   rÈ   rb   r;  Úmulti_modal_projectorÚpooler_output)rW   r�   r=  r>  r5   r(  Úvision_outputsr7  r:  Úfeature_sizerl   s              r:   Úget_image_featuresÚ InternVLModel.get_image_featuresè  s1  € ð$ $—‘¨T¯Z©Z�Ð8ˆàŸ;™;×7Ñ7ÐØ 2Ó%Ø-1ˆFÐ)Ñ*Ø×*Ò*Ða¸ÐRVÑaÐZ`ÑaˆØ 2Ó%Ø,×>Ñ>‰Oà,×:Ñ:Ð;OÑPˆOØ)¨YÓ6Ø-ªa°±²Q¨hÑ7ˆOð #×(Ñ(¨Ñ+ˆÜ˜8 S™=Ó)ˆØ$×*Ñ*¨1Ñ-ˆ
ð *×1Ñ1°*ÈLÐZ\Ó]ˆð ×,Ñ,¨_Ð,Ð\ˆð *×1Ñ1°*¸bÀ/×BWÑBWÐXZÑB[Ó\ˆð ×4Ñ4°_ÓEˆØ'6Ô$àÐr<   Ú	input_idsr%   Úposition_idsÚpast_key_valuesÚinputs_embedsc	                 óØ  • US L US L-  (       a  [        S5      eUc  U R                  5       " U5      nUbc  U R                  UUUSS9R                  n
U
R	                  UR
                  UR                  5      n
U R                  XU
S9nUR                  Xº5      nU R                  " SUUUUS.U	D6n[        UR                  UR                  UR                  UR                  Ub  W
S9$ S S9$ )Nz:You must specify exactly one of input_ids or inputs_embedsT)r�   r=  r>  rA  )rM  r1  )r%   rK  rL  rM  )rò   rL  r[   rû   Úimage_hidden_statesr@   )r‘   r  rH  rE  r’   Údevicer”   Úget_placeholder_maskÚmasked_scatterÚlanguage_modelr4  rò   rL  r[   rû   )rW   rJ  r�   r%   rK  rL  rM  r=  r>  r5   r1  Úspecial_image_maskÚoutputss                r:   rr   ÚInternVLModel.forward  s1  € ð ˜Ð -°tÐ";×<ÜÐYÓZÐZàÑ Ø ×5Ñ5Ô7¸	ÓBˆMàÑ#Ø!×4Ñ4Ø)Ø%9Ø/MØ ð	 5ð ÷
 ‰mð ð ,×.Ñ.¨}×/CÑ/CÀ]×EXÑEXÓYˆNØ!%×!:Ñ!:ØÀ~ð ";ð "Ðð *×8Ñ8Ð9KÓ\ˆMà×%Ò%ð 
Ø)Ø%Ø+Ø'ñ	
ð
 ñ
ˆô +Ø%×7Ñ7Ø#×3Ñ3Ø!×/Ñ/Ø×)Ñ)Ø2>Ñ2J ñ
ð 	
ð
 QUñ
ð 	
r<   r@   )r®   )NN)NNNNNNNN)rB   rC   rD   rE   r.   ru   Úfloatr;  r   r   r   ÚFloatTensorrÈ   ÚlistÚstrr   r   rõ   r   rH  Ú
LongTensorr   r4  rr   rF   r@   r<   r:   r6  r6  Ä  s¤  † ñ!¨U¯\©\ð !Èõ !ðF  ØÙØnñð DHØ59ñ	,à×'Ñ'ð,ð " D¨¡I™o°°S±	Ñ9¸DÑ@ð,ð ),¨d©
ð	,ð
 Ð+Ñ,ð,ð 
Ð+Ñ	+ô,óó ó  ð
,ð\ Øð .2Ø15Ø.2Ø04Ø(,Ø26ØCGØ59ñ-
à×#Ñ# dÑ*ð-
ð ×'Ñ'¨$Ñ.ð-
ð Ÿ™ tÑ+ð	-
ð
 ×&Ñ&¨Ñ-ð-
ð  ™ð-
ð ×(Ñ(¨4Ñ/ð-
ð " D¨¡I™o°°S±	Ñ9¸DÑ@ð-
ð ),¨d©
ð-
ð Ð+Ñ,ð-
ð 
Ð,Ñ	,ô-
ó ó ó-
r<   r6  c                   ó   • \ rS rSrSrg)ÚInternVLCausalLMOutputWithPastiM  r@   NrA   r@   r<   r:   r]  r]  M  rG   r<   r]  c                   ó(   ^ • \ rS rSrU 4S jrSrU =r$ )Ú InternVLForConditionalGenerationiQ  c                  ó:   >• [        5       R                  " S0 U D6  g)as  
Example:

```python
>>> import torch
>>> from transformers import AutoProcessor, AutoModelForImageTextToText

>>> torch_device = "cuda"
>>> processor = AutoProcessor.from_pretrained("OpenGVLab/InternVL3-1B-hf")
>>> model = AutoModelForImageTextToText.from_pretrained(
...     "OpenGVLab/InternVL3-1B-hf", dtype=torch.bfloat16, device_map=torch_device
... )

>>> messages = [
...     {
...         "role": "user",
...         "content": [
...             {
...                 "type": "image",
...                 "url": "https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg",
...             },
...             {
...                 "type": "image",
...                 "url": "https://thumbs.dreamstime.com/b/golden-gate-bridge-san-francisco-purple-flowers-california-echium-candicans-36805947.jpg",
...             },
...             {"type": "text", "text": "These images depict two different landmarks. Can you identify them?"},
...         ],
...     },
... ]

>>> inputs = processor.apply_chat_template(messages, add_generation_prompt=True, tokenize=True, return_dict=True, return_tensors="pt").to(torch_device)
>>> generate_ids = model.generate(**inputs, max_new_tokens=200)
>>> print(processor.decode(generate_ids[0, inputs["input_ids"].shape[1] :], skip_special_tokens=True))
The images depict the Statue of Liberty and the Golden Gate Bridge.
```Nr@   )rN   rr   )Úsuper_kwargsrY   s    €r:   rr   Ú(InternVLForConditionalGeneration.forwardR  s   ø€ ôH 	‹�ŠÑ'˜,Ó'r<   r@   )rB   rC   rD   rE   rr   rF   rv   rw   s   @r:   r_  r_  Q  s   ø† ÷$(ó $(r<   r_  )r÷   r  r"  r6  r_  )r]   )JÚcollections.abcr¥   r   Údataclassesr   r.   Útorch.nnr1   Ú r   rþ   Úactivationsr   Úcache_utilsr   Úmodeling_layersr	   Úmodeling_outputsr
   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   Úutils.genericr   r   Úutils.output_capturingr   Úclip.modeling_clipr   Újanus.modeling_janusr   Úllama.modeling_llamar   Úllava.modeling_llavar   r   r   r   r   Úconfiguration_internvlr   r    ÚModuleru   rW  rÈ   r;   r>   rI   r{   r   r›   rË   r  rØ   rÏ   rè   r÷   r  r"  ÚINTERNVL_INPUTS_DOCSTRINGr%  r4  r6  r]  r_  Ú__all__r@   r<   r:   Ú<module>rx     s  ðó  Ý $Ý !ã Ý å &Ý !Ý  Ý 9ß Kß FÝ &ß BÑ Bß IÝ 5Ý (Ý 7Ý /÷õ ÷ Ið ñ%Ø�I‰Ið%à�<‰<ð%ð 
�‰ð%ð �<‰<ð	%ð
 —L‘L 4Ñ'ð%ð ð%ð �S‰[õ%ô4	˜Lô 	ô3$Ð2ô 3$ñl ðñð
 ôÐ+Eó ó óðô  B§I¡Iô  ôJ[˜rŸy™yô [ô|	˜ô 	ð Ÿ™Ð3HÑ
I€ô+Ð4ô +ô\
˜BŸI™Iô 
ð& ôP Oó Pó ðPð@ ô&
Ð7ó &
ó ð&
ôR2Ð2ô 2ð !Ð ô "§)¡)ô ô$	Ð":ô 	ôF
�Jô F
ôR	Ð%@ô 	ô%(Ð'Dô %(òP�r<   