ó
    ˆŽ¡j¦`  ã                   óx  • S SK Jr  S SKrS SKJr  SSKJr  SSKJ	r	  SSK
Jr  SSKJrJr  SSKJrJr  SS	KJr  SS
KJrJrJr  SSKJr  SSKJrJr  SSKJr  SSKJ r    " S S\RB                  5      r" " S S\RB                  5      r# " S S\RB                  5      r$ S5S\RB                  S\RJ                  S\RJ                  S\RJ                  S\RJ                  S-  S\&S\&S\\   4S jjr'S r(S \RJ                  S!\)S"\RJ                  4S# jr*S$\RJ                  S%\RJ                  S&\RJ                  S'\RJ                  S"\+\RJ                  \RJ                  4   4
S( jr, " S) S*\RB                  5      r- " S+ S,\5      r. " S- S.\RB                  5      r/\ " S/ S0\5      5       r0\" S1S29 " S3 S4\05      5       r1S0S4/r2g)6é    )ÚCallableNé   )Úinitialization)ÚACT2FN)ÚGradientCheckpointingLayer)ÚBaseModelOutputÚBaseModelOutputWithPooling)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚ	torch_int)Údeprecate_kwarg)Úmaybe_autocastÚmerge_with_config_defaults)Úcapture_outputsé   )ÚMLCDVisionConfigc                   ób   ^ • \ rS rSrU 4S jrS\R                  S\R                  4S jrSrU =r	$ )ÚMLCDMLPé&   c                 ó  >• [         TU ]  5         Xl        [        UR                     U l        [        R                  " UR                  UR                  5      U l
        [        R                  " UR                  UR                  5      U l        g ©N)ÚsuperÚ__init__Úconfigr   Ú
hidden_actÚactivation_fnÚnnÚLinearÚhidden_sizeÚintermediate_sizeÚfc1Úfc2©Úselfr   Ú	__class__s     €Ú´C:\Users\johnn\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.13_qbz5n2kfra8p0\LocalCache\local-packages\Python313\site-packages\transformers/models/mlcd/modeling_mlcd.pyr   ÚMLCDMLP.__init__'   sb   ø€ Ü‰ÑÔØŒÜ# F×$5Ñ$5Ñ6ˆÔÜ—9’9˜V×/Ñ/°×1IÑ1IÓJˆŒÜ—9’9˜V×5Ñ5°v×7IÑ7IÓJˆ�ó    Úhidden_statesÚreturnc                 ól   • U R                  U5      nU R                  U5      nU R                  U5      nU$ r   )r$   r   r%   )r'   r,   s     r)   ÚforwardÚMLCDMLP.forward.   s4   € ØŸ™ Ó/ˆØ×*Ñ*¨=Ó9ˆØŸ™ Ó/ˆØÐr+   )r   r   r$   r%   )
Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r   ÚtorchÚTensorr/   Ú__static_attributes__Ú__classcell__©r(   s   @r)   r   r   &   s)   ø† õKð U§\¡\ð °e·l±l÷ ò r+   r   c            	       óÒ   ^ • \ rS rSrSr\" SSS9SS\4U 4S jjj5       r\\" SSS9SS\S\	\
R                  \4   4S	 jj5       5       r\
R                  " 5       S
 5       rS rSrU =r$ )ÚMLCDRotaryEmbeddingé5   zí
Simple axial 2D rope with same freqs used for H and W grids. The freqs are
pre-computed using `head-dim//4` which is later used to concat H and W positions.
The final angles rotate over the whole head dim, no partial rotation involved.
Údevicez5.18)Úversionr   c                 ó¶  >• [         TU ]  5         Xl        U R                  R                  S   U l        U R
                  nU R                  S:w  a.  [        U R                  R                   SU R                   35      eU" U R                  U5      u  o@l	        [        R                  " USS9U l        [        R                  " UR                  5       SS9U l        g )NÚ	rope_typeÚaxialz) supports only axial rope, but requested F©Ú
persistent)r   r   r   Úrope_parametersr@   Úcompute_axial_rope_parametersÚ
ValueErrorr(   r1   Úattention_scalingr    ÚBufferÚinv_freqÚcloneÚoriginal_inv_freq)r'   r   r=   Úrope_init_fnrI   r(   s        €r)   r   ÚMLCDRotaryEmbedding.__init__<   s¬   ø€ ä‰ÑÔØŒàŸ™×4Ñ4°[ÑAˆŒØ!%×!CÑ!CˆØ�>‰>˜WÓ$Ü §¡× 7Ñ 7Ð8Ð8aÐbf×bpÑbpÐaqÐrÓsÐsÙ+7¸¿¹ÀVÓ+LÑ(ˆÔ(äŸ	š	 (°uÑ=ˆŒÜ!#§¢¨8¯>©>Ó+;ÈÑ!NˆÕr+   r-   c           	      ó
  • U R                   S   n[        U SS5      =(       d    U R                  U R                  -  nUS-  nSnSU[        R
                  " SUS[        R                  S9U-  -  -  nUR                  U5      U4$ )a~  
Computes the inverse frequencies according to the original RoPE implementation
Args:
    config ([`~transformers.PreTrainedConfig`]):
        The model configuration.
Returns:
    Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
    post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
Ú
rope_thetaÚhead_dimNé   g      ð?r   ©Údtype)rD   Úgetattrr"   Únum_attention_headsr5   ÚarangeÚfloatÚto)r   r=   ÚkwargsÚbaseÚdimÚspatial_dimÚattention_factorrI   s           r)   rE   Ú1MLCDRotaryEmbedding.compute_axial_rope_parametersJ   s„   € ð ×%Ñ% lÑ3ˆÜ�f˜j¨$Ó/×c°6×3EÑ3EÈ×IcÑIcÑ3cˆØ˜Q‘hˆàÐØ˜$¤5§<¢<°°;ÀÌÏÉÑ#UÐXcÑ#cÑdÑeˆØ�{‰{˜6Ó"Ð$4Ð4Ð4r+   c                 ó  • US   R                  5       n[        UR                  R                  [        5      (       a0  UR                  R                  S:w  a  UR                  R                  OSn[        USS9   X0R                  R                  5       -  nUR                  5       U R                  -  nUR                  5       U R                  -  nS S S 5        U R                  W5      nU R                  W5      nXg4$ ! , (       d  f       N3= f)N).NÚmpsÚcpuF)Údevice_typeÚenabled)rW   Ú
isinstancer=   ÚtypeÚstrr   rI   ÚcosrG   ÚsinÚrecomposition_frequencies)r'   ÚxÚposition_idsÚposition_ids_expandedrb   Úfreqsrg   rh   s           r)   r/   ÚMLCDRotaryEmbedding.forward^   sÐ   € ð !-¨YÑ 7× =Ñ =Ó ?ÐÜ'1°!·(±(·-±-Ä×'EÑ'EÈ!Ï(É(Ï-É-Ð[`ÓJ`�a—h‘h—m’mÐfkˆÜ¨¸UÓCØ)¯M©M×,?Ñ,?Ó,AÑAˆEØ—)‘)“+ × 6Ñ 6Ñ6ˆCØ—)‘)“+ × 6Ñ 6Ñ6ˆC÷ Dð
 ×,Ñ,¨SÓ1ˆØ×,Ñ,¨SÓ1ˆØˆxˆ÷ DÕCús   Á9AC=Ã=
Dc                 ó|   • USS2S4   USS2S4   p2[         R                  " X#/SS9n[         R                  " XD/SS9$ )zM
Recompose the frequencies into the final spatial layout used per each grid.
Nr   r   éÿÿÿÿ©r[   )r5   Úcat)r'   ÚfreqÚfreq_hÚfreq_wÚfreq_hws        r)   ri   Ú-MLCDRotaryEmbedding.recomposition_frequenciesl   sB   € ð ša ˜d™ Tª!¨Q¨$¡Z�Ü—)’)˜VÐ,°"Ñ5ˆÜ�yŠy˜'Ð+°Ñ4Ð4r+   )rG   r   rI   rK   r@   r   )r1   r2   r3   r4   Ú__doc__r   r   r   ÚstaticmethodÚtupler5   r6   rW   rE   Úno_gradr/   ri   r7   r8   r9   s   @r)   r;   r;   5   s—   ø† ññ �X vÑ.ñOÐ/÷ Oó /ðOð Ù�X vÑ.ñ5Ð.>ð 5ÐZ_Ð`e×`lÑ`lÐnsÐ`sÑZtô 5ó /ó ð5ð$ ‡]‚]ƒ_ñó ð÷5ð 5r+   r;   c                   ó¬   ^ • \ rS rSrS\4U 4S jjrS\R                  S\S\S\R                  4S jr	S	\R                  S\R                  4S
 jrSrU =r$ )ÚMLCDVisionEmbeddingséu   r   c                 óˆ  >• [         TU ]  5         Xl        UR                  U l        UR
                  U l        UR                  U l        [        R                  " [        R                  " U R                  5      5      U l        [        R                  " UR                  U R                  U R                  U R                  SS9U l        U R
                  U R                  -  S-  U l        U R                  S-   U l        [        R"                  " [        R$                  " U R                   5      R'                  S5      SS9U l        g )NF)Úin_channelsÚout_channelsÚkernel_sizeÚstrideÚbiasrQ   r   ©r   rp   rB   )r   r   r   r"   Ú	embed_dimÚ
image_sizeÚ
patch_sizer    Ú	Parameterr5   ÚrandnÚclass_embeddingÚConv2dÚnum_channelsÚpatch_embeddingÚnum_patchesÚnum_positionsrH   rV   Úexpandrk   r&   s     €r)   r   ÚMLCDVisionEmbeddings.__init__v   sç   ø€ Ü‰ÑÔØŒØ×+Ñ+ˆŒØ ×+Ñ+ˆŒØ ×+Ñ+ˆŒä!Ÿ|š|¬E¯KªK¸¿¹Ó,GÓHˆÔä!ŸyšyØ×+Ñ+ØŸ™ØŸ™Ø—?‘?Øñ 
ˆÔð !ŸO™O¨t¯©Ñ>À1ÑDˆÔØ!×-Ñ-°Ñ1ˆÔÜŸIšI¤e§l¢l°4×3EÑ3EÓ&F×&MÑ&MÈgÓ&VÐchÑiˆÕr+   Ú
embeddingsÚheightÚwidthr-   c                 óÈ  • UR                   S   S-
  nU R                  R                  R                  S5      nUR                   S   S-
  n[        R
                  R                  5       (       d%  XF:X  a   X#:X  a  U R                  U R                  5      $ USS2SS24   nUSS2SS24   nUR                   S   n	X R                  -  n
X0R                  -  n[        US-  5      nUR                  SXÌU	5      nUR                  SSSS5      n[        R                  R                  UX«4SS	S
9nUR                  SSSS5      R                  SSU	5      n[        R                   " Xx4SS9$ )aÐ  
This method allows to interpolate the pre-trained position encodings, to be able to use the model on higher resolution
images. This method is also adapted to support torch.jit tracing.

Adapted from:
- https://github.com/facebookresearch/dino/blob/de9ee3df6cf39fac952ab558447af1fa1365362a/vision_transformer.py#L174-L194, and
- https://github.com/facebookresearch/dinov2/blob/e1277af2ba9496fbadf7aec6eba56e8d882d1e35/dinov2/models/vision_transformer.py#L179-L211
r   r   Nrp   g      à?r   rQ   ÚbicubicF)ÚsizeÚmodeÚalign_cornersrq   )ÚshapeÚposition_embeddingÚweightÚ	unsqueezer5   ÚjitÚ
is_tracingrk   rˆ   r   ÚreshapeÚpermuter    Ú
functionalÚinterpolateÚviewrr   )r'   r“   r”   r•   r�   rœ   r�   Úclass_pos_embedÚpatch_pos_embedr[   Ú
new_heightÚ	new_widthÚsqrt_num_positionss                r)   Úinterpolate_pos_encodingÚ-MLCDVisionEmbeddings.interpolate_pos_encoding‹   si  € ð !×&Ñ& qÑ)¨AÑ-ˆØ!×4Ñ4×;Ñ;×EÑEÀaÓHÐØ*×0Ñ0°Ñ3°aÑ7ˆô �y‰y×#Ñ#×%Ñ%¨+Ó*FÈ6Ë?Ø×*Ñ*¨4×+<Ñ+<Ó=Ð=à,ªQ°°°¨UÑ3ˆØ,ªQ°±¨UÑ3ˆà×Ñ˜rÑ"ˆàŸ™Ñ.ˆ
ØŸ_™_Ñ,ˆ	ä& }°cÑ'9Ó:ÐØ)×1Ñ1°!Ð5GÐ]`ÓaˆØ)×1Ñ1°!°Q¸¸1Ó=ˆäŸ-™-×3Ñ3ØØÐ(ØØð	 4ð 
ˆð *×1Ñ1°!°Q¸¸1Ó=×BÑBÀ1ÀbÈ#ÓNˆä�yŠy˜/Ð;ÀÑCÐCr+   Úpixel_valuesc                 óH  • UR                   S   nU R                  R                  R                  nU R                  UR	                  US95      nUR                  S5      R                  SS5      nU R                  R                  USS5      n[        R                  " XT/SS9nU$ )Nr   rR   rQ   r   rp   rq   )r›   rŽ   r�   rS   rX   ÚflattenÚ	transposer‹   r‘   r5   rr   )r'   r­   Ú
batch_sizeÚtarget_dtypeÚpatch_embedsÚclass_embedsr“   s          r)   r/   ÚMLCDVisionEmbeddings.forward´   s”   € Ø!×'Ñ'¨Ñ*ˆ
Ø×+Ñ+×2Ñ2×8Ñ8ˆà×+Ñ+¨L¯O©OÀ,¨OÐ,OÓPˆØ#×+Ñ+¨AÓ.×8Ñ8¸¸AÓ>ˆà×+Ñ+×2Ñ2°:¸qÀ"ÓEˆÜ—Y’Y Ð;ÀÑCˆ
àÐr+   )	r‹   r   r†   r‡   r�   r�   rŽ   rˆ   rk   )r1   r2   r3   r4   r   r   r5   r6   Úintr«   ÚFloatTensorr/   r7   r8   r9   s   @r)   r}   r}   u   sg   ø† ðjÐ/÷ jð*'D°5·<±<ð 'DÈð 'DÐUXð 'DÐ]b×]iÑ]iô 'DðR
 E×$5Ñ$5ð 
¸%¿,¹,÷ 
ò 
r+   r}   ÚmoduleÚqueryÚkeyÚvalueÚattention_maskÚscalingÚdropoutrY   c                 ó  • [        X R                  5      n[        X0R                  5      n	[        R                  " XR	                  SS5      5      U-  n
Ub  X¤-   n
[
        R                  R                  U
S[        R                  S9R                  UR                  5      n
[
        R                  R                  X¦U R                  S9n
[        R                  " X©5      nUR	                  SS5      R                  5       nXº4$ )NrQ   r   rp   )r[   rS   )ÚpÚtrainingr   )Ú	repeat_kvÚnum_key_value_groupsr5   Úmatmulr°   r    r£   ÚsoftmaxÚfloat32rX   rS   r¾   rÁ   Ú
contiguous)r¸   r¹   rº   r»   r¼   r½   r¾   rY   Ú
key_statesÚvalue_statesÚattn_weightsÚattn_outputs               r)   Úeager_attention_forwardrÌ   Á   sÒ   € ô ˜3× ;Ñ ;Ó<€JÜ˜U×$?Ñ$?Ó@€Lä—<’< ×';Ñ';¸A¸qÓ'AÓBÀWÑL€LØÑ!Ø#Ñ4ˆä—=‘=×(Ñ(¨¸2ÄUÇ]Á]Ð(ÐS×VÑVÐW\×WbÑWbÓc€LÜ—=‘=×(Ñ(¨È6Ï?É?Ð(Ð[€LÜ—,’,˜|Ó:€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€KàÐ$Ð$r+   c                 ó–   • U SSU R                   S   S-  24   nU SU R                   S   S-  S24   n[        R                  " U* U4SS9$ )z*Rotates half the hidden dims of the input..Nrp   rQ   rq   )r›   r5   rr   )rj   Úx1Úx2s      r)   Úrotate_halfrÐ   Ú   sZ   € à	
ˆ3Ð"�!—'‘'˜"‘+ Ñ"Ð"Ð"Ñ	#€BØ	
ˆ3�—‘˜‘˜qÑ Ñ"Ð"Ñ	#€BÜ�9Š9�r�c˜2�Y BÑ'Ð'r+   r,   Ún_repr-   c                 ó    • U R                   u  p#pEUS:X  a  U $ U SS2SS2SSS2SS24   R                  X#XU5      n U R                  X#U-  XE5      $ )zÈ
This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
r   N)r›   r‘   r¡   )r,   rÑ   ÚbatchÚnum_key_value_headsÚslenrP   s         r)   rÂ   rÂ   á   s_   € ð
 2?×1DÑ1DÑ.€E Ø�ƒzØÐØ!¢!¢Q¨ªa²Ð"2Ñ3×:Ñ:¸5ÐW\ÐdlÓm€MØ× Ñ  ¸eÑ(CÀTÓTÐTr+   ÚqÚkrg   rh   c                 ó€  • U R                   nUR                   nU R                  5       UR                  5       pUR                  S5      R                  5       UR                  S5      R                  5       p2X-  [        U 5      U-  -   nX-  [        U5      U-  -   nUR	                  U5      nUR	                  U5      nXg4$ )Néþÿÿÿ)rS   rW   rž   rÐ   rX   )rÖ   r×   rg   rh   Úorig_q_dtypeÚorig_k_dtypeÚq_embedÚk_embeds           r)   Úapply_rotary_pos_emb_visionrÞ   í   s£   € ð —7‘7€LØ—7‘7€LØ�7‰7‹9�a—g‘g“i€qØ�}‰}˜RÓ ×&Ñ&Ó(¨#¯-©-¸Ó*;×*AÑ*AÓ*CˆØ‰wœ; q›>¨CÑ/Ñ0€GØ‰wœ; q›>¨CÑ/Ñ0€GØ�j‰j˜Ó&€GØ�j‰j˜Ó&€GØÐÐr+   c                   óö   ^ • \ rS rSrSrS\4U 4S jjr SS\R                  S\	\R                  \R                  4   S\R                  S-  S	\
\   S
\	\R                  \R                  S-  4   4
S jjrSrU =r$ )ÚMLCDAttentionéû   zëMulti-headed attention with RoPE. Refer to papers:
- Attention is all you need:
    https://huggingface.co/papers/1706.03762
- RoFormer: Enhanced Transformer with Rotary Position Embedding:
    https://huggingface.co/papers/2104.09864
r   c                 ó¨  >• [         TU ]  5         Xl        UR                  U l        UR
                  U l        U R                  U R                  -  U l        U R                  S-  U l        UR                  U l
        SU l        [        R                  " U R                  U R                  5      U l        [        R                  " U R                  U R                  5      U l        [        R                  " U R                  U R                  5      U l        [        R                  " U R                  U R                  5      U l        UR$                  U l        g )Nç      à¿F)r   r   r   r"   r†   rU   Ú	num_headsrP   ÚscaleÚattention_dropoutr¾   Ú	is_causalr    r!   Úk_projÚv_projÚq_projÚout_projrÃ   r&   s     €r)   r   ÚMLCDAttention.__init__  sÛ   ø€ Ü‰ÑÔØŒØ×+Ñ+ˆŒØ×3Ñ3ˆŒØŸ™¨$¯.©.Ñ8ˆŒØ—]‘] DÑ(ˆŒ
Ø×/Ñ/ˆŒØˆŒä—i’i §¡°·±Ó?ˆŒÜ—i’i §¡°·±Ó?ˆŒÜ—i’i §¡°·±Ó?ˆŒÜŸ	š	 $§.¡.°$·.±.ÓAˆŒØ$*×$?Ñ$?ˆÕ!r+   Nr,   Úposition_embeddingsr¼   rY   r-   c                 ó–  • UR                   SS u  pVU R                  U5      R                  XVU R                  U R                  45      nU R                  U5      R                  XVU R                  U R                  45      nU R                  U5      R                  XVU R                  U R                  45      n	US   R                  S5      R                  5       n
US   R                  S5      R                  5       n[        XxX«5      u  pxUR                  SSSS5      R                  5       nUR                  SSSS5      R                  5       nU	R                  SSSS5      R                  5       n	[        R                  " U R                  R                  [         5      nU" U UUU	U4U R"                  (       d  SOU R$                  U R&                  U R(                  S.UD6u  pÞUR                  SSSS5      R                  5       nUR+                  XeS5      nU R-                  U5      nUR                  SSS5      R                  5       nXÞ4$ )	z#Input shape: Batch x Time x ChannelNrp   r   r   rQ   r   ç        )r¾   r½   rç   )r›   rê   r¡   rä   rP   rè   ré   rž   rW   rÞ   r¢   rÇ   r
   Úget_interfacer   Ú_attn_implementationrÌ   rÁ   r¾   rå   rç   r¥   rë   )r'   r,   rí   r¼   rY   r±   Ú
seq_lengthÚquery_statesrÈ   rÉ   rg   rh   Úattention_interfacerË   rÊ   s                  r)   r/   ÚMLCDAttention.forward  s&  € ð "/×!4Ñ!4°S°bÐ!9Ñˆ
ð —{‘{ =Ó1×9Ñ9¸:ÐSW×SaÑSaÐcg×cpÑcpÐ:qÓrˆØ—[‘[ Ó/×7Ñ7¸ÐQU×Q_ÑQ_Ðae×anÑanÐ8oÓpˆ
Ø—{‘{ =Ó1×9Ñ9¸:ÐSW×SaÑSaÐcg×cpÑcpÐ:qÓrˆð " !Ñ$×.Ñ.¨qÓ1×7Ñ7Ó9ˆØ! !Ñ$×.Ñ.¨qÓ1×7Ñ7Ó9ˆÜ#>¸|ÐY\Ó#bÑ ˆð $×+Ñ+¨A¨q°!°QÓ7×BÑBÓDˆØ×'Ñ'¨¨1¨a°Ó3×>Ñ>Ó@ˆ
Ø#×+Ñ+¨A¨q°!°QÓ7×BÑBÓDˆä(?×(MÒ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØð
%
ð  $Ÿ}Ÿ}‘C°$·,±,Ø—J‘JØ—n‘nñ
%
ð ñ
%
Ñ!ˆð "×)Ñ)¨!¨Q°°1Ó5×@Ñ@ÓBˆØ!×&Ñ& z¸rÓBˆØ—m‘m KÓ0ˆØ!×)Ñ)¨!¨Q°Ó2×=Ñ=Ó?ˆØÐ(Ð(r+   )r   r¾   r†   rP   rç   rè   rä   rÃ   rë   rê   rå   ré   r   )r1   r2   r3   r4   rx   r   r   r5   r6   rz   r   r   r/   r7   r8   r9   s   @r)   rà   rà   û   s’   ø† ñð@Ð/÷ @ð( /3ñ	-)à—|‘|ð-)ð # 5§<¡<°·±Ð#=Ñ>ð-)ð Ÿ™ tÑ+ð	-)ð
 Ð+Ñ,ð-)ð 
ˆu�|‰|˜UŸ\™\¨DÑ0Ð0Ñ	1÷-)ó -)r+   rà   c                   óÔ   ^ • \ rS rSrS\4U 4S jjr SS\R                  S\\R                  \R                  4   S\R                  S-  S\	\
   S	\\R                     4
S
 jjrSrU =r$ )ÚMLCDEncoderLayeriC  r   c                 ó<  >• [         TU ]  5         UR                  U l        [	        U5      U l        [        R                  " U R                  UR                  S9U l	        [        U5      U l        [        R                  " U R                  UR                  S9U l        g )N©Úeps)r   r   r"   r†   rà   Ú	self_attnr    Ú	LayerNormÚlayer_norm_epsÚlayer_norm1r   ÚmlpÚlayer_norm2r&   s     €r)   r   ÚMLCDEncoderLayer.__init__D  sl   ø€ Ü‰ÑÔØ×+Ñ+ˆŒÜ& vÓ.ˆŒÜŸ<š<¨¯©¸F×<QÑ<QÑRˆÔÜ˜6“?ˆŒÜŸ<š<¨¯©¸F×<QÑ<QÑRˆÕr+   Nr,   rí   r¼   rY   r-   c                 ó´   • UnU R                  U5      nU R                  " SUUUS.UD6u  pXQ-   nUnU R                  U5      nU R                  U5      nXQ-   nU$ )ah  
Args:
    hidden_states (`torch.FloatTensor`):
        Input to the layer of shape `(batch, seq_len, embed_dim)`.
        Represents the hidden states from the previous layer or the input embeddings.
    position_embeddings (`tuple[torch.Tensor, torch.Tensor]`):
        A tuple of two tensors, each of shape `(batch, seq_len, embed_dim)`.
        Represents absolute positional embeddings for the query and key in the attention mechanism.
    attention_mask (`torch.FloatTensor`):
        Attention mask of shape `(batch, 1, q_len, k_v_seq_len)` where padding elements are indicated by very large negative values.
)r,   rí   r¼   © )rþ   rû   r   rÿ   )r'   r,   rí   r¼   rY   ÚresidualÚ_s          r)   r/   ÚMLCDEncoderLayer.forwardL  s}   € ð$ !ˆà×(Ñ(¨Ó7ˆØŸ>š>ð 
Ø'Ø 3Ø)ñ
ð ñ	
Ñˆð !Ñ0ˆà ˆØ×(Ñ(¨Ó7ˆØŸ™ Ó/ˆØ Ñ0ˆàÐr+   )r†   rþ   r   rÿ   rû   r   )r1   r2   r3   r4   r   r   r5   r6   rz   r   r   r·   r/   r7   r8   r9   s   @r)   r÷   r÷   C  s�   ø† ðSÐ/÷ Sð /3ñ	"à—|‘|ð"ð # 5§<¡<°·±Ð#=Ñ>ð"ð Ÿ™ tÑ+ð	"ð
 Ð+Ñ,ð"ð 
ˆu× Ñ Ñ	!÷"ó "r+   r÷   c                   óÄ   ^ • \ rS rSrSrS\4U 4S jjr SS\R                  S\	\R                  \R                  4   S\R                  S-  S	\\   S
\	\-  4
S jjrSrU =r$ )ÚMLCDEncoderiq  zŸ
Transformer encoder consisting of `config.num_hidden_layers` self attention layers. Each layer is a
[`MLCDEncoderLayer`].

Args:
    config: MLCDVisionConfig
r   c                 óÖ   >• [         TU ]  5         Xl        [        R                  " [        UR                  5       Vs/ s H  n[        U5      PM     sn5      U l        SU l	        gs  snf )z3Overwrite dummy `MLCDConfig` to `MLCDVisionConfig`.FN)
r   r   r   r    Ú
ModuleListÚrangeÚnum_hidden_layersr÷   ÚlayersÚgradient_checkpointing)r'   r   r  r(   s      €r)   r   ÚMLCDEncoder.__init__z  sS   ø€ ä‰ÑÔØŒÜ—m’mÄuÈV×MeÑMeÔGfÓ$gÒGfÀ!Ô%5°fÖ%=ÑGfÑ$gÓhˆŒØ&+ˆÕ#ùò %hs   ½A&NÚinputs_embedsrí   r¼   rY   r-   c                 óT   • UnU R                    H  nU" UUU40 UD6nM     [        US9$ )aÍ  
Args:
    inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
        Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
        This is useful if you want more control over how to convert `input_ids` indices into associated vectors
        than the model's internal embedding lookup matrix.
    position_embeddings (`tuple[torch.Tensor, torch.Tensor]`):
        A tuple of two tensors, each of shape `(batch, seq_len, embed_dim)`.
        Represents absolute positional embeddings for the query and key in the attention mechanism.
    attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
        Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:
        - 1 for tokens that are **not masked**,
        - 0 for tokens that are **masked**.
        [What are attention masks?](../glossary#attention-mask)
)Úlast_hidden_state)r  r   )r'   r  rí   r¼   rY   r,   Úencoder_layers          r)   r/   ÚMLCDEncoder.forward�  sF   € ð, &ˆØ!Ÿ[œ[ˆMÙ)ØØ#Øñð ñ	ŠMñ )ô Ø+ñ
ð 	
r+   )r   r  r  r   )r1   r2   r3   r4   rx   r   r   r5   r·   rz   r6   r   r   r   r/   r7   r8   r9   s   @r)   r  r  q  s€   ø† ñð,Ð/÷ ,ð /3ñ	!
à×(Ñ(ð!
ð # 5§<¡<°·±Ð#=Ñ>ð!
ð Ÿ™ tÑ+ð	!
ð
 Ð+Ñ,ð!
ð 
�Ñ	 ÷!
ó !
r+   r  c                   óˆ   ^ • \ rS rSr% \\S'   SrS/rSrSr	Sr
SrSrSr\\S.r\R$                  " 5       U 4S j5       rS	rU =r$ )
ÚMLCDPreTrainedModeli¥  r   Úvision_modelr÷   TF)r,   Ú
attentionsc                 ó  >• [         TU ]  U5        U R                  R                  n[	        U[
        5      (       aâ  U R                  R                  n[        R                  " UR                  SUR                  S-  U-  S9  [        R                  " UR                  R                  UR                  R                  U-  S9  [        R                  " UR                  [        R                   " UR                  R"                  S   5      R%                  S5      5        g[	        U[&        5      (       aü  U R                  R                  nUR                  S-  SUR                  R(                  -  S-  -  U-  nUR                  S-  U-  n[        R                  " UR*                  R                  US9  [        R                  " UR,                  R                  US9  [        R                  " UR.                  R                  US9  [        R                  " UR0                  R                  US9  g[	        U[2        5      (       aÁ  U R                  R                  nUR                  R4                  S-  SUR                  R(                  -  S-  -  U-  nSUR                  R4                  -  S-  U-  n[        R                  " UR6                  R                  US9  [        R                  " UR8                  R                  US9  g[	        U[:        5      (       am  U R                  R                  nUR                  R4                  UR                  R<                  -  S-  S-  U-  n[        R                  " UR>                  SUS9  gg)	zInitialize the weightsrï   rã   )ÚmeanÚstd)r  rp   r…   rQ   N) r   Ú_init_weightsr   Úinitializer_factorrd   r}   ÚinitÚnormal_r‹   r†   rŽ   r�   Úinitializer_rangeÚcopy_rk   r5   rV   r›   r‘   rà   r  rê   rè   ré   rë   r   r"   r$   r%   ÚMLCDVisionModelrU   Úclass_pos_emb)r'   r¸   ÚfactorÚin_proj_stdÚout_proj_stdÚfc_stdÚpos_emb_stdr(   s          €r)   r  Ú!MLCDPreTrainedModel._init_weightsµ  s‘  ø€ ô 	‰Ñ˜fÔ%Ø—‘×/Ñ/ˆÜ�fÔ2×3Ñ3Ø—[‘[×3Ñ3ˆFÜ�LŠL˜×/Ñ/°c¸v×?OÑ?OÐQUÑ?UÐX^Ñ?^Ò_Ü�LŠL˜×/Ñ/×6Ñ6¸F¿M¹M×<[Ñ<[Ð^dÑ<dÒeÜ�JŠJ�v×*Ñ*¬E¯LªL¸×9LÑ9L×9RÑ9RÐSUÑ9VÓ,W×,^Ñ,^Ð_fÓ,gÕhÜ˜¤×.Ñ.Ø—[‘[×3Ñ3ˆFØ!×+Ñ+¨TÑ1°q¸6¿=¹=×;ZÑ;ZÑ7ZÐ_cÑ6cÑdÐgmÑmˆKØ"×,Ñ,¨dÑ2°fÑ<ˆLÜ�LŠL˜Ÿ™×-Ñ-°;Ò?Ü�LŠL˜Ÿ™×-Ñ-°;Ò?Ü�LŠL˜Ÿ™×-Ñ-°;Ò?Ü�LŠL˜Ÿ™×/Ñ/°\ÓBÜ˜¤×(Ñ(Ø—[‘[×3Ñ3ˆFØ!Ÿ=™=×4Ñ4°dÑ:ÀÀFÇMÁM×DcÑDcÑ@cÐhlÑ?lÑmÐpvÑvˆKØ˜&Ÿ-™-×3Ñ3Ñ3¸Ñ<¸vÑEˆFÜ�LŠL˜Ÿ™×*Ñ*°Ò7Ü�LŠL˜Ÿ™×*Ñ*°Ó<Ü˜¤×0Ñ0Ø—[‘[×3Ñ3ˆFØ!Ÿ=™=×4Ñ4¸¿¹×8YÑ8YÑYÐ]^Ñ^ÐcgÑgÐjpÑpˆKÜ�LŠL˜×-Ñ-°C¸[ÓIð 1r+   r  )r1   r2   r3   r4   r   Ú__annotations__Úbase_model_prefixÚ_no_split_modulesÚsupports_gradient_checkpointingÚaccepts_loss_kwargsÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_supports_attention_backendr÷   rà   Ú_can_record_outputsr5   r{   r  r7   r8   r9   s   @r)   r  r  ¥  se   ø‡ àÓØ&ÐØ+Ð,ÐØ&*Ð#ØÐØÐØ€NØÐØ"&Ðà)Ø#ñÐð
 ‡]‚]ƒ_ôJó öJr+   r  zN
    The vision model from M_L_C_D without any head or projection on top.
    )Úcustom_introc                   ó°   ^ • \ rS rSr% \\S'   SrSrSrS\4U 4S jjr	\
\" SS9\ SS\R                  S	-  S
\\   S\\-  4S jj5       5       5       rSrU =r$ )r"  iÓ  r   r­   )ÚimagerŽ   c                 óÜ  >• [         TU ]  U5        UR                  n[        U5      U l        [
        R                  " X!R                  S9U l        [        U5      U l
        [
        R                  " X!R                  S9U l        [        U5      U l        [
        R                  " [        R                   " SUR                  UR"                  -  S-  5      5      U l        U R'                  5         g )Nrù   r   rQ   )r   r   r"   r}   r“   r    rü   rý   Úpre_layrnormr  ÚencoderÚpost_layernormr;   Úvision_rotary_embeddingr‰   r5   rŠ   rU   r#  Ú	post_init)r'   r   r†   r(   s      €r)   r   ÚMLCDVisionModel.__init__Þ  s«   ø€ Ü‰Ñ˜Ô Ø×&Ñ&ˆ	ä.¨vÓ6ˆŒÜŸLšL¨×8MÑ8MÑNˆÔÜ" 6Ó*ˆŒÜ Ÿlšl¨9×:OÑ:OÑPˆÔÜ':¸6Ó'BˆÔ$ÜŸ\š\¬%¯+ª+°a¸×9KÑ9KÈv×OiÑOiÑ9iÐmnÑ9nÓ*oÓpˆÔØ�‰Õr+   F)Útie_last_hidden_statesNrY   r-   c                 óî  • Uc  [        S5      eU R                  U5      nU R                  U5      nUR                  S   U R                  R
                  -  nUR                  S   U R                  R
                  -  n[        R                  " XAR                  S9R                  S5      R                  SU5      n[        R                  " XQR                  S9R                  S5      R                  US5      n[        R                  " UR                  5       UR                  5       /SS9nU R                  X85      n	[        R                  " U R                  U R                  /SS9n
[        R                  " U
R!                  5       U	S   /SS9[        R                  " U
R#                  5       U	S   /SS9/n	U R$                  " SUU	S	.UD6nUS   nUSS2SSS24   nU R'                  U5      n[)        UUS
9$ )ac  
Example:

```python
>>> import httpx
>>> from io import BytesIO
>>> from PIL import Image
>>> from transformers import AutoProcessor, MLCDVisionModel
>>> model = MLCDVisionModel.from_pretrained("DeepGlint-AI/mlcd-vit-bigG-patch14-448")
>>> processor = AutoProcessor.from_pretrained("DeepGlint-AI/mlcd-vit-bigG-patch14-448")

>>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
>>> with httpx.stream("GET", url) as response:
...     image = Image.open(BytesIO(response.read()))
>>> inputs = processor(images=image, return_tensors="pt")

>>> with torch.no_grad():
...     outputs = model(**inputs, output_attentions=True)

>>> features = outputs.last_hidden_state
>>> print(f"Extracted features shape: {features.shape}")
>>> print(f"Number of attention layers: {len(outputs.attentions)}")
>>> print(f"Attention shape: {outputs.attentions[0].shape}")
```Nz You have to specify pixel_valuesrÙ   rp   )r=   r   r   rq   )r  rí   )r  Úpooler_outputr  )rF   r“   r8  r›   r   rˆ   r5   rV   r=   rž   r‘   Ústackr¯   r;  rr   r#  rg   rh   r9  r:  r	   )r'   r­   rY   r,   Únum_patches_heightÚnum_patches_widthÚhpos_idsÚwpos_idsÚpos_idsrí   r#  Úencoder_outputsr  Úpooled_outputs                 r)   r/   ÚMLCDVisionModel.forwardê  sê  € ð@ ÑÜÐ?Ó@Ð@àŸ™¨Ó5ˆØ×)Ñ)¨-Ó8ˆà)×/Ñ/°Ñ3°t·{±{×7MÑ7MÑMÐØ(×.Ñ.¨rÑ2°d·k±k×6LÑ6LÑLÐä�LŠLÐ+×4GÑ4GÑH×RÑRÐSTÓU×\Ñ\Ð]_ÐarÓsð 	ô �LŠLÐ*×3FÑ3FÑG×QÑQÐRSÓT×[Ñ[Ð\nÐprÓsð 	ô —+’+˜x×/Ñ/Ó1°8×3CÑ3CÓ3EÐFÈBÑOˆØ"×:Ñ:¸=ÓRÐÜŸ	š	 4×#5Ñ#5°t×7IÑ7IÐ"JÐPRÑSˆä�IŠI�}×(Ñ(Ó*Ð,?ÀÑ,BÐCÈÑKÜ�IŠI�}×(Ñ(Ó*Ð,?ÀÑ,BÐCÈÑKð
Ðð
 Ÿ,š,ð 
Ø'Ø 3ñ
ð ñ
ˆð ,¨AÑ.ÐØ)ª!¨Q²¨'Ñ2ˆØ×+Ñ+¨MÓ:ˆä)Ø/Ø'ñ
ð 	
r+   )r#  r“   r9  r:  r8  r;  r   )r1   r2   r3   r4   r   r*  Úmain_input_nameÚinput_modalitiesÚ_input_embed_layerr   r   r   r   r5   r·   r   r   rz   r	   r/   r7   r8   r9   s   @r)   r"  r"  Ó  s�   ø‡ ð ÓØ$€OØ!ÐØ*Ðð
Ð/÷ 
ð  Ù¨EÑ2Øð 26ñ@
à×'Ñ'¨$Ñ.ð@
ð Ð+Ñ,ð@
ð 
Ð+Ñ	+ô	@
ó ó 3ó  ö@
r+   r"  )rï   )3Úcollections.abcr   r5   Útorch.nnr    Ú r   r  Úactivationsr   Úmodeling_layersr   Úmodeling_outputsr   r	   Úmodeling_utilsr
   r   Úprocessing_utilsr   Úutilsr   r   r   Úutils.deprecationr   Úutils.genericr   r   Úutils.output_capturingr   Úconfiguration_mlcdr   ÚModuler   r;   r}   r6   rW   rÌ   rÐ   r¶   rÂ   rz   rÞ   rà   r÷   r  r  r"  Ú__all__r  r+   r)   Ú<module>r\     sß  ðõ( %ã Ý å &Ý !Ý 9ß Kß FÝ &ß BÑ BÝ 0ß GÝ 5Ý 0ôˆb�i‰iô ô=5˜"Ÿ)™)ô =5ô@I˜2Ÿ9™9ô Iðf ñ%Ø�I‰Ið%à�<‰<ð%ð 
�‰ð%ð �<‰<ð	%ð
 —L‘L 4Ñ'ð%ð ð%ð ð%ð Ð'Ñ(õ%ò2(ð	U˜UŸ\™\ð 	U°#ð 	U¸%¿,¹,ô 	UðØ‡|�|ðØŸ™ðØ+0¯<©<ðØ>C¿l¹lðà
ˆ5�<‰<˜Ÿ™Ð%Ñ&ôôE)�B—I‘Iô E)ôP+Ð1ô +ô\1
�"—)‘)ô 1
ðh ô*J˜/ó *Jó ð*JñZ ðñô
U
Ð)ó U
óð
U
ðp !Ð"3Ð
4�r+   