ó
    ÷“,j×  ã                   ój  • S SK Jr  S SKrS SKJrJrJrJrJrJ	r	J
r
Jr  S SKrS SKJr  S SKJr  S SKJr  SSKJrJr  SS	KJr   " S
 S\R0                  5      r " S S\R0                  5      r " S S\R4                  5      r " S S\R4                  5      r " S S\R4                  5      r " S S\R4                  5      r " S S\R4                  5      r " S S\R4                  5      r  " S S\R4                  5      r! " S S\R4                  5      r" " S S\R4                  5      r#S \$4S! jr% " S" S#\R4                  5      r&   S1S$\RN                  S%\\RN                     S&\(S'\\$   S(\RN                  4
S) jjr) " S* S+\R4                  5      r* " S, S-\#5      r+ S2S.\R4                  S/\$4S0 jjr,g)3é    )ÚOrderedDictN)ÚCallableÚDictÚListÚOptionalÚSequenceÚTupleÚTypeÚUnion)Únn)Ú
functional)Ú
checkpointé   )Ú	to_2tupleÚfeature_take_indices)Úget_2d_sincos_pos_embedc                   ó:   • \ rS rSrSrS\R                  4S jrSrg)ÚLayerNormFp32é   zKSubclass torch's LayerNorm to handle fp16 (by casting to float32 and back).Úxc                 óú   • UR                   n[        R                  " UR                  [        R
                  5      U R                  U R                  U R                  U R                  5      nUR                  U5      $ ©N)
ÚdtypeÚFÚ
layer_normÚtoÚtorchÚfloat32Únormalized_shapeÚweightÚbiasÚeps©Úselfr   Ú	orig_types      Ú£C:\Users\johnn\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.13_qbz5n2kfra8p0\LocalCache\local-packages\Python313\site-packages\open_clip/transformer.pyÚforwardÚLayerNormFp32.forward   sT   € Ø—G‘Gˆ	Ü�LŠL˜Ÿ™œeŸm™mÓ,¨d×.CÑ.CÀTÇ[Á[ÐRV×R[ÑR[Ð]a×]eÑ]eÓfˆØ�t‰t�I‹Ðó    © N©	Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__r   ÚTensorr'   Ú__static_attributes__r*   r)   r&   r   r      s   † ÙUð˜Ÿ™÷ r)   r   c                   ó:   • \ rS rSrSrS\R                  4S jrSrg)Ú	LayerNormé   z;Subclass torch's LayerNorm (with cast back to input dtype).r   c                 ó¾   • UR                   n[        R                  " XR                  U R                  U R
                  U R                  5      nUR                  U5      $ r   )r   r   r   r   r    r!   r"   r   r#   s      r&   r'   ÚLayerNorm.forward   s?   € Ø—G‘Gˆ	Ü�LŠL˜×1Ñ1°4·;±;ÀÇ	Á	È4Ï8É8ÓTˆØ�t‰t�I‹Ðr)   r*   Nr+   r*   r)   r&   r4   r4      s   † ÙEð˜Ÿ™÷ r)   r4   c                   ó6   • \ rS rSrS\R
                  4S jrSrg)Ú	QuickGELUé    r   c                 ó:   • U[         R                  " SU-  5      -  $ )Ng¬Zd;û?)r   Úsigmoid©r$   r   s     r&   r'   ÚQuickGELU.forward"   s   € Ø”5—=’= ¨¡Ó+Ñ+Ð+r)   r*   N)r,   r-   r.   r/   r   r1   r'   r2   r*   r)   r&   r9   r9       s   † ð,˜Ÿ™÷ ,r)   r9   c                   ó2   ^ • \ rS rSrSU 4S jjrS rSrU =r$ )Ú
LayerScaleé&   c                 ó’   >• [         TU ]  5         X0l        [        R                  " U[
        R                  " U5      -  5      U l        g r   )ÚsuperÚ__init__Úinplacer   Ú	Parameterr   ÚonesÚgamma)r$   ÚdimÚinit_valuesrE   Ú	__class__s       €r&   rD   ÚLayerScale.__init__'   s0   ø€ Ü‰ÑÔØŒÜ—\’\ +´·
²
¸3³Ñ"?Ó@ˆ�
r)   c                 óv   • U R                   (       a  UR                  U R                  5      $ XR                  -  $ r   )rE   Úmul_rH   r=   s     r&   r'   ÚLayerScale.forward,   s&   € Ø%)§\§\ˆq�v‰v�d—j‘jÓ!ÐE°q¿:¹:±~ÐEr)   )rH   rE   )gñhãˆµøä>F)r,   r-   r.   r/   rD   r'   r2   Ú__classcell__©rK   s   @r&   r@   r@   &   s   ø† ÷A÷
Fð Fr)   r@   c                   óF   ^ • \ rS rSrSr  SS\S\4U 4S jjjrS rSr	U =r
$ )	ÚPatchDropouté0   z"
https://arxiv.org/abs/2212.00794
ÚprobÚexclude_first_tokenc                 ó^   >• [         TU ]  5         SUs=::  a  S:  d   e   eXl        X l        g )Nr   g      ð?)rC   rD   rU   rV   )r$   rU   rV   rK   s      €r&   rD   ÚPatchDropout.__init__5   s0   ø€ ô
 	‰ÑÔØ�D�~˜2‹~Ð‰~Ðˆ~ØŒ	Ø#6Õ r)   c                 ó�  • U R                   (       a  U R                  S:X  a  U$ U R                  (       a  US S 2S S24   US S 2SS 24   pO7[        R                  R                  [        R                  US S 2S S24   5      nUR                  5       S   nUR                  5       S   n[        R                  " U5      nUS   nSU R                  -
  n[        S[        XF-  5      5      n[        R                  " X45      nUR                  USS9R                  n	XU	4   nU R                  (       a  [        R                  " X!4SS9nU$ )Nç        r   r   ).Néÿÿÿÿ©rI   )ÚtrainingrU   rV   r   ÚjitÚannotater1   ÚsizeÚarangeÚmaxÚintÚrandnÚtopkÚindicesÚcat)
r$   r   Ú
cls_tokensÚbatchÚ
num_tokensÚbatch_indicesÚ	keep_probÚnum_patches_keepÚrandÚpatch_indices_keeps
             r&   r'   ÚPatchDropout.forward?   s  € Ø�}�} §	¡	¨R£ØˆHà×#×#Øša  ! ˜e™H aª¨1©2¨¡h™äŸ™×+Ñ+¬E¯L©L¸!ºA¸rÀ¸r¸E¹(ÓCˆJà—‘“˜‘ˆØ—V‘V“X˜a‘[ˆ
äŸš UÓ+ˆØ% iÑ0ˆà˜Ÿ	™	‘Mˆ	Ü˜q¤# jÑ&<Ó"=Ó>Ðä�{Š{˜5Ó-ˆØ!ŸY™YÐ'7¸R˜YÐ@×HÑHÐàÐ/Ð/Ñ0ˆà×#×#Ü—	’	˜:˜/¨qÑ1ˆAàˆr)   )rV   rU   )g      à?T)r,   r-   r.   r/   r0   ÚfloatÚboolrD   r'   r2   rP   rQ   s   @r&   rS   rS   0   s6   ø† ñð Ø(,ñ7àð7ð "&÷7ð 7÷ð r)   rS   c                   óÖ   ^ • \ rS rSrSSSSSS\R
                  " S5      \SS4
S\S\S	\S
\S\S\S\S\	S\
\R                     S\	S\	4U 4S jjjrSS\\R                      4S jjrSrU =r$ )Ú	Attentioné\   é   TFg      Y@rZ   rI   Ú	num_headsÚqkv_biasÚqk_normÚscaled_cosineÚscale_headsÚ
inner_normÚlogit_scale_maxÚ
norm_layerÚ	attn_dropÚ	proj_dropc           
      óD  >• [         TU ]  5         U(       a  U(       a   S5       eXPl        X`l        X-  S:X  d   S5       eX l        X-  U l        U R
                  S-  U l        X€l        [        [        R                  S5      U l        [        R                  " [        R                  " US-  U45      U R                  -  5      U l        U(       a3  [        R                  " [        R                   " US-  5      5      U l        OS U l        U(       a/  U	" U R
                  5      U l        U	" U R
                  5      U l        O4[        R(                  " 5       U l        [        R(                  " 5       U l        U R                  (       aJ  [        R                  " [        R*                  " S[        R,                  " USS45      -  5      5      U l        OS U l        [        R0                  " U
5      U l        U R                  (       a3  [        R                  " [        R,                  " USS45      5      U l        OS U l        U(       a  U	" U5      U l        O[        R(                  " 5       U l        [        R8                  " X5      U l        [        R0                  " U5      U l        g )	Nz7Cannot activate both scaled cosine and QK normalizationr   z$dim should be divisible by num_headsç      à¿Úscaled_dot_product_attentioné   é
   r   )rC   rD   rz   r{   rw   Úhead_dimÚscaler}   Úhasattrr   r   Ú	use_fsdparF   r   rd   Úin_proj_weightÚzerosÚin_proj_biasÚln_qÚln_kÚIdentityÚlogrG   Úlogit_scaleÚDropoutr   Ú
head_scaleÚln_innerÚLinearÚout_projÚout_drop)r$   rI   rw   rx   ry   rz   r{   r|   r}   r~   r   r€   rK   s               €r&   rD   ÚAttention.__init__]   sÓ  ø€ ô 	‰ÑÔÞ!¦gÐiÐ0iÓiÐ.Ø*ÔØ&ÔØ‰ !Ó#ÐKÐ%KÓKÐ#Ø"ŒØÑ(ˆŒØ—]‘] dÑ*ˆŒ
Ø.ÔÜ ¤§¡Ð0NÓOˆŒô !Ÿlšl¬5¯;ª;¸¸a¹À°~Ó+FÈÏÉÑ+SÓTˆÔÞÜ "§¢¬U¯[ª[¸¸q¹Ó-AÓ BˆDÕà $ˆDÔö Ù" 4§=¡=Ó1ˆDŒIÙ" 4§=¡=Ó1ˆD�IäŸš›ˆDŒIÜŸš›ˆDŒIð ××Ü!Ÿ|š|¬E¯IªI°b¼5¿:º:ÀyÐRSÐUVÐFWÓ;XÑ6XÓ,YÓZˆDÕà#ˆDÔäŸš IÓ.ˆŒð ××Ü Ÿlšl¬5¯:ª:°yÀ!ÀQÐ6GÓ+HÓIˆD�Oà"ˆDŒOö Ù& s›OˆD�MäŸKšK›MˆDŒMäŸ	š	 #Ó+ˆŒÜŸ
š
 9Ó-ˆ�r)   Ú	attn_maskc                 ó:  • UR                   u  p4n[        R                  " XR                  U R                  5      R                  SSS9u  pgnUR                  X4U R                  S5      R                  SS5      nUR                  X4U R                  S5      R                  SS5      nUR                  X4U R                  S5      R                  SS5      nUbŸ  UR                  S:X  a  UR                  X0R                  XD5      nUR                  [        R                  :X  a<  [        R                  " X&R                  S9n	U	R                  U[        S5      5        U	nOUR!                  UR                  S9nU R"                  bÆ  [        R$                  " [        R&                  " USS9[        R&                  " USS9R                  SS5      5      n
[        R(                  " U R"                  U R*                  S	9R-                  5       nX«-  n
Ub  X¢-   n
U
R/                  SS9n
U R1                  U
5      n
[        R$                  " X¨5      nOãU R3                  U5      nU R5                  U5      nU R6                  (       a?  [        R8                  " XgUUU R:                  (       a  U R0                  R<                  OS
S9nOqX`R>                  -  n[        R$                  " XgR                  SS5      5      n
Ub  X¢-  n
U
R/                  SS9n
U R1                  U
5      n
[        R$                  " X¨5      nU R@                  b  XR@                  -  nUR                  SS5      R                  X4U5      nU RC                  U5      nU RE                  U5      nU RG                  U5      nU$ )Nr„   r[   r\   r   é   ©r   ú-inféþÿÿÿ)rb   rZ   )r™   Ú	dropout_p)$Úshaper   ÚlinearrŠ   rŒ   ÚchunkÚreshaperw   Ú	transposeÚndimr   r   rr   Ú
zeros_likeÚmasked_fill_rq   r   r‘   ÚbmmÚ	normalizeÚclampr}   ÚexpÚsoftmaxr   r�   rŽ   r‰   rƒ   r]   Úpr‡   r“   r”   r–   r—   )r$   r   r™   ÚNÚLÚCÚqÚkÚvÚnew_attn_maskÚattnr‘   s               r&   r'   ÚAttention.forward�   sà  € Ø—'‘'‰ˆˆaÜ—(’(˜1×1Ñ1°4×3DÑ3DÓE×KÑKÈAÐSUÐKÐV‰ˆˆaØ�I‰I�a˜DŸN™N¨BÓ/×9Ñ9¸!¸QÓ?ˆØ�I‰I�a˜DŸN™N¨BÓ/×9Ñ9¸!¸QÓ?ˆØ�I‰I�a˜DŸN™N¨BÓ/×9Ñ9¸!¸QÓ?ˆàÑ Ø�~‰~ Ó"à%×-Ñ-¨a·±ÀÓF�	Ø�‰¤%§*¡*Ó,Ü %× 0Ò 0°Ç'Á'Ñ J�Ø×*Ñ*¨9´e¸F³mÔDØ)‘	à%ŸL™L¨q¯w©w˜LÐ7�	à×ÑÑ'Ü—9’9Ü—’˜A 2Ñ&Ü—’˜A 2Ñ&×0Ñ0°°RÓ8óˆDô  Ÿ+š+ d×&6Ñ&6¸D×<PÑ<PÑQ×UÑUÓWˆKØÑ%ˆDØÑ$ØÑ'�Ø—<‘< B�<Ð'ˆDØ—>‘> $Ó'ˆDÜ—	’	˜$Ó"‰Aà—	‘	˜!“ˆAØ—	‘	˜!“ˆAØ�~�~Ü×2Ò2Ø˜!Ø'Ø26·-·-˜dŸn™n×.Ò.ÀRñ‘ð Ÿ
™
‘N�Ü—y’y §K¡K°°BÓ$7Ó8�ØÑ(ØÑ%�DØ—|‘|¨�|Ð+�Ø—~‘~ dÓ+�Ü—I’I˜dÓ&�ð �?‰?Ñ&Ø—O‘OÑ#ˆAØ�K‰K˜˜1Ó×%Ñ% a¨AÓ.ˆØ�M‰M˜!ÓˆØ�M‰M˜!ÓˆØ�M‰M˜!ÓˆØˆr)   )r   r†   r“   rŒ   rŠ   r”   rŽ   r�   r‘   r}   rw   r—   r–   r‡   r{   rz   r‰   r   )r,   r-   r.   r/   Úmathr�   r   rc   rr   rq   r
   r   ÚModulerD   r   r   r1   r'   r2   rP   rQ   s   @r&   rt   rt   \   sÅ   ø† ð Ø!Ø!Ø"'Ø %Ø$Ø%)§X¢X¨iÓ%8Ø*7Ø!Ø!ñ>.àð>.ð ð>.ð ð	>.ð
 ð>.ð  ð>.ð ð>.ð ð>.ð #ð>.ð ˜RŸY™Y™ð>.ð ð>.ð ÷>.ð >.ñ@7 H¨U¯\©\Ñ$:÷ 7ó 7r)   rt   c                   ól   ^ • \ rS rSrSS\4S\S\S\S\S\4
U 4S	 jjjrS
\R                  4S jr
SrU =r$ )ÚAttentionalPooleré×   rv   é   Úd_modelÚcontext_dimÚn_headÚ	n_queriesr~   c                 óê   >• [         TU ]  5         [        R                  " [        R
                  " XA5      5      U l        [        R                  " XX"SS9U l        U" U5      U l	        U" U5      U l
        g )NT)ÚkdimÚvdimÚbatch_first)rC   rD   r   rF   r   rd   ÚqueryÚMultiheadAttentionrµ   r�   rŽ   )r$   r½   r¾   r¿   rÀ   r~   rK   s         €r&   rD   ÚAttentionalPooler.__init__Ø   sX   ø€ ô 	‰ÑÔÜ—\’\¤%§+¢+¨iÓ"AÓBˆŒ
Ü×)Ò)¨'ÀÐkoÑpˆŒ	Ù˜wÓ'ˆŒ	Ù˜{Ó+ˆ�	r)   r   c                 óä   • UR                   S   nU R                  U5      nU R                  U R                  5      nU R	                  UR                  S5      R                  USS5      XSS9S   nU$ )Nr   r[   F)Úneed_weights)r    rŽ   r�   rÅ   rµ   Ú	unsqueezeÚexpand)r$   r   r®   r±   Úouts        r&   r'   ÚAttentionalPooler.forwardæ   sd   € Ø�G‰G�A‰JˆØ�I‰I�a‹LˆØ�I‰I�d—j‘jÓ!ˆØ�i‰i˜Ÿ™ A›×-Ñ-¨a°°RÓ8¸!ÈUˆiÐSÐTUÑVˆØˆ
r)   )rµ   rŽ   r�   rÅ   )r,   r-   r.   r/   r4   rc   r   rD   r   r1   r'   r2   rP   rQ   s   @r&   rº   rº   ×   s]   ø† ð
 Ø Ø#,ñ,àð,ð ð,ð ð	,ð
 ð,ð !÷,ð ,ð˜Ÿ™÷ ò r)   rº   c                   óž  ^ • \ rS rSrSS\R
                  \SS4S\S\S\S	\S
\	S\	S\
S\
4U 4S jjjrS\R                  4S jr   SS\R                  S\\R                     S\\R                     S\\R                     4S jjr   SS\R                  S\\R                     S\\R                     S\\R                     4S jjrSrU =r$ )ÚResidualAttentionBlockéî   ç      @NFTr½   r¿   Ú	mlp_ratioÚls_init_valueÚ	act_layerr~   Úis_cross_attentionrÄ   c	                 ó,  >• [         T
U ]  5         U" U5      U l        [        R                  " XUS9U l        Ub  [        X5      O[        R                  " 5       U l        U(       a  U" U5      U l	        U" U5      U l
        [        X-  5      n	[        R                  " [        S[        R                  " X5      4SU" 5       4S[        R                  " X‘5      4/5      5      U l        Ub  [        X5      U l        g [        R                  " 5       U l        g )N)rÄ   Úc_fcÚgeluÚc_proj)rC   rD   Úln_1r   rÆ   rµ   r@   r�   Úls_1Úln_1_kvÚln_2rc   Ú
Sequentialr   r•   ÚmlpÚls_2)r$   r½   r¿   rÒ   rÓ   rÔ   r~   rÕ   rÄ   Ú	mlp_widthrK   s             €r&   rD   ÚResidualAttentionBlock.__init__ï   sá   ø€ ô 	‰ÑÔá˜wÓ'ˆŒ	Ü×)Ò)¨'À{ÑSˆŒ	Ø:GÑ:S”J˜wÔ6ÔY[×YdÒYdÓYfˆŒ	ÞÙ% gÓ.ˆDŒLá˜wÓ'ˆŒ	Ü˜Ñ+Ó,ˆ	Ü—=’=¤Ø”R—Y’Y˜wÓ2Ð3Ø‘Y“[Ð!Ø”r—y’y Ó4Ð5ð.
ó "ó ˆŒð
 ;HÑ:S”J˜wÓ6ˆ�	ÔY[×YdÒYdÓYfˆ�	r)   Úreturnc                 óà   • [        U R                  R                  S5      (       a   U R                  R                  R                  $ U R                  R                  R                  R
                  $ ©NÚint8_original_dtype©rˆ   rß   r×   ræ   r    r   ©r$   s    r&   Úget_weight_dtypeÚ'ResidualAttentionBlock.get_weight_dtype  óF   € Ü�4—8‘8—=‘=Ð"7×8Ñ8Ø—8‘8—=‘=×4Ñ4Ð4Ø�x‰x�}‰}×#Ñ#×)Ñ)Ð)r)   Úq_xÚk_xÚv_xr™   c                 óˆ   • Ub  UOUnUb  UOUnUb  UR                  UR                  5      OS nU R                  XUSUS9S   $ )NF)rÉ   r™   r   )r   r   rµ   )r$   rì   rí   rî   r™   s        r&   Ú	attentionÚ ResidualAttentionBlock.attention  s^   € ð ‘_‰c¨#ˆØ‘_‰c¨#ˆà/8Ñ/D�I—L‘L §¡Ô+È$ˆ	Ø�y‰yØ�cØØð ð 
ð ñ	ð 	r)   c           
      óf  • [        U S5      (       a  Ub  U R                  U5      OS n[        U S5      (       a  Ub  U R                  U5      OS nXR                  U R                  U R	                  U5      X#US95      -   nXPR                  U R                  U R                  U5      5      5      -   nU$ )NrÜ   )rì   rí   rî   r™   )rˆ   rÜ   rÛ   rð   rÚ   rà   rß   rÝ   )r$   rì   rí   rî   r™   r   s         r&   r'   ÚResidualAttentionBlock.forward!  s–   € ô $+¨4°×#;Ñ#;ÀÁˆd�l‰l˜3ÔÐUYˆÜ#*¨4°×#;Ñ#;ÀÁˆd�l‰l˜3ÔÐUYˆØ—)‘)˜DŸN™N¨t¯y©y¸«~À3Ð[d˜NÐeÓfÑfˆØ—	‘	˜$Ÿ(™( 4§9¡9¨Q£<Ó0Ó1Ñ1ˆØˆr)   )rµ   rÚ   rÜ   rÝ   rÛ   rà   rß   )NNN)r,   r-   r.   r/   r   ÚGELUr4   rc   rq   r   rr   rD   r   r   ré   r1   r   rð   r'   r2   rP   rQ   s   @r&   rÏ   rÏ   î   s<  ø† ð
  #Ø#'Ø"$§'¡'Ø#,Ø',Ø $ñgàðgð ðgð ð	gð
 !ðgð  ðgð !ðgð !%ðgð ÷gð gð8* %§+¡+ô *ð +/Ø*.Ø04ñà—‘ðð ˜%Ÿ,™,Ñ'ðð ˜%Ÿ,™,Ñ'ð	ð
   §¡Ñ-õð( +/Ø*.Ø04ñà—‘ðð ˜%Ÿ,™,Ñ'ðð ˜%Ÿ,™,Ñ'ð	ð
   §¡Ñ-÷ó r)   rÏ   c                   ó(  ^ • \ rS rSrSS\R
                  \SSSSSSS4S\S\S\S	\S
\	\R                     S\	\R                     S\S\S\S\S\S\S\4U 4S jjjrS\R                  4S jrSS\R                   S\\R                      4S jjrSrU =r$ )ÚCustomResidualAttentionBlocki/  rÑ   NFTr½   r¿   rÒ   rÓ   rÔ   r~   ry   Úscale_cosine_attnr{   Úscale_attn_innerÚ
scale_attnÚscale_fcrÄ   c                 ó°  >• [         TU ]  5         U(       d   S5       eU" U5      U l        [        UUUUU	U
US9U l        U(       a  U" U5      O[
        R                  " 5       U l        Ub  [        X5      O[
        R                  " 5       U l	        U" U5      U l
        [        X-  5      n[
        R                  " [        S[
        R                  " X5      4SU" 5       4SU(       a  U" U5      O[
        R                  " 5       4S[
        R                  " Xá5      4/5      5      U l        Ub  [        X5      U l        g [
        R                  " 5       U l        g )Nz9batch_first must be True for CustomResidualAttentionBlock)ry   rz   r{   r|   r~   r×   rØ   ÚlnrÙ   )rC   rD   rÚ   rt   rµ   r   r�   Úln_attnr@   rÛ   rÝ   rc   rÞ   r   r•   rß   rà   )r$   r½   r¿   rÒ   rÓ   rÔ   r~   ry   r÷   r{   rø   rù   rú   rÄ   rá   rK   s                  €r&   rD   Ú%CustomResidualAttentionBlock.__init__0  s  ø€ ô  	‰ÑÔÞÐWÐWÓWˆ{á˜wÓ'ˆŒ	ÜØØØØ+Ø#Ø'Ø!ñ
ˆŒ	ö /9‘z 'Ô*¼b¿kºk»mˆŒØ:GÑ:S”J˜wÔ6ÔY[×YdÒYdÓYfˆŒ	á˜wÓ'ˆŒ	Ü˜Ñ+Ó,ˆ	Ü—=’=¤Ø”R—Y’Y˜wÓ2Ð3Ø‘Y“[Ð!Ø®H‘:˜iÔ(¼"¿+º+»-ÐHØ”r—y’y Ó4Ð5ð	.
ó "ó ˆŒð ;HÑ:S”J˜wÓ6ˆ�	ÔY[×YdÒYdÓYfˆ�	r)   rã   c                 óà   • [        U R                  R                  S5      (       a   U R                  R                  R                  $ U R                  R                  R                  R
                  $ rå   rç   rè   s    r&   ré   Ú-CustomResidualAttentionBlock.get_weight_dtypeZ  rë   r)   r   r™   c           
      óæ   • XR                  U R                  U R                  U R                  U5      US95      5      -   nXR	                  U R                  U R                  U5      5      5      -   nU$ )N©r™   )rÛ   rý   rµ   rÚ   rà   rß   rÝ   )r$   r   r™   s      r&   r'   Ú$CustomResidualAttentionBlock.forward_  sX   € Ø—	‘	˜$Ÿ,™, t§y¡y°·±¸1³È yÐ'SÓTÓUÑUˆØ—	‘	˜$Ÿ(™( 4§9¡9¨Q£<Ó0Ó1Ñ1ˆØˆr)   )rµ   rÚ   rÝ   rý   rÛ   rà   rß   r   )r,   r-   r.   r/   r   rô   r4   rc   rq   r
   r¸   rr   rD   r   r   ré   r1   r   r'   r2   rP   rQ   s   @r&   rö   rö   /  s	  ø† ð
  #Ø#'Ø)+¯©Ø*3Ø!Ø&+Ø %Ø%*Ø$Ø"Ø $ñ(gàð(gð ð(gð ð	(gð
 !ð(gð ˜BŸI™I‘ð(gð ˜RŸY™Y™ð(gð ð(gð  $ð(gð ð(gð #ð(gð ð(gð ð(gð ÷(gð (gðT* %§+¡+ô *ñ
˜Ÿ™ð °(¸5¿<¹<Ñ2H÷ ó r)   rö   c                   ó¬  ^ • \ rS rSrSrSS\R                  \SS4S\S\S	\S
\	S\	S\
\R                     S\
\R                     S\S\\\\   4   4U 4S jjjrS\R$                  4S jr   SS\R(                  S\\R(                     S\\\\\   4      S\4S jjrSS\\\\   4   4S jjrSS\R(                  S\\R(                     4S jjrSrU =r$ )ÚCustomTransformerie  z9A custom transformer that can use different block types. rÑ   NTrö   ÚwidthÚlayersÚheadsrÒ   rÓ   rÔ   r~   rÄ   Úblock_typesc
                 óT  >^^^^^^^• [         TU ]  5         TU l        X l        TU l        SU l        [        U	[        5      (       a  U	/U-  n	[        U	5      U:X  d   eS[        4UUUUUUU4S jjn
[        R                  " U	 Vs/ s H  nU
" U5      PM     sn5      U l        g s  snf )NFÚbtc           
      ó2   >• U S:X  a  [        TTTTTTTS9$  e)Nrö   )rÒ   rÓ   rÔ   r~   rÄ   )rö   )r  rÔ   rÄ   r  rÓ   rÒ   r~   r  s    €€€€€€€r&   Ú_create_blockÚ1CustomTransformer.__init__.<locals>._create_block}  s5   ø€ ØÐ3Ó3Ü3ØØØ'Ø"/Ø'Ø)Ø +ñð ð �ur)   )rC   rD   r  r  rÄ   Úgrad_checkpointingÚ
isinstanceÚstrÚlenr   Ú
ModuleListÚ	resblocks)r$   r  r  r  rÒ   rÓ   rÔ   r~   rÄ   r	  r  r  rK   s    ` ``````   €r&   rD   ÚCustomTransformer.__init__g  s    ÿ€ ô 	‰ÑÔØˆŒ
ØŒØ&ˆÔØ"'ˆÔä�k¤3×'Ñ'Ø&˜-¨&Ñ0ˆKÜ�;Ó 6Ó)Ð)Ð)ð	œc÷ 	õ 	ô Ÿšá!ó(
â!�ñ ˜"ÖÙ!ñ(
ó ˆ�ùò (
s   ÂB%rã   c                 ó<   • U R                   S   R                  5       $ ©Nr   ©r  ré   rè   s    r&   Úget_cast_dtypeÚ CustomTransformer.get_cast_dtype�  ó   € Ø�~‰~˜aÑ ×1Ñ1Ó3Ð3r)   r   r™   rf   Ú
stop_earlyc           
      ó¶  • [        [        U R                  5      U5      u  pVU R                  (       d   UR	                  SS5      R                  5       n/ n[        R                  R                  5       (       d  U(       d  U R                  nOU R                  S US-    n[        U5       Hˆ  u  pšU R                  (       a1  [        R                  R                  5       (       d  [        X¡S S USS9nOU
" XS9nX•;   d  MT  UR                  U R                  (       d  UR	                  SS5      OU5        MŠ     U R                  (       d  UR	                  SS5      nX4$ ©Nr   r   F©Úuse_reentrantr  ©r   r  r  rÄ   r¤   Ú
contiguousr   r^   Úis_scriptingÚ	enumerater  r   Úappend©r$   r   r™   rf   r  Útake_indicesÚ	max_indexÚintermediatesÚblocksÚiÚblks              r&   Úforward_intermediatesÚ'CustomTransformer.forward_intermediates“  ó  € ô #7´s¸4¿>¹>Ó7JÈGÓ"TÑˆà××Ø—‘˜A˜qÓ!×,Ñ,Ó.ˆAàˆÜ�9‰9×!Ñ!×#Ñ#®:Ø—^‘^‰Fà—^‘^ N Y°¡]Ð3ˆFÜ Ö'‰FˆAØ×&×&¬u¯y©y×/EÑ/E×/GÑ/GÜ˜s t¨T°9ÈEÑR‘á˜Ñ/�àÕ Ø×$Ñ$¸d×>N×>N Q§[¡[°°AÔ%6ÐTUÖVñ (ð ××Ø—‘˜A˜qÓ!ˆAàÐÐr)   c                 óv   • [        [        U R                  5      U5      u  p#U R                  SUS-    U l        U$ ©ú?Prune layers not required for specified intermediates.
        Nr   ©r   r  r  ©r$   rf   r'  r(  s       r&   Úprune_intermediate_layersÚ+CustomTransformer.prune_intermediate_layers²  ó8   € ô #7´s¸4¿>¹>Ó7JÈGÓ"TÑˆØŸ™¨¨°Q©Ð7ˆŒØÐr)   c           
      óJ  • U R                   (       d  UR                  SS5      nU R                   HL  nU R                  (       a2  [        R
                  R                  5       (       d  [        X1S S USS9nMF  U" XS9nMN     U R                   (       d  UR                  SS5      nU$ r  )rÄ   r¤   r  r  r   r^   r#  r   ©r$   r   r™   Úrs       r&   r'   ÚCustomTransformer.forward¹  s€   € Ø××Ø—‘˜A˜qÓ!ˆAà—”ˆAØ×&×&¬u¯y©y×/EÑ/E×/GÑ/Gä˜q T¨4°È%ÑP’á�aÑ-’ñ  ð ××Ø—‘˜A˜qÓ!ˆAØˆr)   ©rÄ   r  r  r  r  ©NNF©r   r   )r,   r-   r.   r/   r0   r   rô   r4   rc   rq   r
   r¸   rr   r   r  r   rD   r   r   r  r1   r   r-  r5  r'   r2   rP   rQ   s   @r&   r  r  e  sA  ø† ÙDð  #Ø#'Ø)+¯©Ø*3Ø $Ø1Oñ'àð'ð ð'ð ð	'ð
 ð'ð !ð'ð ˜BŸI™I‘ð'ð ˜RŸY™Y™ð'ð ð'ð ˜s D¨¡I˜~Ñ.÷'ð 'ðR4 §¡ô 4ð 15Ø7;Ø$ñ à�|‰|ð ð   §¡Ñ-ð ð ˜e C¨¨c© NÑ3Ñ4ð	 ð
 õ ñ>°°s¸DÀ¹I°~Ñ1Fõ ñ˜Ÿ™ð °(¸5¿<¹<Ñ2H÷ ó r)   r  c                   óÂ  ^ • \ rS rSrSS\R
                  \SSSSSSSS4S\S\S\S	\S
\S\	\R                     S\	\R                     S\S\\   S\S\S\S\S\S\4U 4S jjjrS\R                   4S jr   S S\R$                  S\\R$                     S\\\\\   4      S\4S jjrS!S\\\\   4   4S jjrS"S\R$                  S\\R$                     4S jjrSrU =r$ )#ÚTransformeriÉ  rÑ   NTFr  r  r  rÒ   rÓ   rÔ   r~   rÄ   Ú
block_typery   Úscaled_cosine_attnr{   rø   rù   rú   c                 óÀ  >• [         TU ]  5         Xl        X l        X€l        SU l        U	c  [        X«XÍXï/5      (       a  Sn	OSn	U	S:X  aI  [        R                  " [        U5       Vs/ s H  n[        UUUUUUU
UUUUUUS9PM     sn5      U l        g [        R                  " [        U5       Vs/ s H  n[        UUUUUUUS9PM     sn5      U l        g s  snf s  snf )NFÚcustomÚdefault)
rÓ   rÔ   r~   ry   r÷   r{   rø   rù   rú   rÄ   )rÓ   rÔ   r~   rÄ   )rC   rD   r  r  rÄ   r  Úanyr   r  Úrangerö   r  rÏ   )r$   r  r  r  rÒ   rÓ   rÔ   r~   rÄ   rA  ry   rB  r{   rø   rù   rú   Ú_rK   s                    €r&   rD   ÚTransformer.__init__Ê  s  ø€ ô$ 	‰ÑÔØŒ
ØŒØ&ÔØ"'ˆÔð ÑÜ�G°ÐPZÐe×fÑfØ%‘
à&�
à˜Ó!ÜŸ]š]ô  ˜vœó!,ò  '�Aô -ØØØØ"/Ø'Ø)Ø#Ø&8Ø +Ø%5Ø)Ø%Ø +ôñ 'ñ!,ó ˆD�Nô&  Ÿ]š]ô ˜vœó,ò '�Aô 'ØØØØ"/Ø'Ø)Ø +ôñ 'ñ,ó ˆD�Nùò',ùò&,s   Á'CÂ0Crã   c                 ó<   • U R                   S   R                  5       $ r  r  rè   s    r&   r  ÚTransformer.get_cast_dtype
  r  r)   r   r™   rf   r  c           
      ó¶  • [        [        U R                  5      U5      u  pVU R                  (       d   UR	                  SS5      R                  5       n/ n[        R                  R                  5       (       d  U(       d  U R                  nOU R                  S US-    n[        U5       Hˆ  u  pšU R                  (       a1  [        R                  R                  5       (       d  [        X¡S S USS9nOU
" XS9nX•;   d  MT  UR                  U R                  (       d  UR	                  SS5      OU5        MŠ     U R                  (       d  UR	                  SS5      nX4$ r  r!  r&  s              r&   r-  Ú!Transformer.forward_intermediates  r/  r)   c                 óv   • [        [        U R                  5      U5      u  p#U R                  SUS-    U l        U$ r1  r3  r4  s       r&   r5  Ú%Transformer.prune_intermediate_layers,  r7  r)   c           
      óf  • U R                   (       d   UR                  SS5      R                  5       nU R                   HL  nU R                  (       a2  [
        R                  R                  5       (       d  [        X1S S USS9nMF  U" XS9nMN     U R                   (       d  UR                  SS5      nU$ r  )	rÄ   r¤   r"  r  r  r   r^   r#  r   r9  s       r&   r'   ÚTransformer.forward3  s‰   € Ø××Ø—‘˜A˜qÓ!×,Ñ,Ó.ˆAà—”ˆAØ×&×&¬u¯y©y×/EÑ/E×/GÑ/Gä˜q T¨4°È%ÑP’á�aÑ-’ñ  ð ××Ø—‘˜A˜qÓ!ˆAØˆr)   r<  r=  r>  r   )r,   r-   r.   r/   r   rô   r4   rc   rq   r
   r¸   rr   r   r  rD   r   r   r  r1   r   r   r-  r5  r'   r2   rP   rQ   s   @r&   r@  r@  É  sƒ  ø† ð  #Ø#'Ø)+¯©Ø*3Ø $Ø(,Ø!Ø',Ø %Ø%*Ø$Ø"ñ!>àð>ð ð>ð ð	>ð
 ð>ð !ð>ð ˜BŸI™I‘ð>ð ˜RŸY™Y™ð>ð ð>ð ! ™ð>ð ð>ð !%ð>ð ð>ð #ð>ð ð>ð  ÷!>ð >ð@4 §¡ô 4ð 15Ø7;Ø$ñ à�|‰|ð ð   §¡Ñ-ð ð ˜e C¨¨c© NÑ3Ñ4ð	 ð
 õ ñ>°°s¸DÀ¹I°~Ñ1Fõ ñ˜Ÿ™ð °(¸5¿<¹<Ñ2H÷ ó r)   r@  Ú
batch_sizec                 óJ   • U R                  SSS5      R                  USS5      $ )Nr   r[   )ÚviewrË   )ÚtokenrR  s     r&   Ú_expand_tokenrV  C  s$   € Ø�:‰:�a˜˜BÓ×&Ñ& z°2°rÓ:Ð:r)   c            5       óŽ  ^ • \ rS rSr% \R
                  R                  \   \S'   SSSSSSSS	S
S\	R                  \SSSSSSSS4S\S\S\S\S\S\S\S\S\S\S\S\S\S\S\S\S\S\S\S\\   S\S\S \S!\S"\S#\44U 4S$ jjjrS=S%\S&\4S' jjrS( r\R
                  R(                  S>S)\4S* jj5       r\R
                  R(                  S+ 5       rS,\R.                  S-\\R.                  \R.                  4   4S. jrS,\R.                  S-\R.                  4S/ jrS,\R.                  S-\\R.                  \R.                  4   4S0 jr      S?S,\R.                  S1\\\\\   4      S2\S3\S4\S5\S6\S-\\\\R.                  \\R.                     4   4   4S7 jjr   S@S1\\\\   4   S8\S9\4S: jjr S,\R.                  4S; jr!S<r"U =r#$ )AÚVisionTransformeriG  Úoutput_tokensNFr¼   rv   é   rZ   Ú	learnableÚtokÚ
image_sizeÚ
patch_sizer  r  r  rÒ   rÓ   Úattentional_poolÚattn_pooler_queriesÚattn_pooler_headsÚ
output_dimÚpatch_dropoutÚ	no_ln_preÚpos_embed_typeÚ	pool_typeÚfinal_ln_after_poolrÔ   r~   rA  ry   rB  r{   rø   rù   rú   c                 ó~  >• [         T!U ]  5         US;   d   eUU l        [        U5      =u  nnU l        [        U5      =u  nnU l        UU-  UU-  4U l        UU l        X°l        [        R                  " SUUUSS9U l        US-  n[        R                  " U[        R                  " U5      -  5      U l        US:X  aT  [        R                  " U[        R                  " U R                  S   U R                  S   -  S-   U5      -  5      U l        OáUS	:X  aÕ  U R                  S   U R                  S   :X  d   S
5       e[        R                  " [        R"                  " U R                  S   U R                  S   -  S-   U5      SS9U l        [%        X0R                  S   SS9nU R                   R&                  R)                  [        R*                  " U5      R-                  5       5        O[.        eUS:”  a  [1        U5      O[        R2                  " 5       U l        U(       a  [        R2                  " 5       OU" U5      U l        [9        UUUUUUUUUUUUUUS9U l        U(       au  [=        U[>        5      (       a8  X€l         SU l!        US;   a#  [E        UUU
U	S9U l#        [E        UUU
SS9U l$        O' eSU l         Xðl!        [E        UUU
U	S9U l#        S U l$        Un OS U l#        Un Xðl!        U" U 5      U l%        [        R                  " U[        R                  " U U5      -  5      U l&        U RO                  5         g )N)r\  ÚavgÚnoner„   F)Úin_channelsÚout_channelsÚkernel_sizeÚstrider!   r‚   r[  r   r   Ú
sin_cos_2dz=currently sin cos 2d pos embedding only supports square input)Úrequires_gradT)Ú	cls_tokenrZ   )
rÓ   rÔ   r~   rA  ry   rB  r{   rø   rù   rú   rj  )ÚparallelÚcascade)r¿   rÀ   Ú )(rC   rD   rY  r   r]  r^  Ú	grid_sizerg  rb  r   ÚConv2dÚconv1rF   r   rd   Úclass_embeddingÚpositional_embeddingr‹   r   ÚdataÚcopy_Ú
from_numpyrq   Ú
ValueErrorrS   r�   rc  Úln_prer@  Útransformerr  r  Úattn_pool_typerf  rº   Ú	attn_poolÚattn_pool_contrastiveÚln_postÚprojÚinit_parameters)"r$   r]  r^  r  r  r  rÒ   rÓ   r_  r`  ra  rb  rc  rd  re  rf  rg  rÔ   r~   rY  rA  ry   rB  r{   rø   rù   rú   Úimage_heightÚimage_widthÚpatch_heightÚpatch_widthr‡   Úpool_dimrK   s"                                    €r&   rD   ÚVisionTransformer.__init__J  s  ø€ ô: 	‰ÑÔØÐ2Ó2Ð2Ð2Ø*ˆÔÜ6?À
Ó6KÐKÑ!ˆ�k D¤OÜ6?À
Ó6KÐKÑ!ˆ�k D¤OØ&¨,Ñ6¸À{Ñ8RÐSˆŒØ#6ˆÔ Ø$Œä—Y’YØØØ"ØØñ
ˆŒ
ð ˜‘ˆÜ!Ÿ|š|¨E´E·K²KÀÓ4FÑ,FÓGˆÔØ˜[Ó(Ü(*¯ªØœŸš D§N¡N°1Ñ$5¸¿¹ÀqÑ8IÑ$IÈAÑ$MÈuÓUÑUó)WˆDÕ%à˜|Ó+à—>‘> !Ñ$¨¯©°qÑ(9Ó9ð PØOóPÐ9ä(*¯ªÜ—’˜DŸN™N¨1Ñ-°·±¸qÑ0AÑAÀAÑEÀuÓMÐ]bñ)dˆDÔ%ä4°U¿N¹NÈ1Ñ<MÐY]Ñ^ˆNØ×%Ñ%×*Ñ*×0Ñ0´×1AÒ1AÀ.Ó1Q×1WÑ1WÓ1YÕZäÐð =JÈBÓ<Nœ\¨-Ô8ÔTV×T_ÒT_ÓTaˆÔæ'0”b—k’k”m±jÀÓ6GˆŒÜ&ØØØØØ'ØØ!Ø!ØØ1Ø#Ø-Ø!Øñ
ˆÔö" ÜÐ*¬C×0Ñ0Ø&6Ô#Ø!'�”Ø#Ð'>Ó>Ü%6Ø"ØØ0Ø"5ñ	&�D”Nô 2CØ"ØØ0Ø"#ñ	2�DÕ.ð !˜5à&(�Ô#Ø!*”Ü!2ØØØ,Ø1ñ	"�”ð .2�Ô*Ø!‰Hà!ˆDŒNØˆHØ&ŒNá! (Ó+ˆŒÜ—L’L ¬¯ª°X¸zÓ)JÑ!JÓKˆŒ	à×ÑÕr)   Úunlocked_groupsÚfreeze_bn_statsc                 ód  ^• U R                  5        H
  nSUl        M     US:w  a‹  U R                  U R                  U R                  U R
                  //U R                  R                  S S QU R                  R                  S   U R                  /PU R                  PnU4S jmT" XA* S  5        g g )NFr   r[   c                 óò   >• [        U [        5      (       a  U  H  nT" U5        M     g [        U [        R                  R                  5      (       a  SU l        g U R                  5        H
  nSUl        M     g ©NT)r  r   r   r   rF   rp  Ú
parameters)r   Úgr­   Ú_unlocks      €r&   r“  Ú'VisionTransformer.lock.<locals>._unlockÚ  sX   ø€ Ü˜a¤×*Ñ*Û˜Ù ž
ò ô " !¤U§X¡X×%7Ñ%7×8Ñ8Ø*.˜�à!"§¡¦˜AØ.2˜AžOò "0r)   )
r‘  rp  rw  rx  ry  r~  r  r  rƒ  r„  )r$   rŒ  r�  ÚparamÚgroupsr“  s        @r&   ÚlockÚVisionTransformer.lockÆ  sÃ   ø€ Ø—_‘_Ö&ˆEØ"'ˆEÖñ 'ð ˜aÓð —J‘JØ×(Ñ(Ø×-Ñ-Ø—K‘Kð	ðð ×!Ñ!×+Ñ+¨C¨RÐ0ðð ×$Ñ$×.Ñ.¨rÑ2Ø—L‘Lððð —	‘	ðˆFõ	3ñ �FÐ+Ð,Ð-Õ.ð7  r)   c                 ó   • g r   r*   rè   s    r&   r…  Ú!VisionTransformer.init_parametersç  s   € ð$ 	r)   Úenablec                 ó$   • XR                   l        g r   ©r  r  ©r$   r›  s     r&   Úset_grad_checkpointingÚ(VisionTransformer.set_grad_checkpointingû  ó   € à.4×ÑÕ+r)   c                 ó   • SS1nU$ )Nry  rx  r*   ©r$   Úno_wds     r&   Úno_weight_decayÚ!VisionTransformer.no_weight_decayÿ  s   € ð (Ð):Ð;ˆØˆr)   r   rã   c                 óÄ   • U R                   S:X  a%  US S 2SS 24   R                  SS9US S 2SS 24   p2X#4$ U R                   S:X  a  US S 2S4   US S 2SS 24   p2X#4$ U=p#X#4$ )Nri  r   r\   r\  r   )rf  Úmean©r$   r   ÚpooledÚtokenss       r&   Ú_global_poolÚVisionTransformer._global_pool  s�   € Ø�>‰>˜UÓ"Øšq !¡"˜u™XŸ]™]¨q˜]Ð1°1²Q¸¹°U±8�Fð ˆ~Ðð �^‰^˜uÓ$Øšq !˜t™W aª¨1©2¨¡h�Fð ˆ~Ðð  ÐˆFàˆ~Ðr)   c                 óÜ  • U R                  U5      nUR                  UR                  S   UR                  S   S5      nUR                  SSS5      n[        R
                  " [        U R                  UR                  S   5      R                  UR                  5      U/SS9nXR                  R                  UR                  5      -   nU R                  U5      nU R                  U5      nU$ )Nr   r   r[   r›   r\   )rw  r£   r    Úpermuter   rg   rV  rx  r   r   ry  rc  r~  r=   s     r&   Ú_embedsÚVisionTransformer._embeds  sÃ   € Ø�J‰J�q‹MˆØ�I‰I�a—g‘g˜a‘j !§'¡'¨!¡*¨bÓ1ˆØ�I‰I�a˜˜AÓˆô �IŠI”} T×%9Ñ%9¸1¿7¹7À1¹:ÓF×IÑIÈ!Ï'É'ÓRÐTUÐVÐ\]Ñ^ˆà×)Ñ)×,Ñ,¨Q¯W©WÓ5Ñ5ˆð ×Ñ˜qÓ!ˆð �K‰K˜‹NˆØˆr)   c                 ó:  • U R                   b°  U R                  bk  U R                  U5      nU R                  U5      nU R                  S:X  a  U R                  U5      nO\U R                  S:X  d   eU R                  U5      n X24$ U R                  U5      nU R                  U5      nU R	                  U5      u  p2X24$ U R
                  (       a'  U R	                  U5      u  p2U R                  U5      nX24$ U R                  U5      nU R	                  U5      u  p2X24$ )Nrr  rs  )r�  r‚  rƒ  r€  r¬  rg  )r$   r   r«  rª  s       r&   Ú_poolÚVisionTransformer._pool   s  € Ø�>‰>Ñ%Ø×)Ñ)Ñ5à—L‘L “O�ØŸ™¨Ó*�Ø×&Ñ&¨*Ó4Ø!×7Ñ7¸Ó:‘Fà×.Ñ.°)Ó;Ð;Ð;Ø!×7Ñ7¸Ó?‘Fð ˆ~Ðð —N‘N 1Ó%�Ø—L‘L “O�Ø!%×!2Ñ!2°1Ó!5‘�ð ˆ~Ðð ×%×%Ø!×.Ñ.¨qÓ1‰NˆFØ—\‘\ &Ó)ˆFð
 ˆ~Ðð —‘˜Q“ˆAØ!×.Ñ.¨qÓ1‰NˆFàˆ~Ðr)   rf   r  Únormalize_intermediatesÚintermediates_onlyÚ
output_fmtÚoutput_extra_tokensc           	      ó  • US;   d   S5       eUS:H  nUR                   u  pšp¼U R                  U5      nU R                  R                  UUUS9u  pU(       a   U Vs/ s H  oàR	                  U5      PM     nnSnU(       a7  U Vs/ s H  nUSS2SU24   PM     nnU Vs/ s H  nUSS2US24   PM     nnOSnU(       af  X°R
                  S   -  XÀR
                  S   -  nnU Vs/ s H7  nUR                  U	UUS5      R                  SS	SS
5      R                  5       PM9     nnSU0nUb  U(       a  UUS'   U(       a  U$ U R                  U5      u  nn
U R                  b  UU R                  -  nUUS'   U$ s  snf s  snf s  snf s  snf )a÷  Forward features that returns intermediates.

Args:
    x: Input image tensor
    indices: Take last n blocks if int, all if None, select matching indices if sequence
    stop_early: Stop iterating over blocks when last desired intermediate hit
    intermediates_only: Only return intermediate features
    normalize_intermediates: Apply final norm layer to all intermediates
    output_fmt: Shape of intermediate feature outputs
    output_extra_tokens: Return both extra prefix class tokens
Returns:

)ÚNCHWÚNLCz)Output format must be one of NCHW or NLC.rº  )rf   r  r   Nr   r[   r„   r›   Úimage_intermediatesÚimage_intermediates_prefixÚimage_features)r    r°  r  r-  rƒ  r^  r£   r¯  r"  r³  r„  )r$   r   rf   r  rµ  r¶  r·  r¸  r£   ÚBrH  Úheightr  r)  ÚxiÚnum_prefix_tokensÚyÚprefix_tokensÚHÚWÚoutputrª  s                         r&   r-  Ú'VisionTransformer.forward_intermediates9  sµ  € ð. ˜_Ó,ÐYÐ.YÓYÐ,Ø Ñ&ˆð  Ÿg™gÑˆˆfØ�L‰L˜‹OˆØ×+Ñ+×AÑAØØØ!ð Bð 
Ñˆö #á8EÓFº°"Ÿ\™\¨"Ö-¹ˆMÐFØÐÞá@MÓNÂ¸1˜Qšq !Ð$5Ð"5Ð5Ô6ÁˆMÐNÙ?LÓMº}¸!˜QšqÐ"3Ñ"4Ð4Ô5¹}ˆMÐMˆMà ˆMÞàŸ_™_¨QÑ/Ñ/°¿/¹/È!Ñ:LÑ1LˆqˆAÙ^kÓlÒ^kÐYZ˜QŸY™Y q¨!¨Q°Ó3×;Ñ;¸A¸qÀ!ÀQÓG×RÑRÖTÑ^kˆMÐlà'¨Ð7ˆØÑ$Ö)<Ø3@ˆFÐ/Ñ0æØˆMà—J‘J˜q“M‰	ˆ�à�9‰9Ñ Ø˜dŸi™iÑ'ˆFà#)ˆÐÑ àˆùò7 Gùò OùÚMùò ms   ÁE9ÂE>ÂFÃ&>FÚ
prune_normÚ
prune_headc                 óš   • U R                   R                  U5      nU(       a  [        R                  " 5       U l        U(       a  SU l        U$ ©r2  N)r  r5  r   r�   rƒ  r„  ©r$   rf   rÉ  rÊ  r'  s        r&   r5  Ú+VisionTransformer.prune_intermediate_layers|  s:   € ð ×'Ñ'×AÑAÀ'ÓJˆÞÜŸ;š;›=ˆDŒLÞØˆDŒIØÐr)   c                 óÎ   • U R                  U5      nU R                  U5      nU R                  U5      u  p#U R                  b  X R                  -  nU R                  (       a  X#4$ U$ r   )r°  r  r³  r„  rY  r©  s       r&   r'   ÚVisionTransformer.forward‹  sZ   € Ø�L‰L˜‹OˆØ×Ñ˜QÓˆØŸ™ A›‰ˆà�9‰9Ñ ØŸi™iÑ'ˆFà××Ø�>Ð!àˆr)   )r�  r‚  r€  rx  rw  rg  ru  r]  rƒ  r~  rb  rY  rc  r^  rf  ry  r„  r  )r   F©T©NFFFrº  F©r   FT)$r,   r-   r.   r/   r   r^   ÚFinalrr   Ú__annotations__r   rô   r4   rc   rq   r  r   r   rD   r—  r…  ÚignorerŸ  r¥  r1   r	   r¬  r°  r³  r   r   r   r-  r5  r'   r2   rP   rQ   s   @r&   rX  rX  G  sD  ø‡ Ø—9‘9—?‘? 4Ñ(Ó(ð $(Ø%*Ø'*Ø%&Ø!Ø#%Ø#Ø"-Ø"Ø(-Ø"$§'¡'Ø#,Ø"'Ø(,Ø!Ø',Ø %Ø%*Ø$Ø"ñ7zàðzð ðzð ð	zð
 ðzð ðzð ðzð !ðzð #ðzð "%ðzð  #ðzð ðzð !ðzð ðzð  ðzð  ð!zð" "&ð#zð$  ð%zð& !ð'zð(  ð)zð* ! ™ð+zð, ð-zð. !%ð/zð0 ð1zð2 #ð3zð4 ð5zð6 ÷7zð zñx/ Cð /¸dõ /òBð( ‡Y�Y×Ññ5¨Tô 5ó ð5ð ‡Y�Y×Ññó ðð
˜eŸl™lð ¨u°U·\±\À5Ç<Á<Ð5OÑ/Pô ð˜Ÿ™ð ¨¯©ô ð"�u—|‘|ð ¨¨e¯l©l¸E¿L¹LÐ.HÑ(Iô ð8 8<Ø$Ø,1Ø',Ø$Ø(-ñAà�|‰|ðAð ˜e C¨¨c© NÑ3Ñ4ðAð ð	Að
 &*ðAð !%ðAð ðAð "&ðAð 
ˆc�5˜Ÿ™ t¨E¯L©LÑ'9Ð9Ñ:Ð:Ñ	;õAðJ ./Ø$Ø#ñ	à˜3  S¡	˜>Ñ*ðð ðð õ	ð˜Ÿ™÷ ò r)   rX  r   Útextrf  Úeos_token_idrã   c                 ó¨  • US:X  a  U S S 2S4   nU$ US:X  a  U S S 2S4   nU$ US:X  aE  Uc   eU [         R                  " U R                  S   U R                  S9UR	                  SS94   nU$ US:X  a\  Uc   eUc   eX:H  R                  5       R	                  SS9nU [         R                  " U R                  S   U R                  S9U4   nU$ U nU$ )	NÚfirstr   Úlastr[   Úargmax)Údevicer\   Úeos)r   ra   r    rÝ  rÜ  rc   )r   r×  rf  rØ  rª  Úidxs         r&   Útext_global_poolrà  ™  s  € ð �GÓØ’1�a�4‘ˆð  €Mð 
�fÓ	Ø’1�b�5‘ˆð €Mð 
�hÓ	àÑÐÐØ”5—<’< §¡¨¡
°1·8±8Ñ<¸d¿k¹kÈb¸kÐ>QÐQÑRˆð €Mð 
�eÓ	àÑÐÐØÑ'Ð'Ð'ØÑ#×(Ñ(Ó*×1Ñ1°bÐ1Ð9ˆØ”5—<’< §¡¨¡
°1·8±8Ñ<¸cÐAÑBˆð €Mð ˆà€Mr)   c            7       ón  ^ • \ rS rSr% \R
                  R                  \   \S'   SSSSSSS	SS
S
S
S
SSSSS
\	R                  \S
S	S
S
S
S
S
S
4S\S\S\S\S\S\S\S\\   S\S\S\S\S\S\S\S\S\S \\	R"                     S!\\	R"                     S\S"\\   S#\S$\S%\S&\S'\S(\46U 4S) jjjrS* r\R
                  R(                  SCS+ j5       rSDS,\S-\4S. jjr\R
                  R(                  S/ 5       rS0 rS1\R2                  S2\S3\R4                  S4\R2                  4S5 jrS4\\R2                  \\R2                     4   4S6 jr      SES1\R2                  S7\\\\\   4      S8\S9\S:\S;\S<\S4\ \\\R2                  \\R2                     4   4   4S= jjr!   SFS7\\\\   4   S>\S?\4S@ jjr"SA r#SBr$U =r%$ )GÚTextTransformeri³  rY  éM   i Á  rZ  rv   é   rÑ   NFr   r›   rÜ  r¡   Úcontext_lengthÚ
vocab_sizer  r  r  rÒ   rÓ   rb  Ú	embed_clsÚno_causal_maskÚuse_pad_maskÚcorrect_cls_maskÚpad_idÚeos_idrf  Ú	proj_typeÚ	proj_biasrÔ   r~   rA  ry   rB  r{   rø   rù   rú   c                 óÆ  >• [         TU ]  5         US;   d   eUU l        U=U l        U l        X l        X0l        X€l        X@l        XÐl	        Xàl
        Xðl        U=(       a    U
U l        XÀl        [        R                  " X#5      U l        U	(       aE  [        R"                  " [$        R&                  " U5      5      U l        U =R                  S-  sl        OS U l        [        R"                  " [$        R&                  " U R                  U5      5      U l        [-        UUUUUUUUUUUUUUS9U l        U" U5      U l        U
(       a  S U l        OU R5                  SU R7                  5       SS9  US:X  d  U(       d  S U l        ORU(       a  [        R:                  " X85      U l        O/[        R"                  " [$        R&                  " X85      5      U l        U R=                  5         g )N)rÚ  rÛ  rÜ  rÞ  rj  r   )r  r  r  rÒ   rÓ   rÔ   r~   rA  ry   rB  r{   rø   rù   rú   r™   F©Ú
persistentrj  )rC   rD   rY  Únum_posrå  ræ  r  rb  r  rë  rì  rf  ré  rê  r   Ú	EmbeddingÚtoken_embeddingrF   r   ÚemptyÚcls_embry  r@  r  Úln_finalr™   Úregister_bufferÚbuild_causal_maskÚtext_projectionr•   r…  )r$   rå  ræ  r  r  r  rÒ   rÓ   rb  rç  rè  ré  rê  rë  rì  rf  rí  rî  rÔ   r~   rY  rA  ry   rB  r{   rø   rù   rú   rK   s                               €r&   rD   ÚTextTransformer.__init__¶  sˆ  ø€ ô< 	‰ÑÔØÐFÓFÐFÐFØ*ˆÔØ-;Ð;ˆŒ�tÔ*Ø$ŒØŒ
Ø$ŒØŒ
ØŒØŒØ"ŒØ(×;¨^ˆÔØ 0Ôä!Ÿ|š|¨JÓ>ˆÔÞÜŸ<š<¬¯ª°EÓ(:Ó;ˆDŒLØ�LŠL˜AÑŽLàˆDŒLÜ$&§L¢L´·²¸T¿\¹\È5Ó1QÓ$RˆÔ!Ü&ØØØØØ'ØØ!Ø!ØØ1Ø#Ø-Ø!Øñ
ˆÔñ  # 5Ó)ˆŒæØ!ˆD�Nà× Ñ  ¨d×.DÑ.DÓ.FÐSXÐ ÑYà˜Ó¦jØ#'ˆDÕ æÜ')§y¢y°Ó'C�Õ$ä')§|¢|´E·K²KÀÓ4RÓ'S�Ô$à×ÑÕr)   c                 ó  • [         R                  R                  U R                  R                  SS9  [         R                  R                  U R
                  SS9  U R                  b(  [         R                  R                  U R                  SS9  U R                  R                  S-  SU R                  R                  -  S-  -  nU R                  R                  S-  nSU R                  R                  -  S-  nU R                  R                   Hé  n[         R                  R                  UR                  R                  US9  [         R                  R                  UR                  R                  R                  US9  [         R                  R                  UR                  R                  R                  US9  [         R                  R                  UR                  R                   R                  US9  Më     U R"                  bþ  [%        U R"                  [         R&                  5      (       a•  [         R                  R                  U R"                  R                  U R                  R                  S-  S9  U R"                  R(                  b4  [         R                  R+                  U R"                  R(                  5        g g [         R                  R                  U R"                  U R                  R                  S-  S9  g g )Ng{®Gáz”?©Ústdg{®Gáz„?r‚   r›   )r   ÚinitÚnormal_rô  r    ry  rö  r  r  r  r  rµ   rŠ   r–   rß   r×   rÙ   rú  r  r•   r!   Úzeros_©r$   Úproj_stdÚattn_stdÚfc_stdÚblocks        r&   r…  ÚTextTransformer.init_parameters
  s  € Ü
�‰�‰˜×,Ñ,×3Ñ3¸ˆÑ>Ü
�‰�‰˜×1Ñ1°tˆÑ<Ø�<‰<Ñ#Ü�G‰G�O‰O˜DŸL™L¨dˆOÑ3à×$Ñ$×*Ñ*¨dÑ2¸¸D×<LÑ<L×<SÑ<SÑ8SÐX\Ñ7\Ñ]ˆØ×#Ñ#×)Ñ)¨TÑ1ˆØ�d×&Ñ&×,Ñ,Ñ,°Ñ5ˆØ×%Ñ%×/Ô/ˆEÜ�G‰G�O‰O˜EŸJ™J×5Ñ5¸8ˆOÑDÜ�G‰G�O‰O˜EŸJ™J×/Ñ/×6Ñ6¸HˆOÑEÜ�G‰G�O‰O˜EŸI™IŸN™N×1Ñ1°vˆOÑ>Ü�G‰G�O‰O˜EŸI™I×,Ñ,×3Ñ3¸ˆOÓBñ	 0ð ×ÑÑ+Ü˜$×.Ñ.´·	±	×:Ñ:Ü—‘—‘ × 4Ñ 4× ;Ñ ;À×AQÑAQ×AWÑAWÐ[_ÑA_�Ñ`Ø×'Ñ'×,Ñ,Ñ8Ü—G‘G—N‘N 4×#7Ñ#7×#<Ñ#<Õ=ð 9ô —‘—‘ × 4Ñ 4¸$×:JÑ:J×:PÑ:PÐTXÑ:X�ÒYð ,r)   c                 ó$   • XR                   l        g r   r�  rž  s     r&   rŸ  Ú&TextTransformer.set_grad_checkpointing!  r¡  r)   Úunlocked_layersÚfreeze_layer_normc                 ó6   • U(       d   S5       e[        X5        g)zð
Lock the text transformer layers, optionally leaving some layers unlocked.

Args:
    unlocked_layers: Number of layers to leave unlocked (from the end).
    freeze_layer_norm: LayerNorm freeze (only for API compatibility, not functional)
zLUnfreezing LayerNorm is not supported. LayerNorm treated like other weights.N)Úlock_text_tower)r$   r
  r  s      r&   r—  ÚTextTransformer.lock%  s   € ö !ÐpÐ"pÓpÐ Ü˜Õ.r)   c                 óH   • S1nU R                   b  UR                  S5        U$ )Nry  rö  )rö  Úaddr£  s     r&   r¥  ÚTextTransformer.no_weight_decay0  s'   € ð (Ð(ˆØ�<‰<Ñ#Ø�I‰I�iÔ Øˆr)   c                 ó²   • [         R                  " U R                  U R                  5      nUR                  [	        S5      5        UR                  S5        U$ ©Nr�   r   )r   rõ  rò  Úfill_rq   Útriu_©r$   Úmasks     r&   rù  Ú!TextTransformer.build_causal_mask8  s<   € ô �{Š{˜4Ÿ<™<¨¯©Ó6ˆØ�
‰
”5˜“=Ô!Ø�
‰
�1ŒØˆr)   r×  Úseq_lenr   rã   c                 ó²  • XR                   :g  nU R                  bM  UR                  UR                  S5      S5      n[        R
                  " U R                  (       a  XE/OXT/S5      nUR                  S5      R                  SUS5      n[        R                  " XcS9nUR                  U) [        S5      5        UR                  U R                  S5      nU$ )z�
Returns an additive (-inf) mask of shape [B*heads, seq_len, seq_len] that
simultaneously masks padding tokens and (optionally) the CLS token.
r   r   r[   rœ   r�   )rë  rö  Únew_onesr`   r   rg   rê  rÊ   rË   r¦   r§   rq   Úrepeat_interleaver  )r$   r×  r  r   ÚvalidÚ	cls_validÚkey_maskÚadditives           r&   Ú_build_additive_maskÚ$TextTransformer._build_additive_mask@  s´   € ð Ÿ™Ñ#ˆà�<‰<Ñ#ØŸ™ u§z¡z°!£}°aÓ8ˆIä—I’I°D×4I×4I˜uÑ0ÐPYÐOaÐcdÓeˆEð —?‘? 1Ó%×,Ñ,¨R°¸"Ó=ˆÜ×#Ò# HÑ:ˆØ×Ñ˜x˜i¬¨v«Ô7Ø×-Ñ-¨d¯j©j¸!Ó<ˆØˆr)   c           	      óF  • U R                   R                  5       nUR                  u  p4U R                  U5      R	                  U5      nU R
                  bA  [        R                  " U[        U R
                  UR                  S5      5      /S5      nUS-  nU R                  nU R                  (       d  U R
                  b?  U R                  XUR                  5      nUb  US U2S U24   R                  S5      U-   nOUnXPR                  S U R	                  U5      -   nXV4$ )Nr   r   )r  r  r    rô  r   rö  r   rg   rV  r`   r™   ré  r!  r   rÊ   ry  )r$   r×  Ú
cast_dtyper¿  r  r   r™   Úadd_masks           r&   r°  ÚTextTransformer._embedsX  s  € Ø×%Ñ%×4Ñ4Ó6ˆ
Ø—Z‘Z‰
ˆà× Ñ  Ó&×)Ñ)¨*Ó5ˆð �<‰<Ñ#Ü—	’	˜1œm¨D¯L©L¸!¿&¹&À»)ÓDÐEÀqÓIˆAØ�q‰LˆGà—N‘Nˆ	ð ×× §¡Ñ 8Ø×1Ñ1°$ÀÇÁÓIˆHØÑ$à% h w h°°°Ð&8Ñ9×CÑCÀAÓFÈÑQ‘	à$�	à×)Ñ)¨(¨7Ð3×6Ñ6°zÓBÑBˆØˆ|Ðr)   rf   r  rµ  r¶  r·  r¸  c           
      ó  • US;   d   S5       eU R                  U5      u  p‰U R                  R                  UU	UUS9u  pŠU(       a   U
 Vs/ s H  o°R                  U5      PM     n
n0 nU R                  bA  U
 Vs/ s H  o»SS2SS24   PM     nnU(       a  U
 Vs/ s H  o»SS2SS24   PM     nnXìS'   Un
X¬S'   U(       a  U$ U R                  b  [        USS	9nU R                  U5      nO1U R                  U5      n[        X�U R                  [        U S
S5      S9nU R                  bI  [        U R                  [        R                  5      (       a  U R                  U5      nOXðR                  -  nXüS'   U$ s  snf s  snf s  snf )aõ  Forward features that returns intermediates.

Args:
    text: Input text ids
    indices: Take last n blocks if int, all if None, select matching indices if sequence
    stop_early: Stop iterating over blocks when last desired intermediate hit
    normalize_intermediates: Apply norm layer to all intermediates
    intermediates_only: Only return intermediate features
    output_fmt: Shape of intermediate feature outputs
    output_extra_tokens: Return both prefix and intermediate tokens
Returns:

)r»  zOutput format must be NLC.)r™   rf   r  Nr[   Útext_intermediates_suffixÚtext_intermediatesrÛ  ©rf  rì  ©rf  rØ  Útext_features)r°  r  r-  r÷  rö  rà  rf  Úgetattrrú  r  r   r•   )r$   r×  rf   r  rµ  r¶  r·  r¸  r   r™   r)  rÁ  rÇ  Úseq_intermediatesÚcls_intermediatesrª  s                   r&   r-  Ú%TextTransformer.forward_intermediatesq  s’  € ð. ˜XÓ%ÐCÐ'CÓCÐ%à—|‘| DÓ)‰ˆØ×+Ñ+×AÑAØØØØ!ð	 Bð 
Ñˆö #á9FÓGº°2Ÿ]™]¨2Ö.¹ˆMÐGàˆà�<‰<Ñ#Ù6CÓ D²m°¢A s¨ s F¤±mÐÐ DÞ"á:GÓ$Hº-°Bª¨2©3¨¤Z¹-Ð!Ð$HØ6GÐ2Ñ3Ø-ˆMØ'4Ð#Ñ$æØˆMà�<‰<Ñ#ä% a°6Ñ:ˆFØ—]‘] 6Ó*‰Fà—‘˜aÓ ˆAÜ% a¸¿¹ÔV]Ð^bÐdlÐnrÓVsÑtˆFà×ÑÑ+Ü˜$×.Ñ.´·	±	×:Ñ:Ø×-Ñ-¨fÓ5‘à×"6Ñ"6Ñ6�à"(ˆÑàˆùòA Hùò
 !Eùò %Is   Á
E9Á9E>ÂFrÉ  rÊ  c                 óš   • U R                   R                  U5      nU(       a  [        R                  " 5       U l        U(       a  SU l        U$ rÌ  )r  r5  r   r�   r÷  rú  rÍ  s        r&   r5  Ú)TextTransformer.prune_intermediate_layers·  s;   € ð ×'Ñ'×AÑAÀ'ÓJˆÞÜŸKšK›MˆDŒMÞØ#'ˆDÔ ØÐr)   c           
      óì  • U R                  U5      u  p#U R                  X#S9nU R                  b'  [        USS9nU R	                  U5      nUS S 2S S24   nO3U R	                  U5      n[        X!U R
                  [        U SS 5      S9nUnU R                  bI  [        U R                  [        R                  5      (       a  U R                  U5      nOX@R                  -  nU R                  (       a  XE4$ U$ )Nr  rÛ  r*  r[   rì  r+  )r°  r  rö  rà  r÷  rf  r-  rú  r  r   r•   rY  )r$   r×  r   r™   rª  r«  s         r&   r'   ÚTextTransformer.forwardÆ  så   € Ø—|‘| DÓ)‰ˆà×Ñ˜QÐÐ4ˆð �<‰<Ñ#ä% a°6Ñ:ˆFØ—]‘] 6Ó*ˆFØ’q˜#˜2˜#�v‘Y‰Fà—‘˜aÓ ˆAÜ% a¸¿¹ÔV]Ð^bÐdlÐnrÓVsÑtˆFØˆFà×ÑÑ+Ü˜$×.Ñ.´·	±	×:Ñ:Ø×-Ñ-¨fÓ5‘à×"6Ñ"6Ñ6�à××Ø�>Ð!àˆr)   )r™   rö  rå  rê  rì  r  r÷  rò  rb  rY  rë  rf  ry  rú  rô  r  ré  ræ  r  rÑ  )r   TrÒ  rÓ  )&r,   r-   r.   r/   r   r^   rÔ  rr   rÕ  r   rô   r4   rc   rq   r   r  r
   r¸   rD   r…  rÖ  rŸ  r—  r¥  rù  r1   r   r!  r	   r°  r   r   r   r-  r5  r'   r2   rP   rQ   s   @r&   râ  râ  ³  sX  ø‡ Ø—9‘9—?‘? 4Ñ(Ó(ð #%Ø#ØØØØ"Ø#'Ø(+Ø#Ø#(Ø!&Ø%*ØØØ%Ø%Ø#Ø)+¯©Ø*3Ø"'Ø(,Ø!Ø',Ø %Ø%*Ø$Ø"ñ9RàðRð ðRð ð	Rð
 ðRð ðRð ðRð !ðRð ! ™ðRð ðRð !ðRð ðRð #ðRð ðRð ðRð  ð!Rð" ð#Rð$ ð%Rð& ˜BŸI™I‘ð'Rð( ˜RŸY™Y™ð)Rð*  ð+Rð, ! ™ð-Rð. ð/Rð0 !%ð1Rð2 ð3Rð4 #ð5Rð6 ð7Rð8 ÷9Rð RòhZð. ‡Y�Y×Ñó5ó ð5ñ	/ Cð 	/Àõ 	/ð ‡Y�Y×Ññó ðòðà�l‰lðð ðð �{‰{ð	ð
 
�‰ôð0˜u U§\¡\°8¸E¿L¹LÑ3IÐ%IÑJô ð8 8<Ø$Ø,1Ø',Ø$Ø(-ñDà—,‘,ðDð ˜e C¨¨c© NÑ3Ñ4ðDð ð	Dð
 &*ðDð !%ðDð ðDð "&ðDð 
ˆc�5˜Ÿ™ t¨E¯L©LÑ'9Ð9Ñ:Ð:Ñ	;õDðP ./Ø$Ø#ñ	à˜3  S¡	˜>Ñ*ðð ðð õ	÷ð r)   râ  c                   ód  ^ • \ rS rSrSSS\R
                  \SS4S\S\S	\S
\S\S\S\	\R                     S\	\R                     S\S\4U 4S jjjrS rS r   SS\R                   S\\R                      S\\\\\   4      S\4S jjrS r\R,                  R.                  SS j5       rSrU =r$ )ÚMultimodalTransformeriâ  rã  rÑ   NrZ  Tr  r  r  rå  rÒ   rÓ   rÔ   r~   rb  rÄ   c                 ó€  >• [         TU ]  UUUUUUUU
S9  X@l        [        R                  " [        U5       Vs/ s H  n[        UUUUUUSU
S9PM     sn5      U l        U R                  SU R                  5       SS9  U" U5      U l
        [        R                  " [        R                  " X5      5      U l        g s  snf )N)r  r  r  rÒ   rÓ   rÔ   r~   rÄ   T)rÓ   rÔ   r~   rÕ   rÄ   r™   Frð  )rC   rD   rå  r   r  rG  rÏ   Ú
cross_attnrø  Úbuild_attention_maskr÷  rF   r   rõ  rú  )r$   r  r  r  rå  rÒ   rÓ   rÔ   r~   rb  rÄ   rH  rK   s               €r&   rD   ÚMultimodalTransformer.__init__ã  sÕ   ø€ ô 	‰ÑØØØØØ'ØØ!Ø#ð 	ñ 		
ð -ÔÜŸ-š-ô ˜6”]ó)
ò #�ô #ØØØØ+Ø#Ø%Ø#'Ø'ô	ñ #ñ)
ó ˆŒð 	×Ñ˜[¨$×*CÑ*CÓ*EÐRWÐÑXá" 5Ó)ˆŒÜ!Ÿ|š|¬E¯KªK¸Ó,JÓKˆÕùò#)
s   ¹B;c                 ó€  • U R                   R                  S-  SU R                   R                  -  S-  -  nU R                   R                  S-  nSU R                   R                  -  S-  nU R                   R                   Hé  n[        R
                  R                  UR                  R                  US9  [        R
                  R                  UR                  R                  R                  US9  [        R
                  R                  UR                  R                  R                  US9  [        R
                  R                  UR                  R                  R                  US9  Më     U R                   R                   Hé  n[        R
                  R                  UR                  R                  US9  [        R
                  R                  UR                  R                  R                  US9  [        R
                  R                  UR                  R                  R                  US9  [        R
                  R                  UR                  R                  R                  US9  Më     U R                  b@  [        R
                  R                  U R                  U R                   R                  S-  S9  g g )Nr‚   r›   rý  )r  r  r  r  r   rÿ  r   rµ   rŠ   r–   r    rß   r×   rÙ   r8  rú  r  s        r&   r…  Ú%MultimodalTransformer.init_parameters  sâ  € Ø×$Ñ$×*Ñ*¨dÑ2¸¸D×<LÑ<L×<SÑ<SÑ8SÐX\Ñ7\Ñ]ˆØ×#Ñ#×)Ñ)¨TÑ1ˆØ�d×&Ñ&×,Ñ,Ñ,°Ñ5ˆØ×%Ñ%×/Ô/ˆEÜ�G‰G�O‰O˜EŸJ™J×5Ñ5¸8ˆOÑDÜ�G‰G�O‰O˜EŸJ™J×/Ñ/×6Ñ6¸HˆOÑEÜ�G‰G�O‰O˜EŸI™IŸN™N×1Ñ1°vˆOÑ>Ü�G‰G�O‰O˜EŸI™I×,Ñ,×3Ñ3¸ˆOÓBñ	 0ð
 ×%Ñ%×0Ô0ˆEÜ�G‰G�O‰O˜EŸJ™J×5Ñ5¸8ˆOÑDÜ�G‰G�O‰O˜EŸJ™J×/Ñ/×6Ñ6¸HˆOÑEÜ�G‰G�O‰O˜EŸI™IŸN™N×1Ñ1°vˆOÑ>Ü�G‰G�O‰O˜EŸI™I×,Ñ,×3Ñ3¸ˆOÓBñ	 1ð ×ÑÑ+Ü�G‰G�O‰O˜D×0Ñ0°d×6FÑ6F×6LÑ6LÐPTÑ6TˆOÒUð ,r)   c                 ó²   • [         R                  " U R                  U R                  5      nUR                  [	        S5      5        UR                  S5        U$ r  )r   rõ  rå  r  rq   r  r  s     r&   r9  Ú*MultimodalTransformer.build_attention_mask   s@   € ô �{Š{˜4×.Ñ.°×0CÑ0CÓDˆØ�
‰
”5˜“=Ô!Ø�
‰
�1ŒØˆr)   r   r™   rf   r  c                 ó   •  S5       e)Nz<Not currently implemented for MultimodalTransformer w/ xattnr*   )r$   r   r™   rf   r  s        r&   r-  Ú+MultimodalTransformer.forward_intermediates(  s   € ð 	UÐTÓTˆur)   c                 óˆ  • UR                   S   nU R                  (       d&  UR                  SSS5      nUR                  SSS5      n[        U R                  U R
                  5       H‡  u  pEU R                  (       aQ  [        R                  R                  5       (       d.  [        XBS S U R                  S U2S U24   SS9n[        XRXS SS9nMg  U" X R                  S U2S U24   S9nU" X!US9nM‰     U R                  (       d  UR                  SSS5      nU R                  U5      nU R                  b  X`R                  -  nU$ )Nr   r   r›   Fr  r  )rí   rî   )r    rÄ   r¯  Úzipr  r8  r  r   r^   r#  r   r™   r÷  rú  )r$   Ú
image_embsÚ	text_embsr  Úresblockr8  rÌ   s          r&   r'   ÚMultimodalTransformer.forward1  s8  € Ø—/‘/ !Ñ$ˆØ××Ø#×+Ñ+¨A¨q°!Ó4ˆJØ!×)Ñ)¨!¨Q°Ó2ˆIä$'¨¯©¸¿¹Ö$HÑ ˆHØ×&×&¬u¯y©y×/EÑ/E×/GÑ/Gä&Ø¨¨t°T·^±^ÀHÀWÀHÈhÈwÈhÐDVÑ5WÐglñn�	ä&Ø¨:À4ÐW\ñ^’	ñ % Y¿.¹.ÈÈ'ÈÐS[ÐT[ÐS[ÐI[Ñ:\Ñ]�	Ù& yÀjÑQ’	ñ %Ið ××Ø!×)Ñ)¨!¨Q°Ó2ˆIà�m‰m˜IÓ&ˆØ×ÑÑ+Ø×,Ñ,Ñ,ˆCàˆ
r)   c                 ó   • Xl         g r   )r  rž  s     r&   rŸ  Ú,MultimodalTransformer.set_grad_checkpointingK  s   € à"(Õr)   )rå  r8  r  r÷  rú  r=  rÑ  )r,   r-   r.   r/   r   rô   r4   rc   rq   r
   r¸   rr   rD   r…  r9  r   r1   r   r   r   r-  r'   r^   rÖ  rŸ  r2   rP   rQ   s   @r&   r6  r6  â  s/  ø† ð #%Ø"Ø#'Ø)+¯©Ø*3Ø!Ø $ñ)Làð)Lð ð)Lð ð	)Lð
  ð)Lð ð)Lð !ð)Lð ˜BŸI™I‘ð)Lð ˜RŸY™Y™ð)Lð ð)Lð ÷)Lð )LòVVò$ð 15Ø7;Ø$ñUà�|‰|ðUð   §¡Ñ-ðUð ˜e C¨¨c© NÑ3Ñ4ð	Uð
 õUòð4 ‡Y�Y×Ñó)ó ö)r)   r6  Úmodelr
  c                 óÊ  ^• [        U S5      (       a  U R                  nOU n0 n[        USS5      US'   [        USS5      US'   [        USS5      US'   [        USS5      US'   [        USS5      US'   [        USS5      US'   UR                  5        VVs0 s H  u  pEUc  M
  XE_M     nnnUR	                  5        Hj  n[        U[        R                  5      (       a	  S	Ul        M+  [        U[        R                  5      (       d  ML  UR                  5        H
  nS	Ul        M     Ml     US
:X  a  gUS   nU(       a  [        US5      (       d  g[        UR                  5      n	U	S
:X  a  g/ n
/ nS H  nXÃ;   d  M
  UR                  X<   5        M     U(       a  U
R                  U5        U	S:”  a(  UR                  SS  H  nU
R                  U/5        M     UR                  S   /nSU;   a  UR                  US   5        U
R                  U5        SU;   a  U
R                  US   /5        U4S jm[        U[        U
5      5      nX¯* S  H  nT" U5        M     gs  snnf )aJ  
Lock text tower layers for CLIP models.

Works with both model architectures:
- CustomTextCLIP where text components are in self.text
- Standard CLIP where text components are unpacked as attributes

Args:
    model: The CLIP model or TextTransformer module
    unlocked_layers: Number of layers to leave unlocked (from the end)
r×  rô  Nry  rö  r  r÷  rú  Fr   r  )rô  ry  rö  r   r[   c                 ó"  >• [        U [        5      (       a  U  H  nT" U5        M     g [        U [        R                  5      (       a  SU l        g [        U [        R
                  5      (       a!  U R                  5        H  u  p#SUl        M     g g r�  )r  r   r   rF   rp  r¸   Únamed_parameters)ÚmoduleÚmÚnamer•  r“  s       €r&   r“  Ú lock_text_tower.<locals>._unlock¢  sm   ø€ Ü�fœh×'Ñ'Û�Ù˜–
ò ä˜¤§¡×-Ñ-Ø#'ˆFÕ Ü˜¤§	¡	×*Ñ*Ø%×6Ñ6Ö8‘�Ø&*�Ö#ò  9ð +r)   )rˆ   r×  r-  ÚitemsÚvaluesr  r   rF   rp  r¸   r‘  r  r  r%  Úmin)rI  r
  Útext_moduleÚtext_paramsr²   r³   rM  r•  r  Útotal_layersr–  Úembedding_groupÚkeyr  Ú
last_blockÚnum_groups_to_unlockÚgroupr“  s                    @r&   r  r  P  so  ø€ ô  ˆu�f×Ñà—j‘j‰ð ˆð €KÜ%,¨[Ð:KÈTÓ%R€KÐ!Ñ"Ü*1°+Ð?UÐW[Ó*\€KÐ&Ñ'Ü$ [°)¸TÓB€K�	ÑÜ!(¨°mÀTÓ!J€K�ÑÜ% k°:¸tÓD€K�
ÑÜ%,¨[Ð:KÈTÓ%R€KÐ!Ñ"ð %0×$5Ñ$5Ô$7ÔIÒ$7™D˜A¸1“4�1’4Ñ$7€KÑIð ×$Ñ$Ö&ˆÜ�fœbŸl™l×+Ñ+Ø#(ˆFÖ Ü˜¤§	¡	×*Ó*Ø×*Ñ*Ö,�Ø&+�Ö#ó -ñ	 'ð ˜!ÓØð ˜mÑ,€KÞœg k°;×?Ñ?Øä�{×,Ñ,Ó-€LØ�qÓØð €Fð €OÛEˆØÕØ×"Ñ" ;Ñ#3Ö4ñ Fö Ø�‰�oÔ&ð �aÓØ ×*Ñ*¨3¨BÓ/ˆEØ�M‰M˜5˜'Ö"ñ 0ð ×'Ñ'¨Ñ+Ð,€JØ�[Ó Ø×Ñ˜+ jÑ1Ô2Ø
‡M�M�*Ôð ˜KÓ'Ø�‰�{Ð#4Ñ5Ð6Ô7õ+ô ˜´°F³Ó<ÐØÐ-Ð.Ó/ˆÙ�Žò 0ùó{ Js   Â	IÂ$I)NrÜ  N)r   )-Úcollectionsr   r·   Útypingr   r   r   r   r   r	   r
   r   r   r   Útorch.nnr   r   Útorch.utils.checkpointr   Úutilsr   r   Ú	pos_embedr   r4   r   r¸   r9   r@   rS   rt   rº   rÏ   rö   r  r@  rc   rV  rX  r1   r  rà  râ  r6  r  r*   r)   r&   Ú<module>rb     s™  ðÝ #Û ß O× OÓ Oã Ý Ý $Ý -ç 2Ý .ô�B—L‘Lô ô�—‘ô ô,�—	‘	ô ,ôF�—‘ô Fô)�2—9‘9ô )ôXx�—	‘	ô xôv˜Ÿ	™	ô ô.>˜RŸY™Yô >ôB3 2§9¡9ô 3ôla˜Ÿ	™	ô aôHw�"—)‘)ô wðt; Sô ;ôO˜Ÿ	™	ô Oðh
 (,Ø!Ø&*ñ	Ø�<‰<ðà�u—|‘|Ñ$ðð ðð ˜s‘mð	ð
 ‡\�\õô4l�b—i‘iô lô^	k)˜Kô k)ð` ñ_Ø�9‰9ð_àö_r)   