ó
    ÷“,jW#  ã                   óÖ   • S SK Jr  S SKJrJrJrJr  S SKrS SKJr  S SK	J
r  SSKJrJr   " S S	\R                  5      r " S
 S\R                  5      r " S S\R                  5      rg)é    )ÚOrderedDict)ÚDictÚListÚOptionalÚUnionN)Únn)Ú
functionalé   )Úfreeze_batch_norm_2dÚfeature_take_indicesc                   óR   ^ • \ rS rSrSrSU 4S jjrS\R                  4S jrSr	U =r
$ )Ú
Bottlenecké   é   c                 óö  >• [         TU ]  5         [        R                  " XSSS9U l        [        R
                  " U5      U l        [        R                  " SS9U l        [        R                  " X"SSSS9U l	        [        R
                  " U5      U l
        [        R                  " SS9U l        US:”  a  [        R                  " U5      O[        R                  " 5       U l        [        R                  " X"U R                  -  SSS9U l        [        R
                  " X R                  -  5      U l        [        R                  " SS9U l        S U l        X0l        US:”  d  X[*        R                  -  :w  a„  [        R,                  " [/        S[        R                  " U5      4S	[        R                  " XU R                  -  SSSS
94S[        R
                  " X R                  -  5      4/5      5      U l        g g )Nr
   F)ÚbiasT©Úinplaceé   )Úpaddingr   z-1Ú0)Ústrider   Ú1)ÚsuperÚ__init__r   ÚConv2dÚconv1ÚBatchNorm2dÚbn1ÚReLUÚact1Úconv2Úbn2Úact2Ú	AvgPool2dÚIdentityÚavgpoolÚ	expansionÚconv3Úbn3Úact3Ú
downsampler   r   Ú
Sequentialr   )ÚselfÚinplanesÚplanesr   Ú	__class__s       €Ú§C:\Users\johnn\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.13_qbz5n2kfra8p0\LocalCache\local-packages\Python313\site-packages\open_clip/modified_resnet.pyr   ÚBottleneck.__init__   sk  ø€ Ü‰ÑÔô —Y’Y˜x°¸Ñ?ˆŒ
Ü—>’> &Ó)ˆŒÜ—G’G DÑ)ˆŒ	ä—Y’Y˜v¨q¸!À%ÑHˆŒ
Ü—>’> &Ó)ˆŒÜ—G’G DÑ)ˆŒ	à/5¸«z”r—|’| FÔ+¼r¿{º{»}ˆŒä—Y’Y˜v°·±Ñ'>ÀÈÑNˆŒ
Ü—>’> &¯>©>Ñ"9Ó:ˆŒÜ—G’G DÑ)ˆŒ	àˆŒØŒà�A‹:˜¬j×.BÑ.BÑ%BÓBä Ÿmšm¬KØ”r—|’| FÓ+Ð,Ø”b—i’i °4·>±>Ñ*AÀ1ÈQÐUZÑ[Ð\Ø”b—n’n V¯n©nÑ%<Ó=Ð>ð9ó -ó ˆD�Oð Có    Úxc                 óŽ  • UnU R                  U R                  U R                  U5      5      5      nU R                  U R	                  U R                  U5      5      5      nU R                  U5      nU R                  U R                  U5      5      nU R                  b  U R                  U5      nX2-  nU R                  U5      nU$ ©N)r!   r   r   r$   r#   r"   r'   r*   r)   r,   r+   )r.   r5   ÚidentityÚouts       r2   ÚforwardÚBottleneck.forward+   sš   € Øˆà�i‰i˜Ÿ™ §¡¨A£Ó/Ó0ˆØ�i‰i˜Ÿ™ §¡¨C£Ó1Ó2ˆØ�l‰l˜3ÓˆØ�h‰h�t—z‘z #“Ó'ˆà�?‰?Ñ&Ø—‘ qÓ)ˆHà‰ˆØ�i‰i˜‹nˆØˆ
r4   )r!   r$   r+   r'   r   r#   r*   r   r"   r)   r,   r   ©r
   )Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r(   r   ÚtorchÚTensorr:   Ú__static_attributes__Ú__classcell__©r1   s   @r2   r   r      s"   ø† Ø€I÷ð:˜Ÿ™÷ ò r4   r   c            	       óF   ^ • \ rS rSrS	S\S\S\S\4U 4S jjjrS rSrU =r$ )
ÚAttentionPool2dé;   Úspacial_dimÚ	embed_dimÚ	num_headsÚ
output_dimc                 ó�  >• [         TU ]  5         [        R                  " [        R
                  " US-  S-   U5      US-  -  5      U l        [        R                  " X"5      U l        [        R                  " X"5      U l	        [        R                  " X"5      U l
        [        R                  " X$=(       d    U5      U l        X0l        g )Né   r
   g      à?)r   r   r   Ú	ParameterrA   ÚrandnÚpositional_embeddingÚLinearÚk_projÚq_projÚv_projÚc_projrK   )r.   rI   rJ   rK   rL   r1   s        €r2   r   ÚAttentionPool2d.__init__<   sŒ   ø€ Ü‰ÑÔÜ$&§L¢L´·²¸[ÈAÑ=MÐPQÑ=QÐS\Ó1]Ð`iÐmpÑ`pÑ1pÓ$qˆÔ!Ü—i’i 	Ó5ˆŒÜ—i’i 	Ó5ˆŒÜ—i’i 	Ó5ˆŒÜ—i’i 	×+B¸ÓCˆŒØ"�r4   c           
      ó¼  • UR                  UR                  S   UR                  S   UR                  S   UR                  S   -  5      R                  SSS5      n[        R                  " UR                  SSS9U/SS9nXR                  S S 2S S S 24   R                  UR                  5      -   n[        R                  " S0 SU_S	U_S
U_SUR                  S   _SU R                  _SU R                  R                  _SU R                  R                  _SU R                  R                  _SS _S[        R                  " U R                  R                   U R                  R                   U R                  R                   /5      _SS _SS _SS_SS_SU R"                  R                  _SU R"                  R                   _SS_SU R$                  _SS_6u  pUS   $ )Nr   r
   rN   r   T)ÚdimÚkeepdim)rY   ÚqueryÚkeyÚvalueÚembed_dim_to_checkéÿÿÿÿrK   Úq_proj_weightÚk_proj_weightÚv_proj_weightÚin_proj_weightÚin_proj_biasÚbias_kÚbias_vÚadd_zero_attnFÚ	dropout_pg        Úout_proj_weightÚout_proj_biasÚuse_separate_proj_weightÚtrainingÚneed_weights© )ÚreshapeÚshapeÚpermuterA   ÚcatÚmeanrQ   ÚtoÚdtypeÚFÚmulti_head_attention_forwardrK   rT   ÚweightrS   rU   r   rV   rl   )r.   r5   Ú_s      r2   r:   ÚAttentionPool2d.forwardE   s×  € Ø�I‰I�a—g‘g˜a‘j !§'¡'¨!¡*¨a¯g©g°a©j¸1¿7¹7À1¹:Ñ.EÓF×NÑNÈqÐRSÐUVÓWˆÜ�IŠI�q—v‘v !¨T�vÐ2°AÐ6¸AÑ>ˆØ×)Ñ)ª!¨T²1¨*Ñ5×8Ñ8¸¿¹ÓAÑAˆÜ×-Ò-ò 
Ùð
Ùð
Ù"#ð
à Ÿw™w rš{ð
ð —n’nð
ð Ÿ+™+×,Ò,ð	
ð
 Ÿ+™+×,Ò,ð
ð Ÿ+™+×,Ò,ð
ñ  ð
ô Ÿš D§K¡K×$4Ñ$4°d·k±k×6FÑ6FÈÏÉ×HXÑHXÐ#YÔZð
ñ ð
ñ ð
ñ  ð
ñ ð
ð !ŸK™K×.Ò.ð
ð Ÿ+™+×*Ò*ð
ñ &*ð
ð  —]’]ð!
ñ" ð#
‰ˆð( �‰tˆr4   )rV   rS   rK   rQ   rT   rU   r7   )	r=   r>   r?   r@   Úintr   r:   rC   rD   rE   s   @r2   rG   rG   ;   s4   ø† ñ# Cð #°Cð #ÀCð #ÐUX÷ #ð #÷ð r4   rG   c                   óZ  ^ • \ rS rSrSr  SS\\   S\S\S\S\4
U 4S jjjrSS	 jrS
 r	SS jr
\R                  R                  SS j5       rS r      SS\R                   S\\\\\   4      S\S\S\S\S\S\\\\R                   \\R                      4   4   4S jjrS rSrU =r$ )ÚModifiedResNeté`   al  
A ResNet class that is similar to torchvision's but contains the following changes:
- There are now 3 "stem" convolutions as opposed to 1, with an average pool instead of a max pool.
- Performs antialiasing strided convolutions, where an avgpool is prepended to convolutions with stride > 1
- The final pooling layer is a QKV attention instead of an average pool
ÚlayersrL   ÚheadsÚ
image_sizeÚwidthc           	      ó¶  >• [         TU ]  5         X l        X@l        [        R
                  " SUS-  SSSSS9U l        [        R                  " US-  5      U l        [        R                  " SS9U l
        [        R
                  " US-  US-  SSSS9U l        [        R                  " US-  5      U l        [        R                  " SS9U l        [        R
                  " US-  USSSS9U l        [        R                  " U5      U l        [        R                  " SS9U l        [        R"                  " S5      U l        XPl        U R)                  XQS	   5      U l        U R)                  US-  US   SS
9U l        U R)                  US-  US   SS
9U l        U R)                  US-  US   SS
9U l        US-  n[3        US-  XcU5      U l        U R7                  5         g )Nr   rN   r
   F)Úkernel_sizer   r   r   Tr   )r„   r   r   r   )r   r   é   é    )r   r   rL   r�   r   r   r   r   r   r    r!   r"   r#   r$   r)   r*   r+   r%   r'   Ú	_inplanesÚ_make_layerÚlayer1Úlayer2Úlayer3Úlayer4rG   ÚattnpoolÚinit_parameters)r.   r   rL   r€   r�   r‚   rJ   r1   s          €r2   r   ÚModifiedResNet.__init__h   sš  ø€ ô 	‰ÑÔØ$ŒØ$Œô —Y’Y˜q %¨1¡*¸!ÀAÈqÐW\Ñ]ˆŒ
Ü—>’> %¨1¡*Ó-ˆŒÜ—G’G DÑ)ˆŒ	Ü—Y’Y˜u¨™z¨5°A©:À1ÈaÐV[Ñ\ˆŒ
Ü—>’> %¨1¡*Ó-ˆŒÜ—G’G DÑ)ˆŒ	Ü—Y’Y˜u¨™z¨5¸aÈÐQVÑWˆŒ
Ü—>’> %Ó(ˆŒÜ—G’G DÑ)ˆŒ	Ü—|’| A“ˆŒð ŒØ×&Ñ& u°Q©iÓ8ˆŒØ×&Ñ& u¨q¡y°&¸±)ÀAÐ&ÐFˆŒØ×&Ñ& u¨q¡y°&¸±)ÀAÐ&ÐFˆŒØ×&Ñ& u¨q¡y°&¸±)ÀAÐ&ÐFˆŒà˜B‘Jˆ	Ü'¨
°bÑ(8¸)ÈJÓWˆŒà×ÑÕr4   c                 óö   • [        U R                  X5      /nU[         R                  -  U l        [        SU5       H(  nUR	                  [        U R                  U5      5        M*     [
        R                  " U6 $ )Nr
   )r   r‡   r(   ÚrangeÚappendr   r-   )r.   r0   Úblocksr   r   ry   s         r2   rˆ   ÚModifiedResNet._make_layerŒ   s`   € Ü˜TŸ^™^¨VÓ<Ð=ˆà¤*×"6Ñ"6Ñ6ˆŒÜ�q˜&Ö!ˆAØ�M‰Mœ* T§^¡^°VÓ<Ö=ñ "ô �}Š}˜fÐ%Ð%r4   c                 óP  • U R                   Gb  U R                   R                  R                  S-  n[        R                  R                  U R                   R                  R                  US9  [        R                  R                  U R                   R                  R                  US9  [        R                  R                  U R                   R                  R                  US9  [        R                  R                  U R                   R                  R                  US9  U R                  U R                  U R                  U R                  4 HS  nUR                  5        H<  u  p4UR                  S5      (       d  M  [        R                  R!                  U5        M>     MU     g )Ng      à¿)Ústdz
bn3.weight)r�   rV   Úin_featuresr   ÚinitÚnormal_rT   rx   rS   rU   r‰   rŠ   r‹   rŒ   Únamed_parametersÚendswithÚzeros_)r.   r–   Úresnet_blockÚnameÚparams        r2   rŽ   ÚModifiedResNet.init_parameters•   s  € Ø�=‰=Ò$Ø—-‘-×&Ñ&×2Ñ2°dÑ:ˆCÜ�G‰G�O‰O˜DŸM™M×0Ñ0×7Ñ7¸SˆOÑAÜ�G‰G�O‰O˜DŸM™M×0Ñ0×7Ñ7¸SˆOÑAÜ�G‰G�O‰O˜DŸM™M×0Ñ0×7Ñ7¸SˆOÑAÜ�G‰G�O‰O˜DŸM™M×0Ñ0×7Ñ7¸SˆOÑAà!Ÿ[™[¨$¯+©+°t·{±{ÀDÇKÁKÓPˆLØ+×<Ñ<Ö>‘�Ø—=‘= ×.Ó.Ü—G‘G—N‘N 5Ö)ó  ?ò Qr4   c                 ó€   • US:X  d   S5       eU R                  5        H
  nSUl        M     U(       a  [        U 5        g g )Nr   z6partial locking not currently supported for this modelF)Ú
parametersÚrequires_gradr   )r.   Úunlocked_groupsÚfreeze_bn_statsrŸ   s       r2   ÚlockÚModifiedResNet.lock¢   s?   € Ø !Ó#Ð]Ð%]Ó]Ð#Ø—_‘_Ö&ˆEØ"'ˆEÖñ 'æÜ  Õ&ð r4   c                 ó   • g r7   rn   )r.   Úenables     r2   Úset_grad_checkpointingÚ%ModifiedResNet.set_grad_checkpointing©   s   € ð 	r4   c                 óB  • U R                  U R                  U R                  U5      5      5      nU R                  U R	                  U R                  U5      5      5      nU R                  U R                  U R                  U5      5      5      nU R                  U5      nU$ r7   )
r!   r   r   r$   r#   r"   r+   r*   r)   r'   ©r.   r5   s     r2   ÚstemÚModifiedResNet.stem®   sq   € Ø�I‰I�d—h‘h˜tŸz™z¨!›}Ó-Ó.ˆØ�I‰I�d—h‘h˜tŸz™z¨!›}Ó-Ó.ˆØ�I‰I�d—h‘h˜tŸz™z¨!›}Ó-Ó.ˆØ�L‰L˜‹OˆØˆr4   r5   ÚindicesÚ
stop_earlyÚnormalize_intermediatesÚintermediates_onlyÚ
output_fmtÚoutput_extra_tokensÚreturnc                 óÆ  • US;   d   S5       e[        SU5      u  p‰0 n
/ nU R                  U R                  U R                  U R                  U R
                  /n[        R                  R                  5       (       d  U(       d  USU	S-    n[        U5       H%  u  pÞU" U5      nXØ;   d  M  UR                  U5        M'     XºS'   U(       a  U
$ U R                  U5      nXS'   U
$ )aõ  Forward features that returns intermediates.

Args:
    x: Input image tensor
    indices: Take last n blocks if int, all if None, select matching indices if sequence
    stop_early: Stop iterating over blocks when last desired intermediate hit
    normalize_intermediates: Apply final norm layer to all intermediates
    intermediates_only: Only return intermediate features
    output_fmt: Shape of intermediate feature outputs
    output_extra_tokens: Return both extra class, eot tokens
Returns:

)ÚNCHWzOutput format must be == NCHW.é   Nr
   Úimage_intermediatesÚimage_features)r   r®   r‰   rŠ   r‹   rŒ   rA   ÚjitÚis_scriptingÚ	enumerater’   r�   )r.   r5   r°   r±   r²   r³   r´   rµ   Útake_indicesÚ	max_indexÚoutputÚintermediatesr“   ÚiÚblks                  r2   Úforward_intermediatesÚ$ModifiedResNet.forward_intermediatesµ   s×   € ð. ˜YÓ&ÐHÐ(HÓHÐ&ä"6°q¸'Ó"BÑˆàˆØˆØ—)‘)˜TŸ[™[¨$¯+©+°t·{±{ÀDÇKÁKÐPˆÜ�9‰9×!Ñ!×#Ñ#®:Ø˜N˜Y¨™]Ð+ˆFÜ Ö'‰FˆAÙ�A“ˆAØÕ Ø×$Ñ$ QÖ'ñ (ð
 )6Ð$Ñ%æØˆMà�M‰M˜!ÓˆØ#$ÐÑ àˆr4   c                 óÒ   • U R                  U5      nU R                  U5      nU R                  U5      nU R                  U5      nU R	                  U5      nU R                  U5      nU$ r7   )r®   r‰   rŠ   r‹   rŒ   r�   r­   s     r2   r:   ÚModifiedResNet.forwardä   sV   € Ø�I‰I�a‹LˆØ�K‰K˜‹NˆØ�K‰K˜‹NˆØ�K‰K˜‹NˆØ�K‰K˜‹NˆØ�M‰M˜!Óˆàˆr4   )r‡   r!   r$   r+   r�   r'   r   r#   r*   r   r"   r)   r�   r‰   rŠ   r‹   rŒ   rL   )éà   é@   r<   )r   F)T)NFFFr¸   F)r=   r>   r?   r@   Ú__doc__r   r{   r   rˆ   rŽ   r¦   rA   r¼   Úignorerª   r®   rB   r   r   ÚboolÚstrr   rÅ   r:   rC   rD   rE   s   @r2   r}   r}   `   s*  ø† ñð "Øñ"à˜‘Ið"ð ð"ð ð	"ð
 ð"ð ÷"ð "ôH&ò*ô'ð ‡Y�Y×Ñóó ðòð 8<Ø$Ø,1Ø',Ø$Ø(-ñ-à�|‰|ð-ð ˜e C¨¨c© NÑ3Ñ4ð-ð ð	-ð
 &*ð-ð !%ð-ð ð-ð "&ð-ð 
ˆc�5˜Ÿ™ t¨E¯L©LÑ'9Ð9Ñ:Ð:Ñ	;õ-÷^ð r4   r}   )Úcollectionsr   Útypingr   r   r   r   rA   r   Útorch.nnr	   rv   Úutilsr   r   ÚModuler   rG   r}   rn   r4   r2   Ú<module>rÔ      sN   ðÝ #ß .Ó .ã Ý Ý $ç =ô-�—‘ô -ô`"�b—i‘iô "ôJL�R—Y‘Yõ Lr4   