ó
    ÷“,j0  ã                   óÎ  • S r SSKrSSKrSSKJr  SSKJr   SSKrSSKJrJrJ	r	J
r
  SSKJrJrJr  S
SKJr  S r0 rS r\ " S S\R,                  5      5       r\ " S S\R,                  5      5       r\ " S S\R,                  5      5       r\ " S S\R,                  5      5       r " S S\R,                  5      rg! \ a   rSr " S S5      r " S S	5      r
 SrCN­SrCff = f)z–huggingface model adapter

Wraps HuggingFace transformers (https://github.com/huggingface/transformers) models for use as a text tower in CLIP model.
é    N)Ú
TensorType)Ú	AutoModelÚAutoTokenizerÚ
AutoConfigÚPretrainedConfig)ÚBaseModelOutputÚBaseModelOutputWithPoolingÚ,BaseModelOutputWithPoolingAndCrossAttentionsc                   ó   • \ rS rSrSrg)r   é   © N©Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__static_attributes__r   ó    Ú C:\Users\johnn\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.13_qbz5n2kfra8p0\LocalCache\local-packages\Python313\site-packages\open_clip/hf_model.pyr   r      ó   † Úr   r   c                   ó   • \ rS rSrSrg)r   é   r   Nr   r   r   r   r   r      r   r   r   é   )Ú	arch_dictc                 óN   • [         R                  " SSU 5      R                  5       $ )Nz(?<!^)(?=[A-Z])Ú_)ÚreÚsubÚlower)Úss    r   Ú_camel2snaker!      s   € Ü�6Š6Ð$ c¨1Ó-×3Ñ3Ó5Ð5r   c                 ó>   • U [         [        U R                  5      '   U $ )z"Decorator registering pooler class)Ú_POOLERSr!   r   )Úclss    r   Úregister_poolerr%   '   s   € à+.„HŒ\˜#Ÿ,™,Ó'Ñ(Ø€Jr   c                   ó*   • \ rS rSrSrS\S\4S jrSrg)Ú
MeanPooleré-   zMean poolingÚxÚattention_maskc                 ó~   • UR                   UR                  S5      -  nUR                  SS9UR                  SSS9-  $ )Néÿÿÿÿr   )ÚdimT)Úkeepdim)Úlast_hidden_stateÚ	unsqueezeÚsum©Úselfr)   r*   Úmasked_outputs       r   ÚforwardÚMeanPooler.forward1   sF   € Ø×+Ñ+¨n×.FÑ.FÀrÓ.JÑJˆØ× Ñ  QÐ Ð'¨.×*<Ñ*<¸RÈÐ*<Ð*NÑNÐNr   r   N©	r   r   r   r   Ú__doc__r   r   r5   r   r   r   r   r'   r'   -   s   † áðO˜ð O¸*÷ Or   r'   c                   ó*   • \ rS rSrSrS\S\4S jrSrg)Ú	MaxPooleré6   zMax poolingr)   r*   c                 ó¬   • UR                   R                  UR                  S5      [        R                  * 5      nUR                  S5      R                  $ )Nr,   r   )r/   Úmasked_fillr0   ÚtorchÚinfÚmaxÚvaluesr2   s       r   r5   ÚMaxPooler.forward:   sG   € Ø×+Ñ+×7Ñ7¸×8PÑ8PÐQSÓ8TÔW\×W`ÑW`ÐV`ÓaˆØ× Ñ  Ó#×*Ñ*Ð*r   r   Nr7   r   r   r   r:   r:   6   s   † áð+˜ð +¸*÷ +r   r:   c                   óB   ^ • \ rS rSrSrSU 4S jjrS\S\4S jrSr	U =r
$ )	Ú	ClsPooleré?   zCLS token poolingc                 ó<   >• [         TU ]  5         SU l        Xl        g ©Nr   )ÚsuperÚ__init__Úcls_token_positionÚuse_pooler_output)r3   rK   Ú	__class__s     €r   rI   ÚClsPooler.__init__C   s   ø€ Ü‰ÑÔØ"#ˆÔØ!2Õr   r)   r*   c                 óÌ   • U R                   (       a4  [        U[        [        45      (       a  UR                  b  UR                  $ UR
                  S S 2U R                  S S 24   $ ©N)rK   Ú
isinstancer	   r
   Úpooler_outputr/   rJ   ©r3   r)   r*   s      r   r5   ÚClsPooler.forwardH   sQ   € Ø×"×"Ü�qÔ5Ô7cÐd×eÑeØ�_‰_Ñ(à—?‘?Ð"à×"Ñ"¢1 d×&=Ñ&=ºqÐ#@ÑAÐAr   )rJ   rK   ©T©r   r   r   r   r8   rI   r   r   r5   r   Ú__classcell__©rL   s   @r   rD   rD   ?   s'   ø† á÷3ð
B˜ð B¸*÷ Bò Br   rD   c                   ó>   ^ • \ rS rSrSrU 4S jrS\S\4S jrSr	U =r
$ )ÚClsLastHiddenStatePooleréR   z[CLS token pooling
NOTE: this is equivalent to ClsPooler above with use_pooler_output=False
c                 ó0   >• [         TU ]  5         SU l        g rG   )rH   rI   rJ   )r3   rL   s    €r   rI   Ú!ClsLastHiddenStatePooler.__init__X   s   ø€ Ü‰ÑÔØ"#ˆÕr   r)   r*   c                 óB   • UR                   S S 2U R                  S S 24   $ rO   )r/   rJ   rR   s      r   r5   Ú ClsLastHiddenStatePooler.forward\   s    € Ø×"Ñ"¢1 d×&=Ñ&=ºqÐ#@ÑAÐAr   )rJ   rU   rW   s   @r   rY   rY   R   s)   ø† ñõ$ðB˜ð B¸*÷ Bò Br   rY   c                   óú   ^ • \ rS rSr% Sr\R                  R                  \   \	S'        SS\
S\S\S\
S\
S	\S\4U 4S
 jjjrS\4S jrSS\S\4S jjr\R                  R"                  SS j5       rS rSrU =r$ )ÚHFTextEncoderé`   zHuggingFace model adapterÚoutput_tokensÚmodel_name_or_pathÚ
output_dimÚconfigÚpooler_typeÚ	proj_typeÚ
pretrainedc           
      óÄ  >• [         TU ]  5         Xpl        X l        US:H  n[        c  [        S5      eUc½  [        R                  " U5      U l        U(       a  [        R                  U4O[        R                  U R                  4u  pš[        U R                  S5      (       aD  U R                  R                  (       a)  U	" U
5      U l        U R                  R                  U l        O-U	" X¨S9U l        O!X0l        [        R                  " U5      U l        Uc   [        U R                  R                      S   n[#        U R                  SS5      U l        [#        U R                  SS5      U l        [(        U   " 5       U l        [#        U R                  [        U R                  R                      S	   S
   5      nX²:X  a  Uc  [,        R.                  " 5       U l        g US:X  a  [,        R2                  " X²SS9U l        g US:X  a^  X²-   S-  n[,        R4                  " [,        R2                  " X¼SS9[,        R6                  " 5       [,        R2                  " XÂSS95      U l        g g )NÚ
cls_poolerzGPlease `pip install transformers` to use pre-trained HuggingFace modelsÚis_encoder_decoder)Úadd_pooling_layerÚpoolerÚ
vocab_sizer   Úmax_position_embeddingsÚconfig_namesÚwidthÚlinearF)ÚbiasÚmlpé   )rH   rI   rb   rd   ÚtransformersÚRuntimeErrorr   Úfrom_pretrainedre   r   Úfrom_configÚhasattrrk   ÚtransformerÚencoderr   Ú
model_typeÚgetattrrn   Úcontext_lengthr#   rm   ÚnnÚIdentityÚprojÚLinearÚ
SequentialÚGELU)r3   rc   rd   re   rf   rg   rh   rb   Úuses_transformer_poolerÚcreate_funcÚ
model_argsÚd_modelÚhidden_sizerL   s                €r   rI   ÚHFTextEncoder.__init__d   sè  ø€ ô 	‰ÑÔØ*ÔØ$Œð $/°,Ñ#>ÐäÑÜÐhÓiÐiØ‰>Ü$×4Ò4Ð5GÓHˆDŒKÞYc¤y×'@Ñ'@ÐBTÑ&UÜ×%Ñ% t§{¡{ðj4Ñ#ˆKô �t—{‘{Ð$8×9Ñ9¸d¿k¹k×>\×>\Ù#.¨zÓ#:�Ô Ø#'×#3Ñ#3×#;Ñ#;�Õ á#.¨zÑ#e�Õ à ŒKÜ(×4Ò4°VÓ<ˆDÔØÑÜ$ T§[¡[×%;Ñ%;Ñ<¸XÑFˆKô " $§+¡+¨|¸QÓ?ˆŒÜ% d§k¡kÐ3LÈaÓPˆÔä˜{Ò+Ó-ˆŒä˜$Ÿ+™+¤y°·±×1GÑ1GÑ'HÈÑ'XÐY`Ñ'aÓbˆØÓ!¨	Ñ(9ÜŸš›ˆD�IØ˜(Ó"ÜŸ	š	 '¸EÑBˆD�IØ˜%ÓØ"Ñ/°AÑ5ˆKÜŸšÜ—	’	˜'°UÑ;Ü—’“	Ü—	’	˜+¸Ñ>óˆD�Ið  r   r)   c                 óâ  • XR                   R                  :g  R                  5       nU R                  XS9nU R	                  X25      nU R                  U5      nUR                  R                  S   n[        U R                  5      [        :X  aA  UR                  S S 2[        R                  " U5      U R                  R                  :g  S S 24   OUR                  nU R                  (       a  XW4$ U$ )N)Ú	input_idsr*   r   )re   Úpad_token_idÚlongr{   rm   r‚   r/   ÚshapeÚtyperD   r>   ÚarangerJ   rb   )r3   r)   Ú	attn_maskÚoutÚ
pooled_outÚ	projectedÚseq_lenÚtokenss           r   r5   ÚHFTextEncoder.forwardš   sÍ   € ØŸ+™+×2Ñ2Ñ2×8Ñ8Ó:ˆ	Ø×Ñ¨ÐÐEˆØ—[‘[ Ó0ˆ
Ø—I‘I˜jÓ)ˆ	à×'Ñ'×-Ñ-¨aÑ0ˆô �D—K‘KÓ ¤IÓ-ð ×!Ñ!¢!¤U§\¢\°'Ó%:¸d¿k¹k×>\Ñ>\Ñ%\Ò^_Ð"_Ò`à×&Ñ&ð 	ð ××ØÐ$Ð$ØÐr   Úunlocked_layersÚfreeze_layer_normc                 óº  • U(       dG  U R                   R                  5        H(  u  p4SUR                  S5      ;   a  U(       + OSUl        M*     g [	        U R                   S5      (       a  U R                   R
                  OU R                   n[        U[        U R                  R                     S   S   5      n[        SU S[        U5      S	-    S
35        [        U R                   [        U R                  R                     S   S   5      nU/UQS U*  nU H?  n	U	R                  5        H(  u  p4SUR                  S5      ;   a  U(       + OSUl        M*     MA     g )NÚ	LayerNormÚ.Fr|   rp   Ú
layer_attrz
Unlocking Ú/r   z layers of hf modelÚtoken_embeddings_attr)r{   Únamed_parametersÚsplitÚrequires_gradrz   r|   r~   r   re   r}   ÚprintÚlen)
r3   rš   r›   ÚnÚpr|   Ú
layer_listÚ
embeddingsÚmodulesÚmodules
             r   ÚlockÚHFTextEncoder.lock«   s;  € ÞØ×(Ñ(×9Ñ9Ö;‘�Ø=HÈAÏGÉGÐTWËLÓ=XÐ'8Õ#8Ð^c�–ñ <àä.5°d×6FÑ6FÈ	×.RÑ.R�$×"Ñ"×*Ò*ÐX\×XhÑXhˆÜ˜W¤i°·±×0FÑ0FÑ&GÈÑ&WÐXdÑ&eÓfˆ
Ü�
˜?Ð+¨1¬S°«_¸qÑ-@Ð,AÐATÐUÔVÜØ×Ñœi¨¯©×(>Ñ(>Ñ?ÀÑOÐPgÑhójˆ
àÐ+ 
Ð+Ð,=¨oÐ-=Ð>ˆãˆFØ×/Ñ/Ö1‘�Ø=HÈAÏGÉGÐTWËLÓ=XÐ'8Õ#8Ð^c�–ó 2ò r   c                 ó8   • U R                   R                  5         g rO   )r{   Úgradient_checkpointing_enable)r3   Úenables     r   Úset_grad_checkpointingÚ$HFTextEncoder.set_grad_checkpointing¼   s   € à×Ñ×6Ñ6Õ8r   c                 ó   • g rO   r   )r3   s    r   Úinit_parametersÚHFTextEncoder.init_parametersÀ   s   € Ør   )re   r   rd   rb   rm   r‚   r{   rn   )NNNTF)r   TrT   )r   r   r   r   r8   r>   ÚjitÚFinalÚboolÚ__annotations__ÚstrÚintr   rI   r   r5   r­   Úignorer²   rµ   r   rV   rW   s   @r   r`   r`   `   sÃ   ø‡ Ù#Ø—9‘9—?‘? 4Ñ(Ó(ð (,Ø#Ø!Ø#Ø"'ñ4à #ð4ð ð4ð %ð	4ð
 ð4ð ð4ð ð4ð  ÷4ð 4ðl˜ô ñ"d Cð dÀõ dð" ‡Y�Y×Ñó9ó ð9÷ð r   r`   )r8   r   r>   Útorch.nnr€   r   rv   r   r   r   r   Útransformers.modeling_outputsr   r	   r
   ÚImportErrorÚeÚ
hf_configsr   r!   r#   r%   ÚModuler'   r:   rD   rY   r`   r   r   r   Ú<module>rÄ      s	  ðñó 
ã Ý Ý ðÛßSÓS÷5ñ 5õ "ò6ð
 €òð ôO�—‘ó Oó ðOð ô+�—	‘	ó +ó ð+ð ôB�—	‘	ó Bó ðBð$ ô
B˜rŸy™yó 
Bó ð
Bôa�B—I‘Iõ aøða ó 	Ø€L÷ñ ÷ö ûð	ús   ˜B> Â>C$ÃCÃC$