ó
    ÷“,jŸ;  ã                   ó   • S SK r S SKrS SKrS SKrS SKrS SKrS SKrS SKJ	s  J
r  S SKJr   S SKrS SKJrJrJr  S SKJr  S SKJr  S SKJr   " S S\5      rS	 rS
 rS rSS jrSS jr S r!S r"g! \ a    Sr NKf = f)é    N)ÚDistributedDataParallel)Úget_input_dtypeÚCLIPÚCustomTextCLIP)Ú	is_master)Úzero_shot_eval)Úget_autocastc                   ó.   • \ rS rSrSrS rS rSS jrSrg)	ÚAverageMeteré   z1Computes and stores the average and current valuec                 ó$   • U R                  5         g ©N)Úreset©Úselfs    Ú£C:\Users\johnn\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.13_qbz5n2kfra8p0\LocalCache\local-packages\Python313\site-packages\open_clip_train/train.pyÚ__init__ÚAverageMeter.__init__   s   € Ø�
‰
�ó    c                 ó<   • SU l         SU l        SU l        SU l        g )Nr   )ÚvalÚavgÚsumÚcountr   s    r   r   ÚAverageMeter.reset   s   € ØˆŒØˆŒØˆŒØˆ�
r   c                 ó¤   • Xl         U =R                  X-  -  sl        U =R                  U-  sl        U R                  U R                  -  U l        g r   )r   r   r   r   )r   r   Úns      r   ÚupdateÚAverageMeter.update#   s8   € ØŒØ�Š�C‘GÑ�Ø�
Š
�a‰�
Ø—8‘8˜dŸj™jÑ(ˆ�r   )r   r   r   r   N)é   )	Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__r   r   r   Ú__static_attributes__© r   r   r   r      s   † Ù;òò÷)r   r   c                 ó    • U S   U S   U S   S.$ )Nr   r    é   ©Úimage_featuresÚtext_featuresÚlogit_scaler'   )Ú	model_outs    r   Úpostprocess_clip_outputr/   *   s!   € à# A™,Ø" 1™Ø  ‘|ñð r   c                 ó@   • [        U S5      (       a  U R                  $ U $ )NÚmodule)Úhasattrr1   )Úmodels    r   Úunwrap_modelr4   2   s   € Üˆu�h×ÑØ�|‰|Ðàˆr   c                 ój   • Ub   UR                  U 5      R                  5         g U R                  5         g r   )ÚscaleÚbackward)Ú
total_lossÚscalers     r   r7   r7   9   s)   € ØÑØ�‰�ZÓ ×)Ñ)Õ+à×ÑÕr   c
                 óÊ  • [         R                  " UR                  5      n
[        UR                  U
R                  S9n[        UR                  5      nU R                  5         UR                  (       a  UR                  5         US   R                  U5        US   R                  nUR                  UR                  -  n[        R                  " [        R                  " UR                   S-   S5      5      nUR                  S:”  a  / / 0 nnn0 n[#        5       n[#        5       n[$        R$                  " 5       n['        U5       GH„  u  nnUUR                  -  nXã-  U-   nUR(                  (       d  U" U5        Uu  nnUR+                  X¬SS9nUR+                  U
SS9nUR-                  [$        R$                  " 5       U-
  5        UR/                  5         UR                  S:X  aÇ  U" 5          U " UU5      nUS   nUR                  (       a`  [         R0                  " 5          U" UU5      nS S S 5        UR-                  WR3                  5        V V!s0 s H  u  n n!S	U  3U!_M     sn!n 5        U" S,0 UDS
S0D6n"[5        U"R7                  5       5      n#U#U"S'   S S S 5        [9        W#U5        GOÑ[         R0                  " 5          U" 5          U " UU5      nS H  n$UR;                  U$S 5        M     UR3                  5        H(  u  n%n&U%W;   a  UU%   R=                  U&5        M"  U&/UU%'   M*     S S S 5        WR=                  U5        WR=                  U5        S S S 5        US-   UR                  -  S:”  a  GM2  UR/                  5         [?        UR                  5       Hß  n'WU'   nWU'   nU" 5          U " UU5      n0 n(UR;                  S5      =U(S'   nSU;   a  UR;                  S5      U(S'   0 n)WR3                  5        H7  u  n%n&UU%   n*[         R@                  " U*S U' UU%   /-   U*U'S-   S  -   5      U)U%'   M9     U" S,0 U)DU(DS
S0D6n"A)A([5        U"R7                  5       5      n#U#U"S'   S S S 5        [9        W#U5        Má     UGb,  URB                  (       aš  URE                  5         URG                  U5        URH                  bA  [         RJ                  RL                  RO                  U RQ                  5       URH                  SS9  URS                  5          URU                  U5        S S S 5        OpURH                  bR  URG                  U5        [         RJ                  RL                  RO                  U RQ                  5       URH                  SS9  URU                  U5        UR-                  5         O^URH                  bA  [         RJ                  RL                  RO                  U RQ                  5       URH                  SS9  URU                  5         UR                  S:”  a  / / 0 nnn[         R0                  " 5          [W        U 5      RX                  R[                  S[        R                  " S5      5        S S S 5        UR-                  [$        R$                  " 5       U-
  5        [$        R$                  " 5       nUS-   n+[]        U5      (       d  GM‰  UUR^                  -  S:X  d	  U+U:X  d  GM¥  [a        U5      n,U+U,-  UR                  -  URb                  -  n-UR                   n.SU+-  U-  n/W"R3                  5        H<  u  n%n&U%U;  a  [#        5       UU%'   UU%   R-                  U&Re                  5       U,5        M>     WRe                  5       n0SRg                  UR3                  5        V1V2s/ s H5  u  n1n2U1Ri                  5        SU2Rj                  S SU2Rl                  S S3PM7     sn2n15      n3UR                  URn                  -  URb                  -  URj                  -  n4UR                  URn                  -  URj                  -  n5[p        Rr                  " SU SU-SU 3 SU. SU/S SURl                  S SURl                  S S U4S! S"U5S! S#URt                  S   S$   S% S&U0S S3U3-   5        URj                  URj                  U4U5U0URt                  S   S$   S'.n6U6R-                  UR3                  5        V7V&s0 s H  u  n7n&U7U&Rj                  _M     sn&n75        U6R3                  5        V7V&s0 s H  u  n7n&S(U7-   U&_M     n6n7n&U	b-  U6R3                  5        H  u  n7n&U	Rw                  U7U&U5        M     URx                  (       a(  [x        c   S)5       eUU6S*'   [x        R                  " U6US+9  UR{                  5         UR{                  5         GM‡     g ! , (       d  f       GN©= fs  sn!n f ! , (       d  f       GNV= f! , (       d  f       GNÔ= f! , (       d  f       GN¼= f! , (       d  f       GN½= f! , (       d  f       GN = f! , (       d  f       GNÖ= fs  sn2n1f s  sn&n7f s  sn&n7f )-N©Údevice_typeÚtrainr    é
   T©ÚdeviceÚdtypeÚnon_blocking©r@   rB   r-   Údist_Úoutput_dictÚloss)r-   Ú
logit_biasr   rG   g       @)Ú	norm_typeéd   g      Y@Ú ú: z#.5gz (Ú)zTrain Epoch: ú [Ú>Ú/z.0fz%)] Data (t): z.3fz Batch (t): z, z#gz/s, z/s/gpu LR: ÚlrÚ5fz Logit Scale: )Ú	data_timeÚ
batch_timeÚsamples_per_secondÚsamples_per_second_per_gpur6   rP   ztrain/úPlease install wandb.Ústep©rW   r'   )>Útorchr@   r	   Ú	precisionÚtyper   r=   ÚdistillÚevalÚ	set_epochÚ
dataloaderÚnum_batchesÚ
accum_freqÚmathÚceilÚlogÚnum_samplesr   ÚtimeÚ	enumerateÚskip_schedulerÚtor   Ú	zero_gradÚno_gradÚitemsr   Úvaluesr7   ÚpopÚappendÚrangeÚcatÚhorovodÚsynchronizeÚunscale_Úgrad_clip_normÚnnÚutilsÚclip_grad_norm_Ú
parametersÚskip_synchronizerW   r4   r-   Úclamp_r   Úlog_every_n_stepsÚlenÚ
world_sizeÚitemÚjoinÚ
capitalizer   r   Ú
batch_sizeÚloggingÚinfoÚparam_groupsÚ
add_scalarÚwandbr   )8r3   ÚdatarF   ÚepochÚ	optimizerr9   Ú	schedulerÚ
dist_modelÚargsÚ	tb_writerr@   ÚautocastÚinput_dtyper_   Únum_batches_per_epochÚsample_digitsÚaccum_imagesÚaccum_textsÚaccum_featuresÚlosses_mÚbatch_time_mÚdata_time_mÚendÚiÚbatchÚi_accumrW   ÚimagesÚtextsr.   r-   Údist_model_outÚkÚvÚlossesr8   ÚfÚkeyr   ÚjÚinputs_no_accumÚinputsÚaccumulatedÚbatch_countr‚   re   Úsamples_per_epochÚpercent_completeÚlogit_scale_scalarÚ	loss_nameÚloss_mÚloss_logrT   rU   Úlog_dataÚnames8                                                           r   Útrain_one_epochr²   @   s	  € Ü�\Š\˜$Ÿ+™+Ó&€FÜ˜DŸN™N¸¿¹ÑD€HÜ! $§.¡.Ó1€Kà	‡K�K„MØ‡|‡|Ø�‰Ôàˆ�M×Ñ˜EÔ"Ø�g‘×)Ñ)€JØ&×2Ñ2°d·o±oÑEÐÜ—I’IœdŸhšh z×'=Ñ'=ÀÑ'AÀ2ÓFÓG€Mà‡�˜ÓØ46¸¸B >�kˆà€HÜ“>€LÜ“.€KÜ
�)Š)‹+€CÜ˜j×)‰ˆˆ5Ø�t—‘Ñ&ˆØ$Ñ,¨wÑ6ˆà×"×"Ù�dŒOà‰ˆ�Ø—‘ &È$�ÐOˆØ—‘ °T�Ð:ˆà×Ñœ4Ÿ9š9›;¨Ñ,Ô-Ø×ÑÔà�?‰?˜aÓÙ•Ù! &¨%Ó0�	Ø'¨Ñ6�Ø—<—<ÜŸš�Ù)3°F¸EÓ)B˜÷ )à×$Ñ$À×AUÑAUÔAWÔ%XÒAW¹¸¸A¨¨a¨S k°1¢nÑAWÒ%XÔYÙÑ< 	Ñ<°tÒ<�ä  §¡£Ó1�
Ø!+��v‘÷ ô �Z Ö(ô —’•Ù•ZÙ % f¨eÓ 4�Iã:˜Ø!Ÿ™ a¨Ö.ñ ;ð %.§O¡OÖ$5™˜˜SØ .Ó0Ø*¨3Ñ/×6Ñ6°sÖ;à36°%˜N¨3Ó/ñ	 %6÷  ð ×#Ñ# FÔ+Ø×"Ñ" 5Ô)÷ !ð" �Q‘˜$Ÿ/™/Ñ)¨QÓ.âð
 ×ÑÔ!Ü˜4Ÿ?™?Ö+�Ø% a™�Ø# A™�Ù•ZÙ % f¨eÓ 4�Ià&(�OØCLÇ=Á=ÐQ^ÓC_Ð_�O MÑ2°[Ø# yÓ0Ø8A¿¹ÀlÓ8S˜¨Ñ5à�FØ$2×$8Ñ$8Ö$:™˜˜SØ&4°SÑ&9˜Ü&+§i¢i°¸B¸Q°À9ÈSÁ>ÐBRÑ0RÐU`ÐabÐefÑafÐagÐUhÑ0hÓ&i˜˜s›ñ %;ñ "ÑP FÐP¨oÑPÈ4ÒP�FØØ'Ü!$ V§]¡]£_Ó!5�JØ%/�F˜6‘N÷#  ô& ˜ VÖ,ñ- ,ð0 ÒØ�|�|Ø×%Ñ%Ô'Ø—‘ 	Ô*Ø×&Ñ&Ñ2Ü—H‘H—N‘N×2Ñ2°5×3CÑ3CÓ3EÀt×GZÑGZÐfiÐ2ÑjØ×/Ñ/Õ1Ø—K‘K 	Ô*÷ 2Ð1ð ×&Ñ&Ñ2Ø—O‘O IÔ.Ü—H‘H—N‘N×2Ñ2°5×3CÑ3CÓ3EÀt×GZÑGZÐfiÐ2ÑjØ—‘˜IÔ&Ø�M‰M�Oà×"Ñ"Ñ.Ü—‘—‘×.Ñ.¨u×/?Ñ/?Ó/AÀ4×CVÑCVÐbeÐ.ÑfØ�N‰NÔð �?‰?˜QÓØ8:¸BÀ ~˜+ˆLô �]Š]�_Ü˜Ó×+Ñ+×2Ñ2°1´d·h²h¸s³mÔD÷ ð 	×ÑœDŸIšI›K¨#Ñ-Ô.Ü�iŠi‹kˆØ ‘kˆÜ�T�?Œ? ¨$×*@Ñ*@Ñ @ÀAÓ EÈÐXmÖImÜ˜V›ˆJØ%¨
Ñ2°T·_±_ÑDÀtÇÁÑVˆKØ *× 6Ñ 6ÐØ$ {Ñ2Ð5JÑJÐð #ŸL™LžN‘��SØ˜hÓ&Ü$0£N�H˜S‘MØ˜‘×$Ñ$ S§X¡X£Z°Ö<ñ +ð
 "-×!1Ñ!1Ó!3ÐØ—x‘xð .6¯^©^Ô-=ôâ-=Ñ)˜	 6ð !×+Ñ+Ó-Ð.¨b°·±¸DÐ0AÀÀFÇJÁJÈtÐCTÐTUÓVÙ-=òóˆHð "&§¡°4·?±?Ñ!BÀTÇ_Á_Ñ!TÐWc×WgÑWgÑ!gÐØ)-¯©¸4¿?¹?Ñ)JÈ\×M]ÑM]Ñ)]Ð&Ü�LŠLØ ˜w b¨°Q°}°oÐ4EÐ(FÀaÐHYÐGZÐZ\Ð]mÐnqÐ\rð sØ(Ÿ_™_¨SÐ1ð 2Ø*×.Ñ.¨sÐ3°2Ð6HÈÐ5LÈDÐQkÐlnÐPoð pØ ×-Ñ-¨aÑ0°Ñ6°rÐ:ð ; Ø 2°3Ð7°qð	:ð =Eñ	Eôð )Ÿ_™_Ø*×.Ñ.Ø&8Ø.HØ+Ø×,Ñ,¨QÑ/°Ñ5ñˆHð �O‰O¸(¿.¹.Ô:JÔKÒ:J©h¨d°3˜T #§'¡'š\Ñ:JÒKÔLà>F¿n¹nÔ>NÔOÒ>N±°°s˜ 4™¨Ò,Ñ>NˆHÑOàÑ$Ø!)§¡Ö!1‘I�D˜#Ø×(Ñ(¨¨s°DÖ9ñ "2ð �z�zÜÑ(ÐAÐ*AÓAÐ(Ø#'�˜Ñ Ü—	’	˜(¨Ò.ð ×ÑÔ Ø×Ñ×òE *÷& )žüã%X÷ –ú÷   –Zú÷ !–ú÷8  –Zú÷6 2Ö1ú÷$ Ž_üó(ùó0 LùãOsŒ   Ç.5g'È#
gÈ-&g'Ég!É&2g'ËhËA!g9Ì,*hÎ0B7hÔh/Ø8:iÞ&<iã5iä*iç
gçg'ç'
g6	ç9
hèhè
h	è
h,è/
h>	é
i	c                 ó  • 0 n[        U5      (       d  U$ [        R                  " UR                  5      nU R                  5         [	        XX#US9nUR                  U5        [        UR                  UR                  S9n	[        UR                  5      n
SU;   Gaü  UR                  (       Gaê  X#R                  -  S:X  d  X#R                  :X  GaÈ  UR                  (       a  UR                  (       d  U R                  n US   R                  nSnUR                   nSnSn/ / nn[        R"                  " 5          [%        U5       GH´  u  nnUu  nnUR'                  XzSS9nUR'                  USS9nU	" 5          U " UU5      nUS	   nUS
   nUS   nUR)                  UR+                  5       5        UR)                  UR+                  5       5        UR-                  5       nUU-  UR/                  5       -  nUR/                  5       nUR0                  S   n[        R2                  " UUS9R5                  5       n[6        R8                  " UU5      [6        R8                  " UU5      -   S-  n[;        U5      nS S S 5        UWW-  -  nUU-  n[        U5      (       d  GMW  US-  S:X  d  GMc  [<        R>                  " SU SU SU SXì-  S S3	5        Wc  GM�  UUU-  -  n[<        R>                  " SXü-  S S35        GM·     [A        [        RB                  " U5      [        RB                  " U5      WR+                  5       S9n Xì-  n!UR                  0 U EU!RE                  5       X,S.E5        Wb%  Xü-  nUR                  SURE                  5       05        S S S 5        U(       d  U$ [<        R>                  " SU S3SRG                  URI                  5        V"V#s/ s H  u  n"n#U" S[K        U#S5      S 3PM     sn#n"5      -   5        URI                  5        V$V%s0 s H  u  n$n%SU$-   U%_M     n&n$n%URL                  (       a£  Ub-  U&RI                  5        H  u  n$n%URO                  U$U%U5        M     [Q        [R        RT                  RG                  URV                  S5      S5       n'U'RY                  [Z        R\                  " U5      5        U'RY                  S 5        S S S 5        UR^                  (       a^  [^        c   S!5       eS"U;   a.  US"   R                  nUR`                  URb                  -  n(U(U-  n)OS n)UU&S#'   [^        Rd                  " U&U)S$9  U$ ! , (       d  f       GNã= f! , (       d  f       GNØ= fs  sn#n"f s  sn%n$f ! , (       d  f       N¯= f)%N)Ú	tokenizerr;   r   r   g        Tr?   rC   r+   r,   r-   )r@   r)   rI   zEval Epoch: rM   z / z]	Clip Loss: z.6fÚ	zGenerative Loss: r*   )Úclip_val_lossr‰   re   Úval_generative_lossrJ   rK   é   z.4fzval/zresults.jsonlza+Ú
rV   r=   r‰   rX   )3r   rY   r@   r]   r   r   r	   rZ   r[   r   Úval_frequencyÚepochsÚdistributedrr   r1   r_   re   Úinference_moderg   ri   ro   ÚcpuÚmeanÚtÚshapeÚarangeÚlongÚFÚcross_entropyÚmaybe_compute_generative_lossrƒ   r„   Úget_clip_metricsrq   r   r€   rl   ÚroundÚ	save_logsr†   ÚopenÚosÚpathÚcheckpoint_pathÚwriteÚjsonÚdumpsr‡   r`   ra   rd   )*r3   rˆ   r‰   r�   rŽ   r´   Úmetricsr@   Úzero_shot_metricsr�   r�   r_   re   Úsamples_per_valÚcumulative_lossÚcumulative_gen_lossÚall_image_featuresÚall_text_featuresrš   r›   r�   rž   r.   r+   r,   r-   Úlogits_per_imageÚlogits_per_textr‚   Úlabelsr8   Úgen_lossÚval_metricsrF   r    r¡   r±   r   r°   r£   r‘   rW   s*                                             r   ÚevaluaterÝ   û   s   € Ø€GÜ�T�?‰?ØˆÜ�\Š\˜$Ÿ+™+Ó&€FØ	‡J�J„Lä& u°EÈ9ÑUÐØ‡N�NÐ$Ô%ä˜DŸN™N¸¿¹ÑD€HÜ! $§.¡.Ó1€Kà�„}˜$×,×,Ð,°5×;MÑ;MÑ3MÐRSÓ2SÐW\×`kÑ`kÔWkà×× D§L§LØ—L‘LˆEØ˜%‘[×+Ñ+ˆ
ØˆØ$×0Ñ0ˆð ˆØ!ÐØ02°BÐ-ÐÜ×!Ò!Õ#Ü% j×1‘��5Ø %‘�˜ØŸ™¨&ÐRV˜ÐW�ØŸ™¨¸T˜ÐB�á•ZÙ % f¨eÓ 4�IØ%.Ð/?Ñ%@�NØ$-¨oÑ$>�MØ"+¨MÑ":�Kð '×-Ñ-¨n×.@Ñ.@Ó.BÔCØ%×,Ñ,¨]×->Ñ->Ó-@ÔAØ"-×"2Ñ"2Ó"4�KØ'2°^Ñ'CÀmÇoÁoÓFWÑ'WÐ$Ø&6×&8Ñ&8Ó&:�Oà!'§¡¨a¡�JÜ"Ÿ\š\¨*¸VÑD×IÑIÓK�FäŸšÐ(8¸&ÓAÜŸš¨¸Ó@ñAàñ"�Jô
  =¸YÓG�H÷)  ð,   :°
Ñ#:Ñ:�Ø˜zÑ)�Ü˜T—?”?¨¨C©°A¦~Ü—L’LØ& u g¨R°¨}¸CÀÐ?Pð Q&Ø&5Ñ&CÀCÐ%HÈðLôMð  Ô+Ø+¨x¸*Ñ/DÑDÐ+ÜŸšØ/Ð0CÑ0QÐRUÐ/VÐVXÐY÷[ñI 2ôN +Ü$ŸyšyÐ);Ó<Ü#ŸišiÐ(9Ó:Ø'ŸO™OÓ-ñˆKð
 #Ñ0ˆDØ�N‰NØi�;Ði°·±³ÀuÒiôð Ñ#Ø.Ñ<�Ø—‘Ð 5°x·}±}³ÐGÔH÷g $öj Øˆä‡L‚LØ
�u�g˜QÐØ
�)‰)¸7¿=¹=¼?ÔKº?±4°1°a˜�s˜"œU 1 a›[¨Ð-Ó.¹?ÒKÓ
Lñ	Môð
 5<·M±M´OÔD²O¡y t¨S�˜‘˜sÒ"±O€HÑDà‡~‡~ØÑ Ø%Ÿ^™^Ö-‘	��cØ×$Ñ$ T¨3°Ö6ñ .ô ”"—'‘'—,‘,˜t×3Ñ3°_ÓEÀtÔLÐPQØ�G‰G”D—J’J˜wÓ'Ô(Ø�G‰G�DŒM÷ Mð ‡z‡zÜÑ Ð9Ð"9Ó9Ð Ø�d‹?Ø˜g™×1Ñ1ˆJØ$.×$:Ñ$:¸d¿o¹oÑ$MÐ!Ø(¨5Ñ0‰DàˆDØ!ˆ�ÑÜ�	Š	�( Ò&à€N÷[  –Zú÷ $Ö#üót Lùó E÷ MÕLúsP   Ä7?U Å6C<UÉ2#U ÊU Ê&'U ËB=U ÏU2ÐU8Ò7U>Õ
UÕU Õ 
U/Õ>
Vc                 óø  • 0 nX -  UR                  5       -  R                  5       R                  5       nUR                  5       R                  5       R                  5       nXES.n[        R                  " [        U5      5      R                  SS5      nUR                  5        HÓ  u  p‰[        R                  " U	SS9n
[        R                  " X§:H  5      S   nUR                  5       R                  5       R                  5       nUR                  5       S-   X8 S3'   [        R                  " [        R                  " U5      5      S-   X8 S3'   S H"  n[        R                  " X¼:  5      X8 S	U 3'   M$     MÕ     U$ )
N)Úimage_to_textÚtext_to_imageéÿÿÿÿr    T)Ú
descendingÚ
_mean_rankÚ_median_rank)r    é   r>   z_R@)rÀ   Údetachr¾   rY   rÂ   r}   Úviewrl   ÚargsortÚwhereÚnumpyr¿   ÚnpÚfloorÚmedian)r+   r,   r-   rÑ   rØ   rÙ   ÚlogitsÚground_truthr±   ÚlogitÚrankingÚpredsr    s                r   rÇ   rÇ   k  s9  € Ø€GØ#Ñ4°}·±Ó7HÑH×PÑPÓR×VÑVÓXÐØ&×(Ñ(Ó*×1Ñ1Ó3×7Ñ7Ó9€Oà/ÑR€FÜ—<’<¤ MÓ 2Ó3×8Ñ8¸¸QÓ?€Là—|‘|–~‰ˆÜ—-’- °$Ñ7ˆÜ—’˜GÑ3Ó4°QÑ7ˆØ—‘“×"Ñ"Ó$×*Ñ*Ó,ˆØ',§z¡z£|°aÑ'7ˆ�&˜
Ð#Ñ$Ü)+¯ª´"·)²)¸EÓ2BÓ)CÀaÑ)Gˆ�&˜Ð%Ñ&ÛˆAÜ')§w¢w¨u©yÓ'9ˆG�f˜C ˜s�OÓ$ó ñ &ð €Nr   c                 ó‚   • SU ;   a9  SU ;   a2  U S   nU S   n[         R                  " UR                  SSS5      U5      $ g g )Nrî   rÚ   r   r)   r    )rÄ   rÅ   Úpermute)r.   Útoken_logitsÚtoken_labelss      r   rÆ   rÆ     sM   € Ø�9Ó ¨YÓ!6Ø  Ñ*ˆØ  Ñ*ˆÜ�Š˜|×3Ñ3°A°q¸!Ó<¸lÓKÐKð "7Ðr   r   )NN)#rÏ   rƒ   rb   rË   rf   rê   rë   rY   Útorch.nn.functionalrv   Ú
functionalrÄ   Útorch.nn.parallel.distributedr   r‡   ÚImportErrorÚ	open_clipr   r   r   Úopen_clip_train.distributedr   Úopen_clip_train.zero_shotr   Úopen_clip_train.precisionr	   Úobjectr   r/   r4   r7   r²   rÝ   rÇ   rÆ   r'   r   r   Ú<module>r      s‡   ðÛ Û Û Û 	Û ã Û ß Ð Ý AðÛ÷ <Ñ ;Ý 1Ý 4Ý 2ô)�6ô )ò&òòôw ôvmò`ó(Løðc ó Ø‚Eðús   ­A2 Á2A=Á<A=