ó
    ÷“,j«]  ã                   óî  • S SK r S SKrS SKrS SKrS SKrS SKrS SKrS SKrS SKJr  S SK	J
r
  S SKrS SKrS SKJr   S SKr S SKJs  Jr   S SKJr  S SKJrJrJrJr  S SKJr  S SKJrJrJ r   S SK!J"r"  S S	K#J$r$  S S
K%J&r&J'r'J(r(  S SK)J*r*J+r+  S SK,J-r-J.r.J/r/J0r0  Sr1SS jr2S r3S\4S\54S jr6S r7S r8\9S:X  a  \7" \Rt                  SS 5        gg! \ a    Sr N—f = f! \ a    Sr N›f = f! \ a    Sr N¢f = f)é    N)Údatetime)Úpartial)Úoptim)Úcreate_model_and_transformsÚtrace_modelÚget_tokenizerÚcreate_loss)Úget_data)Ú	is_masterÚinit_distributed_deviceÚbroadcast_object)Úsetup_logging)Ú
parse_args)Ú	cosine_lrÚconst_lrÚconst_lr_cooldown)Útrain_one_epochÚevaluate)Úpt_loadÚcheck_existsÚstart_sync_processÚremote_synczepoch_latest.ptc                 ó¦   • [         R                  " X-   5        [        R                  R	                  X-   5        [        R                  " X-   5        g ©N)ÚtorchÚmanual_seedÚnpÚrandomÚseed)r   Úranks     Ú¢C:\Users\johnn\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.13_qbz5n2kfra8p0\LocalCache\local-packages\Python313\site-packages\open_clip_train/main.pyÚrandom_seedr"   ,   s2   € Ü	×Ò�d‘kÔ"Ü‡I�I‡N�N�4‘;ÔÜ
‡K‚K�‘Õó    c                 ó¶   • [         R                  " SU R                  5       5       Vs/ s H$  oR                  5       (       a  [	        U5      OUPM&     sn$ s  snf )z9See http://www.codinghorror.com/blog/archives/001018.htmlz(\d+))ÚreÚsplitÚlowerÚisdigitÚint)Ústring_Úss     r!   Únatural_keyr,   2   s>   € ä24·(²(¸8ÀWÇ]Á]Ã_Ô2UÓVÒ2U¨Q—i‘i—k‘kŒC�ŒF qÒ(Ñ2UÑVÐVùÒVs   ¨+AÚpathÚremotec           	      ó  • U(       aÁ  [         R                  " SSSU S-   /[         R                  [         R                  S9n[        U5        UR                  S:X  a  g UR
                  R                  5       R                  S5      S S  Vs/ s H4  n[        R                  R                  XR                  S	5      S   5      PM6     nnO[        R                  " U S
-   SS9nU(       a  [        U[        S9nUS   $ g s  snf )NÚawsÚs3ÚlsÚ/)ÚstdoutÚstderré   Ú
éÿÿÿÿÚ z**/*.ptT)Ú	recursive)Úkey)Ú
subprocessÚrunÚPIPEÚprintÚ
returncoder4   Údecoder&   Úosr-   ÚjoinÚglobÚsortedr,   )r-   r.   ÚresultÚxÚcheckpointss        r!   Úget_latest_checkpointrI   7   sÒ   € æÜ—’ ¨¨d°D¸3±JÐ ?Ì
ÏÉÔ`j×`oÑ`oÑpˆÜˆfŒØ×Ñ Ó!ØØEKÇ]Á]×EYÑEYÓE[×EaÑEaÐbfÓEgÐhkÐikÑElÓmÒElÀ”r—w‘w—|‘| D¯'©'°#«,°rÑ*:Ö;ÑElˆÐmˆä—i’i  yÑ 0¸DÑAˆÞÜ˜[¬kÑ:ˆØ˜2‰ÐØùò ns   Â;C<c                 ó`7  ^<^=• [        U 5      n [        R                  R                  5       (       ag  S[        R                  R                  R
                  l        S[        R                  R                  l        S[        R                  R                  l	        [        U 5      nU R                  c¯  U R                  R                  SS5      n[        R                  " 5       R!                  S5      nU R"                  (       a  [%        X5      nSR'                  USU 3SU R(                   3SU R*                   3S	U R,                   3S
U R.                   3/5      U l        U R0                  S:H  n[2        R4                  R'                  U R6                  U R                  5      nS U l        [;        X R<                  S9(       aœ  [2        R>                  " USS9  U R<                  (       a  SU R@                   3OSn[2        R4                  R'                  XV5      U l        [2        R4                  RC                  U R8                  5      (       a  U(       d  [E        S5        gU RF                  (       a  [H        RJ                  O[H        RL                  U l'        [Q        U R8                  U RN                  5        SU RR                  ;   =(       d    SU RR                  ;   U l*        SU RR                  ;   =(       d    SU RR                  ;   U l+        [2        R4                  R'                  US5      U l,        [;        U 5      (       av  U RV                  (       a   [2        R4                  R'                  US5      OSU l-        U RZ                  U RX                  4 H!  nU(       d  M  [2        R>                  " USS9  M#     OSU l-        U(       Gah  S nU RX                  n	U R\                  bn  [2        R4                  R'                  U R\                  U R                  S5      n	U R^                  (       a  [E        S5        gU R`                  S:w  a  [E        S5        g[;        U 5      (       a­  U R^                  (       aK  [2        R4                  R'                  U	[b        5      n[2        R4                  RC                  U5      (       d  S nO[e        X�R\                  S LS9nU(       a  [H        Rf                  " SU S35        O[H        Rf                  " SU	 S35        U R"                  (       a  [%        X5      nX€l        U Rh                  (       a  [i        U 5        S n
[;        U 5      (       GaT  U R\                  GbF  []        [2        R4                  R'                  U R6                  U R                  5      [2        R4                  R'                  U R\                  U R                  5      U R`                  5      nU(       a  [H        Rf                  " S5        O[H        Rf                  " S5        g[k        U Rl                  [2        R4                  R'                  U R6                  U R                  5      [2        R4                  R'                  U R\                  U R                  5      U R`                  5      n
U
Ro                  5         U R.                  S :X  a  [H        Rp                  " S!5        U Rr                  (       aL  [H        Rf                  " S"U Rt                   S#U R@                   S$U Rv                   S%U Rx                   S3	5        O�U R"                  (       aL  [H        Rf                  " S&U Rt                   S#U R@                   S$U Rv                   S%U Rx                   S3	5        O$[H        Rf                  " S'U Rt                   S35        S nU Rz                  S L=(       a    U R|                  S LU l?        U R~                  (       a2  U R€                  S(:X  d   eS)U R                  Rƒ                  5       ;  d   e[…        U R†                  [ˆ        [Š        45      (       a-  [�        U R†                  5      S(:X  a  U R†                  S*   U lC        [�        U R�                  S*5        0 nU R’                  (       a  [”        R–                  " S+5      US,'   S-US.'   [™        U R                  U Rš                  40 S/U R.                  _S0U_S1U Rœ                  _S2U Rž                  _S3U R                   _S4U R¢                  _S5U R†                  _S6U R¤                  _S7U R¦                  _S8U R¨                  _S9U Rª                  _S:U R¬                  _S;U R®                  _S<U R°                  _S=S_S>U R²                  _UD6u  pïnU R~                  (       a:  [™        U Rz                  U R|                  UU R.                  SU R²                  S?9u  n  nU R´                  bq  [E        S@5        S*S K[nS*SAK\J]n  [E        SBU R´                   35        [½        UR¾                  RÀ                  U R´                  5      nU" UU5        URÃ                  U5      n[�        U R�                  U R@                  5        U RÄ                  (       a  [Ç        XàR*                  USC9nU RÈ                  (       a$  URË                  U RÌ                  U RÎ                  SD9  U RÐ                  (       a$  URÓ                  U RÔ                  U RÖ                  SE9  U RØ                  (       a  URÛ                  5         [;        U 5      (       að  [H        Rf                  " SF5        [H        Rf                  " [Ý        U5       5        [H        Rf                  " SG5        [2        R4                  R'                  U R6                  U R                  SH5      n[ß        USI5       n[á        [ã        U 5      5       HB  n[½        U U5      n[H        Rf                  " SJU SKU 35        URå                  U SKU SL35        MD     S S S 5        U R"                  (       aÐ  U Rr                  (       d¿  U Ræ                  (       a)  [        R¾                  Rè                  Rë                  U5      n0 nU Rì                  (       a  SUSM'   [        R¾                  Rî                  Rð                  " U4SNU/0UD6nU R~                  (       a.  [        R¾                  Rî                  Rð                  " U4SNU/0UD6nS nS nU Rò                  (       d  U Rô                  SO:X  Gag  U RÄ                  (       a   SP5       e[½        U SQSR5      Rƒ                  5       nUR÷                  SS5      (       a¹  S*STK|J}n  URý                  SS5      S   n0 nU Rþ                  S L U GR                   S L :X  d   SU5       eU Rþ                  b  U Rþ                  U GR                   4USV'   U GR                  b  U GR                  USW'   U" UU4U R(                  U GR                  U GR                  SX.UD6nGOSY m=U=4SZ jn [‹        UGR	                  5       5      n!U! V"V#s/ s H,  u  n"n#T=" U"U#5      (       d  M  U#GR
                  (       d  M*  U#PM.     n$n"n#U! V"V#s/ s H,  u  n"n#U " U"U#5      (       d  M  U#GR
                  (       d  M*  U#PM.     n%n"n#USR:X  aY  G[        GR                  " U$S[S\.U%U GR                  S\./U R(                  U Rþ                  U GR                   4U GR                  S]9nO
 S^U 35       e[;        U 5      (       a­  G[        GR                  " UGR                  5      n&U GR                  U&S_'   S`R'                  U&GR                  5        V'V(s/ s H  u  n'n(U' SKU( 3PM     sn(n'5      n&[H        Rf                  " SaG[        U5      GR                   SbU GR                   ScU& 35        U Rr                  (       ac  G[        GR                   " UUGR	                  5       Sd9nG[        GR"                  " UGR%                  5       S*Se9  G[        GR&                  " US*Se9  S nU R.                  Sf:X  a    [        GR(                  GR+                  USg9nS*n*U R0                  GbN  G[1        U R0                  ShSi9n+SjU+;   aú  U+Sj   n*U+Sk   n,U R"                  (       do  G[3        G[5        U,GR                  5       5      5      S*   R÷                  Sl5      (       a3  U,GR                  5        V'V(s0 s H  u  n'n(U'[�        Sm5      S  U(_M     n,n'n(UGR7                  U,5        Ub  UGR7                  U+Sn   5        Ub  SoU+;   a  UGR7                  U+So   5        [H        Rf                  " SpU R0                   SqU* Sr35        O9UGR7                  U+5        [H        Rf                  " SsU R0                   SqU* Sr35        G[9        U R                  U R²                  U R¤                  St9n-G[;        U UU4U*U-Su9m<[�        T<5      (       d   Sv5       eS n.SwT<;   Gas  UGbo  T<Sw   GR<                  GR>                  U R€                  -  U GR@                  -  n/U GRB                  Sx:X  a&  G[E        UU R(                  U GRF                  U/5      n.GOU GRB                  Sy:X  a%  G[I        UU R(                  U GRF                  U/5      n.OÌU GRB                  Sz:X  a‰  U GRJ                  c   S{5       eT<Sw   GR<                  GR>                  U R€                  -  U GRJ                  -  n0G[M        UU R(                  U GRF                  U/U0U GRN                  U GRP                  5      n.O2[H        GRR                  " S|U GRB                   S}35        G[U        S(5        U R6                  =(       a/    U R6                  Rƒ                  5       S~:g  =(       a    [;        U 5      U l«        S n1U GRV                  (       a@  U RV                  (       a/  [V        c   S5       e[V        GRX                  " U RZ                  5      n1U RT                  (       Ga7  [;        U 5      (       Ga&  [T        c   S€5       e[H        RF                  " S�5        T<Sw   GR<                  GRZ                  U l®        U GR^                  b   T<S‚   GR<                  GRZ                  U l°        [T        GRb                  " U GRd                  U R                  U R                  U GRf                  / U R0                  S:X  a  SƒOS [ã        U 5      S„9  U RF                  (       a  [T        GRh                  " USS…9  [T        GRj                  " W5        [H        RF                  " S†5        Un2U GRl                  (       Ga  [H        Rf                  " S‡5        U RØ                  (       aH  U R"                  (       a7  [H        Rf                  " Sˆ5        S[        GRn                  GRp                  l¹        S‰n3[H        GRt                  GRv                  GRx                   HQ  nUR÷                  U35      (       d  M  [H        GRz                  " U5      GR}                  [H        GR~                  5        MS     [        GR€                  " U25      nSwT<;  a+  U R´                  b  S*SŠK\JÁn4  U4" U5        G[…        UT<U*U U1U-S‹9  g G[‡        U 5      n5G[‰        U*U GR@                  5       GH  n6[;        U 5      (       a  [H        Rf                  " SŒU6 35        G[‹        UT<U5U6UUU.XÀU1S�9
  U6S(-   n7G[�        U<4SŽ jS� 5       5      (       ag  G[…        UT<U7U U1U-S‹9  U R"                  (       aG  U Rr                  (       a  G[        R&                  " 5         O[        R"                  GR�                  5         U GRV                  (       GaÖ  U7U R                  U2GR%                  5       UGR%                  5       S�.n8Ub  UGR%                  5       U8So'   U7U GR@                  :X  d%  U GR�                  S*:”  aX  U7U GR�                  -  S*:X  aD  [        GRj                  " U8[2        R4                  R'                  U RX                  S‘U7 S’35      5        U GR’                  (       al  [2        R4                  R'                  U RX                  S‘U7S(-
   S’35      n9[2        R4                  RC                  U95      (       a  [2        GR”                  " U95        U R^                  (       a‡  [2        R4                  R'                  U RX                  S“5      n:[2        R4                  R'                  U RX                  [b        5      n;[        GRj                  " U8U:5        [2        R                  " U:U;5        U R"                  (       d  GMÃ  U Rr                  (       a  G[        R&                  " 5         GMí  [        R"                  GR�                  5         GM     U RT                  (       a&  [;        U 5      (       a  [T        GR–                  " 5         U
b×  [H        Rf                  " S”5        U
GR™                  5         []        [2        R4                  R'                  U R6                  U R                  5      [2        R4                  R'                  U R\                  U R                  5      U R`                  5      nU(       a  [H        Rf                  " S•5        g [H        Rf                  " S–5        g g ! , (       d  f       GN= fs  sn#n"f s  sn#n"f s  sn(n'f ! G[,        G[.        4 a5  n)[        R                  GR(                  GR+                  5       n S n)A)GNS n)A)ff = fs  sn(n'f )—NTFr3   Ú-z%Y_%m_%d-%H_%M_%SÚmodel_Úlr_Úb_Új_Úp_Úlatest)Úlocal)Úexist_okzout-zout.logzLError. Experiment already exists. Use --name {} to specify a new experiment.r8   ÚwandbÚallÚtensorboardrH   Ú zFError. Cannot use save-most-recent with remote_sync and resume latest.r1   z<Error. Sync protocol not supported when using resume latest.)r.   z"Found latest resume checkpoint at Ú.z%No latest resume checkpoint found in zremote sync successful.z#Error: remote sync failed. Exiting.Úfp16z‡It is recommended to use AMP mixed-precision instead of FP16. FP16 support needs further verification and tuning, especially for train.zARunning in horovod mode with multiple processes / nodes. Device: z.Process (global: z, local z	), total z=Running in distributed mode with multiple processes. Device: z&Running with a single process. Device r6   Úcocar   é
   Úinit_logit_scaleiöÿÿÿÚinit_logit_biasÚ	precisionÚdeviceÚjitÚforce_quick_geluÚforce_custom_textÚforce_patch_dropoutÚforce_image_sizeÚforce_context_lengthÚ
image_meanÚ	image_stdÚimage_interpolationÚimage_resize_modeÚaug_cfgÚpretrained_imageÚoutput_dictÚ	cache_dir)r_   r^   rl   rm   z¶=> using a layer from bitsandbytes.
   this is an experimental feature which requires two extra pip installs
   pip install bitsandbytes triton   please make sure to use triton 2.0.0)Úreplace_linearz => replacing linear layers with )Ú
batch_sizer_   )Úunlocked_groupsÚfreeze_bn_stats)Úunlocked_layersÚfreeze_layer_normzModel:zParams:z
params.txtÚwz  z: r7   Ústatic_graphÚ
device_idsÚ	syntheticzCannot train with traced modelÚoptÚadamwztimm/)Úcreate_optimizer_v2zUWhen using timm optimizer, BOTH beta1 and beta2 must be specified (or not specified).ÚbetasÚmomentum)ÚlrÚweight_decayÚepsc                 ó€   • UR                   S:  =(       d)    SU ;   =(       d    SU ;   =(       d    SU ;   =(       d    SU ;   $ )Né   ÚbnÚlnÚbiasÚlogit_scale)Úndim)ÚnÚps     r!   Ú<lambda>Úmain.<locals>.<lambda>O  s7   €  1§6¡6¨A¡:×#l°¸±×#l¸dÀa¹i×#lÈ6ÐUVÉ;×#lÐZgÐklÑZlÐ#lr#   c                 ó   >• T" X5      (       + $ r   © )r‡   rˆ   Úexcludes     €r!   r‰   rŠ   P  s   ø€ ¡w¨q£}Õ#4r#   g        )Úparamsr~   )r}   r{   r   zUnknown optimizer r~   z, zCreated z (z) optimizer: )Únamed_parameters)Ú	root_rankÚamp)r_   Úcpu)Úmap_locationÚepochÚ
state_dictÚmodulezmodule.Ú	optimizerÚscalerz=> resuming checkpoint 'z	' (epoch Ú)z=> loaded checkpoint ')rm   Úcontext_length)r”   Ú	tokenizerz5At least one train or eval dataset must be specified.ÚtrainÚcosineÚconstzconst-cooldownzBPlease specify the number of cooldown epochs for this lr schedule.zUnknown scheduler, z7. Available options are: cosine, const, const-cooldown.ÚnonezPlease install tensorboard.zPlease install wandb.zStarting wandb.ÚvalÚauto)ÚprojectÚnameÚidÚnotesÚtagsÚresumeÚconfig)ÚlogzFinished loading wandb.zCompiling model...z?Disabling DDP dynamo optimizer when grad checkpointing enabled.)ztorch._dynamoztorch._inductorztorch._functorchztorch._utils_internalztorch.fx)Ú$convert_int8_model_to_inference_mode)Ú	tb_writerr›   zStart epoch )r«   c              3   ó,   >#   • U  H	  oT;   v •  M     g 7fr   rŒ   )Ú.0ÚvÚdatas     €r!   Ú	<genexpr>Úmain.<locals>.<genexpr>ò  s   øé € ÐIÒ"H˜Q�DŽyÒ"Hùs   ƒ)r    zimagenet-valzimagenet-v2)r”   r£   r•   r—   Úepoch_z.ptztmp.ptzFinal remote sync.zFinal remote sync successful.zFinal remote sync failed.)Ír   r   ÚcudaÚis_availableÚbackendsÚmatmulÚ
allow_tf32ÚcudnnÚ	benchmarkÚdeterministicr   r£   ÚmodelÚreplacer   ÚnowÚstrftimeÚdistributedr   rC   r}   ro   Úworkersr^   r§   rB   r-   ÚlogsÚlog_pathr   Ú	log_localÚmakedirsr    Úexistsr?   ÚdebugÚloggingÚDEBUGÚINFOÚ	log_levelr   Ú	report_torT   rV   Úcheckpoint_pathÚtensorboard_pathr   Úsave_most_recentÚremote_sync_protocolÚLATEST_CHECKPOINT_NAMErI   ÚinfoÚcopy_codebaser   Úremote_sync_frequencyÚstartÚwarningÚhorovodr_   Ú
local_rankÚ
world_sizeÚdistill_modelÚdistill_pretrainedÚdistillÚ
accum_freqr'   Ú
isinstancerd   ÚtupleÚlistÚlenr"   r   Úsiglipr   r©   r   Ú
pretrainedÚtorchscriptra   rb   rc   re   rf   rg   rh   ri   rj   rk   rm   Úuse_bnb_linearÚbitsandbytesÚopen_clip.utilsrn   ÚgetattrÚnnÚtriton_based_modulesÚtoÚtracer   Ú
lock_imageÚlock_image_towerÚlock_image_unlocked_groupsÚlock_image_freeze_bn_statsÚ	lock_textÚlock_text_towerÚlock_text_unlocked_layersÚlock_text_freeze_layer_normÚgrad_checkpointingÚset_grad_checkpointingÚstrÚopenrE   ÚvarsÚwriteÚuse_bn_syncÚSyncBatchNormÚconvert_sync_batchnormÚddp_static_graphÚparallelÚDistributedDataParallelÚ
train_dataÚdataset_typeÚ
startswithÚ
timm.optimrz   r&   Úbeta1Úbeta2r|   Úwdr   r�   Úrequires_gradr   ÚAdamWÚcopyÚdeepcopyÚdefaultsÚitemsÚtypeÚ__name__rx   ÚhvdÚDistributedOptimizerÚbroadcast_parametersr•   Úbroadcast_optimizer_stater‘   Ú
GradScalerÚAttributeErrorÚ	TypeErrorr   ÚnextÚiterÚload_state_dictr   r
   Ú
dataloaderÚnum_batchesÚepochsÚlr_schedulerr   Úwarmupr   Úepochs_cooldownr   Úlr_cooldown_powerÚlr_cooldown_endÚerrorÚexitÚ	save_logsÚSummaryWriterÚnum_samplesÚtrain_szÚval_dataÚval_szÚinitÚwandb_project_nameÚwandb_notesÚwatchÚsaveÚtorchcompileÚ_dynamor¨   Úoptimize_ddpÚrootÚmanagerÚ
loggerDictÚ	getLoggerÚsetLevelÚWARNINGÚcompilerª   r   r	   Úranger   ÚanyÚbarrierÚsave_frequencyÚdelete_previous_checkpointÚremoveÚfinishÚ	terminate)>Úargsr_   Úmodel_name_safeÚdate_strÚresume_latestÚlog_base_pathÚlog_filenameÚdirnameÚresume_fromrÌ   Úremote_sync_processrF   Ú
dist_modelÚmodel_kwargsr»   Úpreprocess_trainÚpreprocess_valÚ_Úbnbrn   Úlinear_replacement_clsÚparams_fileÚfr£   r    Úddp_argsr—   r˜   rx   rz   Útimm_optÚ
opt_kwargsÚincluder�   r‡   rˆ   Úgain_or_bias_paramsÚrest_paramsr  Úkr®   ÚeÚstart_epochÚ
checkpointÚsdr›   Ú	schedulerÚtotal_stepsÚcooldown_stepsÚwriterÚoriginal_modelÚfilter_prefixesrª   Úlossr”   Úcompleted_epochÚcheckpoint_dictÚprevious_checkpointÚtmp_save_pathÚlatest_save_pathr¯   r�   s>                                                               @@r!   Úmainri  G   s  ù€ Ü�dÓ€Dä‡z�z×Ñ× Ñ ð 15Œ�‰×Ñ×"Ñ"Ô-Ø)-Œ�‰×ÑÔ&Ø-2Œ�‰×ÑÔ*ô % TÓ*€Fð ‡y�yÑàŸ*™*×,Ñ,¨S°#Ó6ˆÜ—<’<“>×*Ñ*Ð+>Ó?ˆØ××ä'¨Ó7ˆHØ—H‘HØØ�_Ð%Ð&Ø�$—'‘'�ˆOØ�—‘Ð!Ð"Ø�—‘�ÐØ�—‘Ð Ð!ð
ó ˆŒ	ð —K‘K 8Ñ+€MÜ—G‘G—L‘L §¡¨D¯I©IÓ6€MØ€D„MÜ�Ÿ^™^×,Ü
�Š�M¨DÒ1Ø-1¯^¯^˜˜dŸi™i˜[Ñ)ÀˆÜŸ™Ÿ™ ]ÓAˆŒÜ�7‰7�>‰>˜$Ÿ-™-×(Ñ(¶ÜØ^ôð ð '+§j§j”W—]’]´g·l±l€D„NÜ�$—-‘- §¡Ô0ð ˜DŸN™NÑ*×E¨e°t·~±~Ñ.E€D„JØ$¨¯©Ñ6×Q¸%À4Ç>Á>Ñ:Q€DÔÜŸ7™7Ÿ<™<¨°}ÓE€DÔÜ�‡�ØNR×N^×N^¤§¡§¡¨]¸MÔ JÐdfˆÔØ×-Ñ-¨t×/CÑ/CÓDˆGßˆwÜ—’˜G¨dÔ3ò Eð !#ˆÔçØˆØ×.Ñ.ˆà×ÑÑ'Ü Ÿg™gŸl™l¨4×+;Ñ+;¸T¿Y¹YÈÓVˆOØ×$×$ÜÐ^Ô_ØØ×(Ñ(¨DÓ0ÜÐTÔUØÜ�T�?‰?ð ×$×$ä Ÿg™gŸl™l¨?Ô<RÓS�Ü—w‘w—~‘~ k×2Ñ2à"&�Køô 4°O×L\ÑL\ÐdhÐLhÑi�ÞÜ—’ÐAÀ+ÀÈaÐPÕQä—’ÐDÀ_ÐDUÐUVÐWÔXØ××ä*¨4Ó=ˆKØ!Œà××Ü�dÔð ÐÜ�‡‚˜4×+Ñ+Ò7äÜ�G‰G�L‰L˜Ÿ™ D§I¡IÓ.Ü�G‰G�L‰L˜×)Ñ)¨4¯9©9Ó5Ø×%Ñ%ó
ˆö
 Ü�LŠLÐ2Õ3ä�LŠLÐ>Ô?Øä0Ø×&Ñ&Ü�G‰G�L‰L˜Ÿ™ D§I¡IÓ.Ü�G‰G�L‰L˜×)Ñ)¨4¯9©9Ó5Ø×%Ñ%ó	
Ðð 	×!Ñ!Ô#à‡~�~˜ÓÜ�ŠðXô	Yð ‡|‡|Ü�ŠØOÐPT×P[ÑP[È}ð ] Ø $§	¡	˜{¨(°4·?±?Ð2CÀ9ÈTÏ_É_ÐL]Ð]^ð`õ	að 
×	×	Ü�ŠØKÈDÏKÉKÈ=ð Y Ø $§	¡	˜{¨(°4·?±?Ð2CÀ9ÈTÏ_É_ÐL]Ð]^ð`õ	aô 	�ŠÐ=¸d¿k¹k¸]È!ÐLÔMà€JØ×%Ñ%¨TÐ1×Y°d×6MÑ6MÐUYÐ6Y€D„LØ‡|‡|à�‰ !Ó#Ð#Ð#à˜TŸZ™Z×-Ñ-Ó/Ó/Ð/Ð/ä�$×'Ñ'¬%´¨×7Ñ7¼CÀ×@UÑ@UÓ<VÐZ[Ó<[à $× 5Ñ 5°aÑ 8ˆÔÜ�—	‘	˜1ÔØ€LØ‡{‡{Ü+-¯6ª6°"«:ˆÐ'Ñ(Ø*-ˆÐ&Ñ'Ü.IØ�
‰
Ø�‰ò/ð —.’.ð/ñ ð	/ð
 ×Òð/ð ×.Ò.ð/ð ×0Ò0ð/ð !×4Ò4ð/ð ×.Ò.ð/ð "×6Ò6ð/ð —?’?ð/ð —.’.ð/ð !×4Ò4ð/ð ×0Ò0ð/ð —’ð/ð  ×.Ò.ð!/ñ" ð#/ð$ —.’.Ø
ñ'/Ñ+€E˜^ð* ‡|‡|ä6Ø×ÑØ×#Ñ#ØØ—n‘nØØ—n‘nñ
Ñˆ
�A�qð ×ÑÑ&Üð 8ô 	9ó 	#Ý2ÜÐ0°×1DÑ1DÐ0EÐFÔGÜ!(¨¯©×)DÑ)DÀd×FYÑFYÓ!ZÐÙ�uÐ4Ô5Ø—‘˜Ó ˆä�—	‘	˜4Ÿ9™9Ô%à‡z‡zÜ˜E¯o©oÀfÑMˆà‡‡à×ÑØ ×;Ñ;Ø ×;Ñ;ð 	ñ 	=ð ‡~‡~Ø×ÑØ ×:Ñ:Ø"×>Ñ>ð 	ñ 	@ð ××Ø×$Ñ$Ô&ä�‡�Ü�Š�XÔÜ�Šœ˜E›
�|Ô%Ü�Š�YÔÜ—g‘g—l‘l 4§9¡9¨d¯i©i¸ÓFˆÜ�+˜sÔ# qÜœt D›zÖ*�Ü˜d DÓ)�Ü—’˜r $  r¨#¨Ð/Ô0Ø—‘˜4˜&  3 % rÐ*Ö+ñ +÷ $ð ×× §§Ø××Ü—H‘H×*Ñ*×AÑAÀ%ÓHˆEØˆØ× × à'+ˆH�^Ñ$Ü—‘×!Ñ!×9Ò9¸%ÑaÈVÈHÐaÐX`Ñaˆà�<�<ÜŸ™×*Ñ*×BÒBÀ:ÑoÐ[aÐZbÐoÐfnÑoˆJð €IØ€Fà‡‡˜$×+Ñ+¨{Ô:Ø—:—:Ð?Ð?Ó?ˆ~ä�d˜E 7Ó+×1Ñ1Ó3ˆØ�>‰>˜'×"Ñ"Ý6Ø—y‘y Ó)¨"Ñ-ˆHØˆJØ—J‘J $Ð&¨D¯JªJ¸$Ð,>Ó?ð hØgóhÐ?à�z‰zÑ%Ø'+§z¡z°4·:²:Ð&>�
˜7Ñ#Ø�}Š}Ñ(Ø)-¯ª�
˜:Ñ&Ù+ØØðð —7‘7Ø!ŸWšWØ—H’Hñð ñŠIñ mˆGÜ4ˆGä# E×$:Ò$:Ó$<Ó=ÐÙ1AÔ"gÒ1A©¨¨AÁWÈQÐPQÇ]£1ÐWX×WfÖWf§1Ñ1AÐÑ"gÙ)9Ô_Ò)9¡  A¹WÀQÈ¿]›1ÈqÏÎŸ1Ñ)9ˆKÑ_à�g‹~Ý!ŸK›Kà#6ÈÑKØ#.ÀÇÂÑHðð —w‘wØŸ:™: t§z¢zÐ2ØŸšñ‘	ð 9Ð 2°3°%Ð8Ó8�uä�T�?‰?Ý—}“} Y×%7Ò%7Ó8ˆHØ'+§w¢wˆH�^Ñ$Ø—y‘y¸(¿.º.Ô:JÔ!KÒ:J±$°!°Q Q C r¨!¨£+Ñ:JÒ!KÓLˆHÜ�LŠLØ�4 	›?×3Ò3Ð4°B°t·x²x°jÀÈhÈZÐXôð �<�<Ý×0Ó0°ÈU×McÒMcÓMeÑfˆIÝ×$Ó$ U×%5Ò%5Ó%7À1ÒEÝ×)Ó)¨)¸qÒAàˆØ�>‰>˜UÓ"ð5ÜŸš×-Ò-°VÐ-Ð<�ð
 €KØ‡{�{ÒÝ˜TŸ[™[°uÑ=ˆ
Ø�jÓ à$ WÑ-ˆKØ˜LÑ)ˆBØ×#×#­­T°"·(²(³*Ó-=Ó(>¸qÑ(A×(LÑ(LÈX×(VÑ(VØ8:¿º¼
ÔCº
±°°1�aœ˜I›˜Ð(¨!Ò+¹
�ÑCØ×!Ò! "Ô%ØÑ$Ø×)Ò)¨*°[Ñ*AÔBØÑ! h°*Ó&<Ø×&Ò& z°(Ñ';Ô<Ü�LŠLÐ3°D·K±K°=À	È+ÈÐVWÐXÕYð ×!Ò! *Ô-Ü�LŠLÐ1°$·+±+°¸iÈÀ}ÐTUÐVÔWõ ˜dŸj™j°D·N±NÐSW×SlÑSlÑm€IÝØØ	˜>Ð*ØØñ	€Dô ˆt�9‰9ÐMÐMÓMˆ9ð €IØ�$„˜9Ò0Ø˜G‘}×/Ò/×;Ò;¸t¿¹ÑNÐRV×R]ÒR]Ñ]ˆØ×Ò Ó(Ý! )¨T¯W©W°d·k²kÀ;ÓOŠIØ×Ò 'Ó)Ý  ¨D¯G©G°T·[²[À+ÓN‰IØ×ÒÐ"2Ó2Ø×'Ò'Ñ3ð UØTóUÐ3à" 7™m×6Ò6×BÒBÀdÇoÁoÑUÐY]×YmÒYmÑmˆNÝ)Ø˜4Ÿ7™7 D§K¢K°Ø × 6Ò 6¸×8LÒ8LóN‰Iô �M‹MØ% d×&7Ò&7Ð%8Ð8oÐpôrå�ŒGð —Y‘Y×R 4§9¡9§?¡?Ó#4¸Ñ#>×RÄ9ÈTÃ?€D„NØ€FØ‡~‡~€~˜$×*×*ÜÑ&ÐEÐ(EÓEÐ&Ü×*Ó*¨4×+@Ñ+@ÓAˆà‡z‡z€z”i —o’oÜÑ Ð9Ð"9Ó9Ð Ü�ŠÐ'Ô(Ø˜W™×0Ò0×<Ò<ˆŒØ�=Š=Ñ$Ø˜u™+×0Ò0×<Ò<ˆDŒKä�
‹
Ø×+Ò+Ø—‘Ø�y‰yØ×"Ò"ØØ!Ÿ[™[¨HÓ4‘6¸$Ü˜“:ò	
ð �:�:Ü�K‹K˜ 5Ò)Ü�
‹
�;ÔÜ�ŠÐ/Ô0ð
 €NØ××ÑÜ�ŠÐ)Ô*à×"×" t×'7×'7Ü�LŠLÐZÔ[à05ŒE�MŠM× Ò Ô-ð
ˆô —L’L×(Ò(×3Õ3ˆDØ�‰˜×/Ó/Ü×!Ó! $Ó'×0Ò0´·²ÖAñ 4ô —“˜nÓ-ˆà�dÓà×ÑÑ*ÝLÙ0°Ô7å�˜˜k¨4¸6ÈYÒWØå�tÓ€Då�{ D§K¢K×0ˆÜ�T�?‰?Ü�LŠL˜<¨ wÐ/Ô0å˜˜t T¨5°)¸VÀYÐPZÐlrÒsØ !™)ˆåÔIÑ"HÓI×IÑIÝ�U˜D /°4À6ÐU^Ò_à××Ø—<—<Ý—H’H•Jä×%Ñ%×-Ò-Ô/ð �>�>‰>à(ØŸ	™	Ø,×7Ò7Ó9Ø&×1Ò1Ó3ñ	ˆOð Ñ!Ø,2×,=Ò,=Ó,?� Ñ)à $§+¢+Ó-Ø×#Ò# aÓ'¨_¸t×?RÒ?RÑ-RÐWXÓ,Xä—
“
Ø#Ü—G‘G—L‘L ×!5Ñ!5¸ÀÐ?PÐPSÐ7TÓUôð ×.×.Ð.Ü&(§g¡g§l¡l°4×3GÑ3GÈ6ÐRaÐdeÑReÐQfÐfiÐIjÓ&kÐ#Ü—7‘7—>‘>Ð"5×6Ñ6Ü—I“IÐ1Ô2à×$×$ä "§¡§¡¨T×-AÑ-AÀ8Ó L�Ü#%§7¡7§<¡<°×0DÑ0DÔF\Ó#]Ð Ü—
“
˜?¨MÔ:Ü—
’
˜=Ð*:Ô;ð ××ÒØ�|�|Ý—’—
ä×!Ñ!×)Ò)×+ñg 1ðj ‡z‡z”i —o‘oÜ�‹Œð Ñ&Ü�ŠÐ)Ô*Ø×%Ò%Ô'ÜÜ�G‰G�L‰L˜Ÿ™ D§I¡IÓ.Ü�G‰G�L‰L˜×)Ñ)¨4¯9©9Ó5Ø×%Ñ%ó
ˆö
 Ü�LŠLÐ8Õ9ä�LŠLÐ4Õ5ð '÷G $Ö#üód #hùÛ_ùó$ "Løõ #¥IÐ.ó 5ÜŸ™Ÿš×2Ò2Ó4–ûð5üó Dsi   ïAAl<ù Amù9AmúAmúAmú4AmûAmþAm
ÁA>Am  ÁD3An*Ál<
AmÁm An'Ám2*An"Án"An'c           	      óÞ  • SSK JnJn  [        R                  R                  U R                  U R                  S5      n[        R                  R                  U5      (       a  [        SU S35        g[        SU 35        [        R                  R                  [        5      n[        S5       H"  n[        R                  R                  U5      nM$     U" XCU" S	S
S5      S9  [        S5        g)Nr   )ÚcopytreeÚignore_patternsÚcodez$Error. Experiment already exists at z). Use --name to specify a new experiment.r8   zCopying codebase to é   r©   rÁ   rT   )ÚignorezDone copying code.r6   )Úshutilrk  rl  rB   r-   rC   rÁ   r£   rÅ   r?   ÚrealpathÚ__file__r8  rF  )r@  rk  rl  Únew_code_pathÚcurrent_code_pathrM  s         r!   rÒ   rÒ   2  s¸   € ß0Ü—G‘G—L‘L §¡¨D¯I©I°vÓ>€MÜ	‡w�w‡~�~�m×$Ñ$ÜØ2°=°/ÐAjÐkô	
ð Ü	Ð   Ð
0Ô1ÜŸ™×(Ñ(¬Ó2ÐÜ�1ŽXˆÜŸG™GŸO™OÐ,=Ó>Òñ áÐ±oÀeÈVÐU\Ó6]Ò^Ü	Ð
ÔØr#   Ú__main__r6   )é*   r   );r	  rD   rÇ   rB   r%   r<   Úsysr   r   Ú	functoolsr   Únumpyr   r   r   rT   ÚImportErrorÚtorch.utils.tensorboardÚutilsrV   Úhorovod.torchr  Ú	open_clipr   r   r   r	   Úopen_clip_train.datar
   Úopen_clip_train.distributedr   r   r   Úopen_clip_train.loggerr   Úopen_clip_train.paramsr   Úopen_clip_train.schedulerr   r   r   Úopen_clip_train.trainr   r   Úopen_clip_train.file_utilsr   r   r   r   rÐ   r"   r,   rö   ÚboolrI   ri  rÒ   r  ÚargvrŒ   r#   r!   Ú<module>rˆ     s  ðÛ Û Û Û 	Û 	Û Û 
Û Ý Ý ã Û Ý ðÛðß1Ð1ðÝ÷ [Ó ZÝ )ß \Ñ \Ý 0Ý -ß LÑ Lß ;ß ]Ó ]ð +Ð ôòWð
 ð ¨dô ò h6òVð" ˆzÓÙˆ�‰�!�"ˆÕð øðc ó Ø‚Eðûð
 ó Ø‚Kðûð
 ó Ø
‚Cðús5   ¼C Á	C ÁC) ÃCÃCÃC&Ã%C&Ã)C4Ã3C4