ó
    ð“,j6  ã                   ó¦   • S r SSKrSSKrSSKrSSKrSSKJr  SSKJ	r	J
r
Jr  SSKJr  SSKr " S S\
5      rS rS rS	 rS
 r\S:X  a  \" 5         gg)aŠ  Example Usage
cpu:

s3tokenizer --wav_scp xxx.scp             --device "cpu"             --output_dir "./"             --batch_size 32

gpu:

torchrun --nproc_per_node=8 --nnodes=1      --rdzv_id=2024 --rdzv_backend="c10d" --rdzv_endpoint="localhost:0"     `which s3tokenizer` --wav_scp xxx.scp                 --device "cuda"                 --output_dir "./"                 --batch_size 32

é    N)Ú
DataLoaderÚDatasetÚDistributedSampler)Útqdmc                   ó&   • \ rS rSrS rS rS rSrg)ÚAudioDataseté-   c                 ó(  • / U l         / U l        [        USSS9 nU HY  nUR                  5       R	                  5       u  pEU R                   R                  U5        U R                  R                  U5        M[     S S S 5        g ! , (       d  f       g = f)NÚrzutf-8)Úencoding)ÚdataÚkeysÚopenÚstripÚsplitÚappend)ÚselfÚwav_scpÚfÚlineÚkeyÚ	file_paths         Ú�C:\Users\johnn\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.13_qbz5n2kfra8p0\LocalCache\local-packages\Python313\site-packages\s3tokenizer/cli.pyÚ__init__ÚAudioDataset.__init__/   sn   € ØˆŒ	ØˆŒ	ä�'˜3¨Ò1°QÛ�Ø!%§¡£×!3Ñ!3Ó!5‘�Ø—	‘	× Ñ  Ô+Ø—	‘	× Ñ  Ö%ñ ÷ 2×1Ö1ús   šA BÂ
Bc                 ó,   • [        U R                  5      $ ©N)Úlenr   )r   s    r   Ú__len__ÚAudioDataset.__len__9   s   € Ü�4—9‘9‹~Ðó    c                 óœ   • U R                   U   nU R                  U   n[        R                  " U5      n[        R                  " U5      nX54$ r   )r   r   Ús3tokenizerÚ
load_audioÚlog_mel_spectrogram)r   Úidxr   r   ÚaudioÚmels         r   Ú__getitem__ÚAudioDataset.__getitem__<   sC   € Ø—I‘I˜c‘Nˆ	Ø�i‰i˜‰nˆÜ×&Ò& yÓ1ˆÜ×-Ò-¨eÓ4ˆØˆxˆr!   )r   r   N)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r   r   r)   Ú__static_attributes__© r!   r   r   r   -   s   † ò&òõr!   r   c                 óž   • U  Vs/ s H  oS   PM	     nnU  Vs/ s H  oS   PM	     nn[         R                  " U5      u  p4X#U4$ s  snf s  snf )Nr   é   )r#   Úpadding)ÚbatchÚitemr   ÚmelsÚ	mels_lenss        r   Ú
collate_fnr8   D   sS   € Ù %Ó&¢˜�ŒG¡€DÐ&Ù %Ó&¢˜�ŒG¡€DÐ&Ü!×)Ò)¨$Ó/�O€DØ�yÐ Ð ùò 'ùÚ&s
   …A™A
c                  óÂ  • [        [        R                  R                  SS5      5      n [        [        R                  R                  SS5      5      n[        [        R                  R                  SS5      5      n[	        SR                  U5      SR                  X 5      -   5        [        R                  R                  U5        [        R                  " S5        XU4$ )	NÚ
WORLD_SIZEr2   Ú
LOCAL_RANKr   ÚRANKz'Inference on multiple gpus, this gpu {}z, rank {}, world_size {}Únccl)ÚintÚosÚenvironÚgetÚprintÚformatÚtorchÚcudaÚ
set_deviceÚdistÚinit_process_group)Ú
world_sizeÚ
local_rankÚranks      r   Úinit_distributedrL   K   s¡   € Ü”R—Z‘Z—^‘^ L°!Ó4Ó5€JÜ”R—Z‘Z—^‘^ L°!Ó4Ó5€JÜŒr�z‰z�~‰~˜f aÓ(Ó)€DÜ	Ð
3×
:Ñ
:¸:Ó
FØ
$×
+Ñ
+¨DÓ
=ñ>ô ?ä	‡J�J×Ñ˜*Ô%Ü×Ò˜FÔ#Ø 4Ð'Ð'r!   c                  óŽ  • [         R                  " SS9n U R                  SS[        / SQSS9  U R                  SS[        S	S
9  U R                  SS[        SS/SS9  U R                  SS[        SS
9  U R                  SS[        SS
9  U R                  S[        SSS9  U R                  S[        SSS9  U R                  5       nU$ )Nzextract speech code)Údescriptionz--modelT)Úspeech_tokenizer_v1Úspeech_tokenizer_v1_25hzÚspeech_tokenizer_v2_25hzzmodel version)ÚrequiredÚtypeÚchoicesÚhelpz	--wav_scpz&each line contains `wav_name wav_path`)rR   rS   rU   z--devicerE   Úcpuzdevice for inferencez--output_dirzdir to save resultz--batch_sizez%batch size (per-device) for inferencez--num_workersé   zworkers for dataloader)rS   ÚdefaultrU   z
--prefetché   zprefetch for dataloader)ÚargparseÚArgumentParserÚadd_argumentÚstrr>   Ú
parse_args)ÚparserÚargss     r   Úget_argsra   V   s  € Ü×$Ò$Ð1FÑG€FØ
×Ñ˜	Ø!%Ü ò!ð -ð ñ .ð ×Ñ˜Ø!%Ü ØEð ñ Gð ×Ñ˜
Ø!%Ü Ø!'¨ Ø3ð	 ñ 5ð
 ×Ñ˜Ø!%Ü Ø1ð ñ 3ð ×Ñ˜Ø!%Ü ØDð ñ Fð ×Ñ˜Ü Ø !Ø5ð ñ 7ð ×Ñ˜Ü Ø !Ø6ð ñ 8ð ×ÑÓ€DØ€Kr!   c            
      ó\  • [        5       n [        R                  " U R                  SS9  U R                  S:X  a3  [
        R                  R                  5       (       d   e[        5       u  pnOSu  pn[
        R                  " U R                  5      n[        R                  " U R                  5      R                  U5      n[        U R                  5      nU R                  S:X  a4  [
        R                  R                   R#                  XR/S9n[%        UUUS9nOS n['        UU R(                  USU R*                  U R,                  [.        S9n[1        U5      n	US	:X  a  [3        U	S
SS9n
[5        U R                   SUS-    SU 3S5      nU H¼  u  pÍnU" UR                  U5      UR                  U5      5      u  nn[7        U5       HX  u  nnUUS UU   R9                  5       24   R;                  5       nUR=                  [>        R@                  " UUS.SS9S-   5        MZ     US	:X  d  MŸ  W
RC                  U[1        U5      -  5        M¾     US	:X  a  W
RE                  5         URE                  5         U R                  S:X  a+  [F        RH                  " 5         [F        RJ                  " 5         g g )NT)Úexist_okrE   )r2   r   r   )Ú
device_ids)Únum_replicasrK   F)Ú
batch_sizeÚsamplerÚshuffleÚnum_workersÚprefetch_factorr8   r   Ú
ProcessingÚwavs)ÚtotalÚdescÚunitz/part_r2   Ú_of_Úw)r   Úcode)Úensure_asciiÚ
)&ra   r?   ÚmakedirsÚ
output_dirÚdevicerD   rE   Úis_availablerL   r#   Ú
load_modelÚmodelÚtor   r   ÚnnÚparallelÚDistributedDataParallelr   r   rf   ri   Úprefetchr8   r   r   r   Ú	enumerater5   ÚtolistÚwriteÚjsonÚdumpsÚupdateÚcloserG   ÚbarrierÚdestroy_process_group)r`   rI   rJ   rK   rw   rz   Údatasetrg   Ú
dataloaderÚtotal_stepsÚprogress_barÚwriterr   r6   r7   ÚcodesÚ
codes_lensÚiÚkrr   s                       r   Úmainr’   }   sU  € Ü‹:€DÜ‡K‚K�—‘¨$Ò/à‡{�{�fÓÜ—
‘
×'Ñ'×)Ñ)Ð*Ð)Ü'7Ó'9Ñ$ˆ
¡à'.Ñ$ˆ
 ä�\Š\˜$Ÿ+™+Ó&€FÜ×"Ò" 4§:¡:Ó.×1Ñ1°&Ó9€EÜ˜4Ÿ<™<Ó(€Gà‡{�{�fÓÜ—‘×!Ñ!×9Ñ9Ø˜lð :ð ,ˆä$ WØ2<Ø*.ñ0‰ð ˆä˜GØ'+§¡Ø$+Ø$)Ø(,×(8Ñ(8Ø,0¯M©MÜ'1ñ3€Jô �g“,€KàˆqƒyÜ +°LÀvÑNˆä�T—_‘_Ð% V¨D°1©H¨:°T¸*¸ÐFÈÓL€FÛ!+Ñˆ�IÙ! $§'¡'¨&£/°9·<±<ÀÓ3GÓHÑˆˆzÜ˜d–O‰DˆAˆqØ˜Ð1˜Z¨™]×/Ñ/Ó1Ð1Ð1Ñ2×9Ñ9Ó;ˆDØ�L‰LÜ—
’
ØØ ñð !&ñ'ð *.ñ.ö/ñ $ð �1�9Ø×Ñ 
¬S°«YÑ 6Ö7ñ ",ð ˆqƒyØ×ÑÔØ
‡L�L„NØ‡{�{�fÓÜ�ŠŒÜ×"Ò"Õ$ð r!   Ú__main__)Ú__doc__rZ   rƒ   r?   rD   Útorch.distributedÚdistributedrG   Útorch.utils.datar   r   r   r   r#   r   r8   rL   ra   r’   r+   r0   r!   r   Ú<module>r˜      s]   ðñó& Û Û 	ã Ý  ß DÑ DÝ ã ô�7ô ò.!ò(ò$òN6%ðr ˆzÓÙ…Fð r!   