ó
    ‡Ž¡jUE  ã                   óì  • S SK r S SKJr  S SKJrJr  S SKJr  S SKrSSK	J
r
Jr  SSKJr  \
" 5       (       a  S SKrSr\R                   " S	5      r\R$                  (       aJ  \R&                  " 5       r\R+                  \R,                  " S
5      5        \R/                  \5        S\l        S\\R2                  \\\4   4S jr " S S\5      r\ " S S5      5       r\ " S S5      5       r " S S5      rg)é    N)Údeepcopy)Ú	dataclassÚfield)ÚIntEnumé   )Úis_psutil_availableÚis_torch_xpu_available)ÚloggingéÿÿÿÿÚContinuousBatchingLoggerz4%(asctime)s - %(name)s - %(levelname)s - %(message)sFÚreturnc                  óÈ  • [         R                  R                  5       (       a˜  [         R                  " S5      n [         R                  R	                  5         [         R                  R                  5         [         R                  R                  U 5      u  p[         R                  R                  U 5      nX!-
  nGO$[        5       (       a»  [         R                  " S5      n [         R                  R	                  5         [         R                  R                  5         [         R                  R                  U 5      R                  n[         R                  R                  U 5      n[         R                  R                  U 5      nGOZ[         R                  R                  R                  5       (       až  [         R                  R                  R                  5       (       aq  [         R                  " S5      n [         R                  R!                  5       n[         R                  R#                  5       n[         R                  R%                  5       nO�[         R                  " S5      n ['        5       (       aO  [(        R*                  " 5       R,                  n[(        R.                  " 5       R1                  5       R2                  nUnO[4        R7                  S5        SnSnSnXX44$ )NÚcudaÚxpuÚmpsÚcpuzŽCannot get memory breakdown on CPU without psutil: returning 0 for all memory values. Please install psutil to get an actual memory breakdown.r   )Útorchr   Úis_availableÚdeviceÚempty_cacheÚsynchronizeÚmem_get_infoÚmemory_reservedr	   r   Úget_device_propertiesÚtotal_memoryÚmemory_allocatedÚbackendsr   Úis_builtÚrecommended_max_memoryÚcurrent_allocated_memoryÚdriver_allocated_memoryr   ÚpsutilÚvirtual_memoryÚtotalÚProcessÚmemory_infoÚrssÚloggerÚerror)r   Úfree_memoryr   Úreserved_memoryÚallocated_memorys        ÚÂC:\Users\johnn\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.13_qbz5n2kfra8p0\LocalCache\local-packages\Python313\site-packages\transformers/generation/continuous_batching/requests.pyÚget_device_and_memory_breakdownr.   +   sã  € Ü‡z�z×Ñ× Ñ Ü—’˜fÓ%ˆÜ�
‰
×ÑÔ Ü�
‰
×ÑÔ ô %*§J¡J×$;Ñ$;¸FÓ$CÑ!ˆÜŸ*™*×4Ñ4°VÓ<ˆØ'Ñ5ÒÜ	×	!Ñ	!Ü—’˜eÓ$ˆÜ�	‰	×ÑÔÜ�	‰	×ÑÔÜ—y‘y×6Ñ6°vÓ>×KÑKˆÜŸ)™)×3Ñ3°FÓ;ˆÜ Ÿ9™9×5Ñ5°fÓ=ÒÜ	�‰×	Ñ	×	(Ñ	(×	*Ñ	*¬u¯~©~×/AÑ/A×/JÑ/J×/LÑ/LÜ—’˜eÓ$ˆÜ—y‘y×7Ñ7Ó9ˆÜ Ÿ9™9×=Ñ=Ó?ÐÜŸ)™)×;Ñ;Ó=‰ä—’˜eÓ$ˆÜ× Ñ Ü!×0Ò0Ó2×8Ñ8ˆLÜ%Ÿ~š~Ó/×;Ñ;Ó=×AÑAÐØ.‰Oä�L‰Lð<ôð ˆLØˆOØ Ðà ÐBÐBó    c                   ó,   • \ rS rSrSrSrSrSrSrSr	Sr
g	)
ÚRequestStatuséS   z5Status of a generation request through its lifecycle.r   é   é   r   é   © N)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__ÚPENDINGÚ
PREFILLINGÚDECODINGÚFINISHEDÚFAILEDÚ__static_attributes__r6   r/   r-   r1   r1   S   s   † Ù?à€GØ€JØ€HØ€HØƒFr/   r1   c                   ó  • \ rS rSr% Sr\\S'   \" \S9r	\\
   \S'   \" \S9r\\
   \S'   \" \S9r\\   \S'   Sr\S-  \S	'   \R                   r\\S
'   \" \R&                  S9r\\S'   Sr\\\4   \S'   Sr\\   S-  \S'   S\4S jrSrg)ÚGenerationOutputé]   aƒ  Tracks the output of a generation request.

Attributes:
    request_id (str): The ID of the generation request.
    prompt_ids (list[int]): The IDs of the prompt tokens.
    generated_tokens (list[int]): The generated tokens.
    logprobs (list[float]): The log probabilities of the generated tokens.
    error (Optional[str]): Any error message associated with the request. When None, the request was successful.
    status (RequestStatus): The status of the request.
    created_time (float): The time the request was created.
    lifespan (tuple[float, float]): The time the request was no longer pending and the time the request finished.
Ú
request_id©Údefault_factoryÚ
prompt_idsÚgenerated_tokensÚlogprobsNr)   ÚstatusÚcreated_time©r   r   ÚlifespanÚ
timestampsr   c                 ó<   • U R                   [        R                  :¬  $ )ziWhether the request reached a terminal state, either because it finished generating or because it failed.)rK   r1   r?   ©Úselfs    r-   Úis_finishedÚGenerationOutput.is_finishedv   s   € à�{‰{œm×4Ñ4Ñ4Ð4r/   r6   )r7   r8   r9   r:   r;   ÚstrÚ__annotations__r   ÚlistrH   ÚintrI   rJ   Úfloatr)   r1   r<   rK   ÚtimeÚperf_counterrL   rN   ÚtuplerO   ÚboolrS   rA   r6   r/   r-   rC   rC   ]   s­   ‡ ñð ƒOÙ!°$Ñ7€J��S‘	Ó7Ù"'¸Ñ"=Ð�d˜3‘iÓ=Ù!°$Ñ7€Hˆd�5‰kÓ7Ø€Eˆ3�‰:ÓØ)×1Ñ1€FˆMÓ1Ù°×0AÑ0AÑB€L�%ÓBØ$,€Hˆe�E˜5�LÑ!Ó,Ø%)€J��U‘˜dÑ"Ó)ð5˜T÷ 5r/   rC   c                   óð  • \ rS rSr% Sr\\S'   \\   \S'   Sr	\
\S'   Sr\
\S'   Sr\S	-  \S
'   S	r\\\   -  S	-  \S'   Sr\\S'   \" \S9r\\S'   \" \S9r\\   \S'   \" \S9r\\   \S'   \" \S9r\\   \S'   Sr\\S'   Sr\\S'   \R2                  r\\S'   \" \S9r\\   \S'   \" \R<                  S9r\\S'   S	r \S	-  \S'   Sr!\"\\4   \S'   \" \S9r#\\   \S'   Sr$\\S'   Sr%\\S'   \" \S9r&\\   \S'   Sr'\
\S '   S! r(\)S"\4S# j5       r*\*RV                  S$\4S% j5       r*\)S"\\   S	-  4S& j5       r,S' r-S"\4S( jr.S"\4S) jr/S*\S+\S	-  S"\
4S, jr0S- r1S. r2S/\S"S 4S0 jr3S"\4S1 jr4S4S2 jr5S3r6g	)5ÚRequestStateé{   aw  Tracks the state of a generation request through its lifecycle.

Attributes:
    request_id (str): The ID of the generation request.
    initial_tokens (list[int]): The initial prompt tokens.
    num_children (int): The number of children requests
    full_prompt_ids (list[int] | None): The tokens IDs of the full prompt.
    prompt_ids (list[int] | None): The tokens IDs currently being processed.
    remaining_prompt_ids (list[int]): The initial tokens IDs remaining to be processed.
    static_outputs (list[int]): The generated tokens.
    allocated_blocks (int): The number of blocks allocated to the request.
    position_offset (int): The current position in the sequence for position_ids.
    status (RequestStatus): The status of the request: can be one of PENDING, PREFILLING, PREFILLING_SPLIT,
                            SPLIT_PENDING_REMAINDER, DECODING, FINISHED, FAILED
    max_new_tokens (int | None): The maximum number of new tokens to generate.
    eos_token_id (None | int | list[int]): The ID(s) of the end-of-sequence tokens. Only used in post-init.
    _eos_token_ids (set[int]): The IDs of the end-of-sequence tokens, formatted as a set.
    streaming (bool): Whether to stream tokens as they're generated
    created_time (float): The time the request was created.
    error (Optional[str]): Any error message associated with the request. When None, has had no error yet.
rE   Úinitial_tokensFÚ	streamingÚrecord_timestampsé   NÚmax_new_tokensÚeos_token_idr   Únum_childrenrF   Úlogit_processor_kwargsÚtokens_to_processrI   rJ   Úposition_offsetÚallocated_blocksÚ_statusÚ_eos_token_idsrL   r)   rM   rN   Ú_timestampsÚ_true_initial_tokenséÿÿÿÚ_new_tokens_limitÚremaining_prefill_tokensÚis_cpu_offloadedc                 ó   • U R                   c  SOU R                   U l        U R                  S S  U l        U R                  c  g [        U R                  [        5      (       a7  U R                  S:¼  a&  U R                  R                  U R                  5        g g U R                   H&  nUS:¼  d  M  U R                  R                  U5        M(     g )Nrp   r   )	re   rq   ra   rr   rf   Ú
isinstancerX   rm   Úadd)rR   Útoken_ids     r-   Ú__post_init__ÚRequestState.__post_init__¸   s­   € à/3×/BÑ/BÑ/J¡ÐPT×PcÑPcˆÔà(,×(;Ñ(;¹AÐ(>ˆÔ%à×ÑÑ$Øä˜×)Ñ)¬3×/Ñ/Ø× Ñ  AÓ%Ø×#Ñ#×'Ñ'¨×(9Ñ(9Õ:ð &ð !×-Ô-�Ø˜q•=Ø×'Ñ'×+Ñ+¨HÖ5ò .r/   r   c                 ó   • U R                   $ ©N)rl   rQ   s    r-   rK   ÚRequestState.statusÊ   s   € à�|‰|Ðr/   Úvaluec                 ón  • U R                   [        R                  :X  a  [        R                  " 5       S4U l        U[        R                  :¼  aa  U R
                  S   [        R                  " 5       4U l        [        R                  [        R                  5      (       a  U R                  5         Xl         g )Nr   r   )rl   r1   r<   rZ   r[   rN   r?   r(   ÚisEnabledForr
   ÚDEBUGÚlog_end_of_request)rR   r}   s     r-   rK   r|   Î   s}   € ð �<‰<œ=×0Ñ0Ó0Ü!×.Ò.Ó0°"Ð5ˆDŒMà”M×*Ñ*Ó*Ø!Ÿ]™]¨1Ñ-¬t×/@Ò/@Ó/BÐCˆDŒMÜ×"Ñ"¤7§=¡=×1Ñ1Ø×'Ñ'Ô)Ø�r/   c                 ó@   • U R                   (       a  U R                  $ S $ r{   )rc   rn   rQ   s    r-   rO   ÚRequestState.timestampsÚ   s   € à#'×#9×#9ˆt×ÑÐC¸tÐCr/   c                 ó"  • [        U R                  5      nU R                  5       nU R                  S   U R                  -
  nU R                  S   U R                  -
  n[
        R                  SU R                   SU< SU< SU< SU< 3
5        g )Nr   r3   úRequest z finished: prefill_len = z decode_len = z start_time = z end_time = )Úlenra   Úgenerated_lenrN   rL   r(   ÚdebugrE   )rR   Úprefill_lenÚ
decode_lenÚ
start_timeÚend_times        r-   r�   ÚRequestState.log_end_of_requestÞ   s�   € Ü˜$×-Ñ-Ó.ˆØ×'Ñ'Ó)ˆ
Ø—]‘] 1Ñ%¨×(9Ñ(9Ñ9ˆ
Ø—=‘= Ñ# d×&7Ñ&7Ñ7ˆÜ�‰Ø�t—‘Ð'Ð'A°;Ñ2BÀ/ÀJÁ?ÐRaÐT^ÑSbÐboÐdlÑcpÐqõ	
r/   c                 ó   • U R                   $ )zCGet the current length of the sequence (prompt + generated tokens).)rj   rQ   s    r-   Úcurrent_lenÚRequestState.current_lenç   s   € à×#Ñ#Ð#r/   c                 ó,   • [        U R                  5      $ )z*Get the number of tokens generated so far.)r†   rI   rQ   s    r-   r‡   ÚRequestState.generated_lenë   s   € ä�4×(Ñ(Ó)Ð)r/   rw   Úlogprobc                 óX  • U R                   [        R                  :w  a  gU R                  (       a.  U R                  R                  [        R                  " 5       5        XR                  ;   nU R                  5       nU(       d  X@R                  :  aG  U R                  R                  U5        U/U l        US-  nUb  U R                  R                  U5        O%[        R                  SU R                    SU 35        U(       d  X@R                  :¼  a  [        R"                  U l         gg)z³Update the request with a newly generated token (and optional log probability of the token) and check for
completion. Returns True if the request is now complete, False otherwise.Fr3   r…   z generated a useless token: T)rK   r1   r>   rc   rn   ÚappendrZ   r[   rm   r‡   rq   rI   ri   rJ   r(   ÚwarningrE   r?   )rR   rw   r“   Úis_eosr�   s        r-   Úupdate_and_check_completionÚ(RequestState.update_and_check_completionð   sò   € ð �;‰;œ-×0Ñ0Ó0Øð ×!×!Ø×Ñ×#Ñ#¤D×$5Ò$5Ó$7Ô8ð ×0Ñ0Ñ0ˆØ×(Ñ(Ó*ˆö �k×$:Ñ$:Ó:Ø×!Ñ!×(Ñ(¨Ô2Ø&. ZˆDÔ"Ø˜1ÑˆKØÑ"Ø—‘×$Ñ$ WÔ-øä�N‰N˜X d§o¡oÐ%6Ð6RÐS[ÐR\Ð]Ô^æ�[×$:Ñ$:Ó:Ü'×0Ñ0ˆDŒKØØr/   c                 óŠ  • SU R                    3SU R                   3SU R                  5        3S[        U R                  5       3S[        U R
                  5       3SU R                   3S[        U R                  5       3SU R                   3S	U R                   3S
U R                   3/
nSSR                  U5      -   S-   $ )Nzrequest_id=zstatus=zout_tokens=zquery_length=zremaining_tokens=z
kv_length=zfull_prompt_length=zallocated_blocks=zgenerated_tokens=zlogit_processor_kwargs=zRequestState(
	z,
	z
))rE   rl   r‡   r†   ri   rr   rj   ra   rk   rI   rh   Újoin)rR   Úmsgs     r-   Ú__repr__ÚRequestState.__repr__  sÛ   € à˜$Ÿ/™/Ð*Ð+Ø�d—l‘l�^Ð$Ø˜$×,Ñ,Ó.Ð/Ð0ØœC × 6Ñ 6Ó7Ð8Ð9Ø¤ D×$AÑ$AÓ BÐCÐDØ˜×-Ñ-Ð.Ð/Ø!¤# d×&9Ñ&9Ó":Ð!;Ð<Ø × 5Ñ 5Ð6Ð7Ø × 5Ñ 5Ð6Ð7Ø% d×&AÑ&AÐ%BÐCð
ˆð # W§\¡\°#Ó%6Ñ6¸Ñ>Ð>r/   c                 ó¶  • U R                   (       a@  U R                  U R                   S U R                  -   nU R                  SU R                    nOU R                  SS nU R                  n[        U R                  UUU R
                  SS U R                  U R                  U R                  U R                  U R                  b  U R                  SS S9	$ SS9	$ )z7Convert the request state to a GenerationOutput object.N)	rE   rH   rI   rJ   r)   rK   rL   rN   rO   )ro   ra   rI   rC   rE   rJ   r)   rK   rL   rN   rO   )rR   rI   rH   s      r-   Úto_generation_outputÚ!RequestState.to_generation_output  sÐ   € à×$×$Ø#×2Ñ2°4×3LÑ3LÐ3NÐOÐRV×RgÑRgÑgÐØ×,Ñ,Ð-H¨t×/HÑ/HÐI‰Jà#×4Ñ4±QÐ7ÐØ×,Ñ,ˆJÜØ—‘Ø!Ø-Ø—]‘]¡1Ð%Ø—*‘*Ø—;‘;Ø×*Ñ*Ø—]‘]Ø-1¯_©_Ñ-H�t—‘¡qÐ)ñ

ð 
	
ð OSñ

ð 
	
r/   Únew_request_idc                 ó¸   • [        U 5      nXl        [        R                  " 5       Ul        UR                  S4Ul        / Ul        U R                  SS Ul        U$ )ziFork the request into a new request with the same state except for request_id, created_time and lifespan.r   N)r   rE   rZ   r[   rL   rN   rn   rr   )rR   r¢   Únew_requests      r-   ÚforkÚRequestState.fork2  sV   € ä˜t“nˆà!/ÔÜ#'×#4Ò#4Ó#6ˆÔ Ø +× 8Ñ 8¸"Ð=ˆÔØ"$ˆÔà/3×/LÑ/LÉQÐ/OˆÔ,ØÐr/   c                 óž   • U R                   U R                  U R                  U R                  U R                  [        U R                  5      S.$ )zXGet all the fields necessary to create a request that would have the same configuration.)rb   rc   re   rf   rg   rh   )rb   rc   re   rf   rg   r   rh   rQ   s    r-   Úget_request_configÚRequestState.get_request_config>  sG   € ð Ÿ™Ø!%×!7Ñ!7Ø"×1Ñ1Ø ×-Ñ-Ø ×-Ñ-Ü&.¨t×/JÑ/JÓ&Kñ
ð 	
r/   c           
      óÂ  • U R                  5       nU R                  b%  U R                  [        U R                  5      -
  US'   [	        SU R
                  U R                  U R                  -   U R                  SS U R                  [        U R                  5      -   S.UD6nU R                  (       a  U R                  Ul        U$ [        U R                  5      Ul        U$ )aD  Creates an equivalent new request by removing the generated tokens and adding them to the initial prompt. The
created request has THE SAME request_id. Notably, we can retrieve the original request from the created one with
the _true_initial_tokens attribute. The logprobs of the generated tokens are kept in the new request.Nre   )rE   ra   rJ   ro   r6   )	r¨   re   r†   rI   r_   rE   ra   rJ   ro   )rR   Úrequest_configÚ	new_states      r-   Ú!create_equivalent_initial_requestÚ.RequestState.create_equivalent_initial_requestI  sÔ   € ð
 ×0Ñ0Ó2ˆà×ÑÑ*Ø/3×/BÑ/BÄSÈ×I^ÑI^ÓE_Ñ/_ˆNÐ+Ñ,ä ð 
Ø—‘Ø×.Ñ.°×1FÑ1FÑFØ—]‘]¡1Ð%Ø!%×!:Ñ!:¼SÀ×ATÑATÓ=UÑ!Uñ	
ð
 ñ
ˆ	ð ×$×$Ø-1×-FÑ-FˆIÔ*ð Ðô .1°×1DÑ1DÓ-EˆIÔ*ØÐr/   )rq   rl   rN   rr   rK   ri   )r   r_   )7r7   r8   r9   r:   r;   rU   rV   rW   rX   rb   r]   rc   re   rf   rg   r   Údictrh   ri   rI   rJ   rY   rj   rk   r1   r<   rl   Úsetrm   rZ   r[   rL   r)   rN   r\   rn   ro   rq   rr   rs   rx   ÚpropertyrK   ÚsetterrO   r�   r�   r‡   r˜   r�   r    r¥   r¨   r­   rA   r6   r/   r-   r_   r_   {   s7  ‡ ñð. ƒOØ˜‘IÓð €IˆtÓØ#Ð�tÓ#ð "$€N�C˜$‘JÓ#Ø+/€L�#˜˜S™	‘/ DÑ(Ó/Ø€L�#ÓÙ#(¸Ñ#>Ð˜DÓ>ñ $)¸Ñ#>Ð�t˜C‘yÓ>Ù"'¸Ñ"=Ð�d˜3‘iÓ=Ù!°$Ñ7€Hˆd�5‰kÓ7Ø€O�SÓØÐ�cÓà*×2Ñ2€Gˆ]Ó2Ù$°SÑ9€N�C˜‘HÓ9ñ  °×0AÑ0AÑB€L�%ÓBØ€Eˆ3�‰:ÓØ$,€Hˆe�E˜5�LÑ!Ó,Ù$°TÑ:€K��e‘Ó:Ø !Ð˜#Ó!ð (Ð�sÓ'Ù*/ÀÑ*EÐ˜d 3™iÓEØ"Ð�dÓ"ò6ð$ ð˜ó ó ðð ‡]�]ð	˜Mó 	ó ð	ð ðD˜D ™K¨$Ñ.ó Dó ðDò
ð$˜Sô $ð*˜sô *ð
°Cð À%È$Á,ð ÐSWô ò>?ò
ð(
 3ð 
¨>ô 
ð	
 Dô 	
÷r/   r_   c            
       ó:   • \ rS rSrSrSrS\S\S\S\SS	4
S
 jr	Sr
g	)ÚFutureRequestStateic  zPTracks the current state of a request and the relevant information to update it.©ÚstateÚhas_new_tokenÚcomplete_blocksÚquery_lengthr¶   r·   r¸   r¹   r   Nc                 ó4   • Xl         X l        X0l        X@l        g r{   rµ   )rR   r¶   r·   r¸   r¹   s        r-   Ú__init__ÚFutureRequestState.__init__i  s   € ØŒ
Ø*ÔØ.ÔØ(Õr/   )r¸   r·   r¹   r¶   )r7   r8   r9   r:   r;   Ú	__slots__r_   r]   rX   r»   rA   r6   r/   r-   r´   r´   c  s8   † ÙZð N€Ið)˜lð )¸4ð )ÐRUð )Ðehð )Ðmq÷ )r/   r´   ) rZ   Úcopyr   Údataclassesr   r   Úenumr   r   Úutilsr   r	   Úutils.loggingr
   r"   ÚTMP_TOKEN_IDÚ	getLoggerr(   Ú	propagateÚStreamHandlerÚhandlerÚsetFormatterÚ	FormatterÚ
addHandlerr\   r   rX   r.   r1   rC   r_   r´   r6   r/   r-   Ú<module>rË      sø   ðó Ý ß (Ý ã ç @Ý $ñ ×ÑÛð €ð 
×	Ò	Ð5Ó	6€à	××Ø×#Ò#Ó%€GØ×Ñ˜×*Ò*Ð+aÓbÔcØ
×Ñ�gÔØ€FÔð%C¨¨u¯|©|¸SÀ#ÀsÐ/JÑ)Kô %CôP�Gô ð ÷5ð 5ó ð5ð: ÷dð dó ðd÷N
)ò 
)r/   