ó
    ‡Ž¡jï%  ã                   óÒ   • S r SSKrSSKrSSKrSSKrSSKJr  SSKrSSKJ	r	  SSK
Jr  SSKJr  \	R                  " \5      r " S S	\\R$                  5      r " S
 S5      rS\l         g)z+
CLI entry point for `transformers serve`.
é    N)Ú	Annotated)Úlogging)Úis_serve_availableé   )Úset_torch_seedc                   ó    • \ rS rSrSrSrSrSrg)ÚReasoningModeé#   ÚonÚoffÚauto© N)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__ÚONÚOFFÚAUTOÚ__static_attributes__r   ó    Ú¤C:\Users\johnn\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.13_qbz5n2kfra8p0\LocalCache\local-packages\Python313\site-packages\transformers/cli/serve.pyr	   r	   #   s   † Ø	€BØ
€CØƒDr   r	   c            3       ól  • \ rS rSrSSSSS\R
                  R                  SSSSSSSSSSSSSSSS4S	\\S-  \	R                  " S
S94   S\\\	R                  " SS94   S\\S-  \	R                  " SS94   S\\\	R                  " SS94   S\\S-  \	R                  " SS94   S\\\	R                  " SS94   S\\S-  \	R                  " SS94   S\\\	R                  " SS94   S\\S-  \	R                  " SS94   S\\\	R                  " SS94   S\\\	R                  " SS94   S \\S-  \	R                  " S!S94   S"\\S-  \	R                  " S#S94   S$\\S-  \	R                  " S%S94   S&\\S-  \	R                  " S'S94   S(\\S-  \	R                  " S)S94   S*\\\	R                  " S+S94   S,\\\	R                  " S-S94   S.\\\	R                  " S/S94   S0\\\	R                  " S1S94   S2\\S-  \	R                  " S3S94   S4\\\	R                  " S5S6S794   S8S4.S9 jjrS: rS; rS< rS=rg)>ÚServeé)   NFr   i,  Ú	localhosti@  ÚwarningÚforce_modelz‘Model to preload and use for all requests. GGUF weights are named `<repo>:<file>.gguf`, and their config and tokenizer come from the file itself.)ÚhelpÚcontinuous_batchingzMEnable continuous batching with paged attention. Configure with --cb-* flags.Úattn_implementationz2Attention implementation (e.g. flash_attention_2).Úcompilez*Enable torch.compile for faster inference.Úquantizationz.Quantization method: 'bnb-4bit' or 'bnb-8bit'.Ú	reasoninguÀ   Reasoning mode. 'auto' uses the chat template default. Only applies to models that support reasoning via their chat template (e.g. Qwen3, Gemma 4) â€” for other models this flag has no effect.Úchat_template_kwargsz‰Default JSON kwargs forwarded to apply_chat_template (e.g. '{"enable_thinking": true}'); per-request chat_template_kwargs override these.Údevicez4Device for inference (e.g. 'auto', 'cuda:0', 'cpu').Údtypez2Override model dtype. 'auto' derives from weights.Útrust_remote_codezTrust remote code when loading.Úmodel_timeoutzGSeconds before idle model is unloaded. Ignored when force_model is set.Úcb_block_sizez6KV cache block size in tokens for continuous batching.Úcb_num_blocksz2Number of KV cache blocks for continuous batching.Úcb_max_batch_tokensz1Maximum tokens per batch for continuous batching.Úcb_max_memory_percentz/Max GPU memory fraction for KV cache (0.0-1.0).Úcb_use_cuda_graphz+Enable CUDA graphs for continuous batching.ÚhostzServer listen address.ÚportzServer listen port.Úenable_corszEnable permissive CORS.Ú	log_levelz'Logging level (e.g. 'info', 'warning').Údefault_seedzDefault torch seed.Únon_blockingTz1Run server in a background thread. Used by tests.)Úhiddenr   Úreturnc           
      ó\  • [        5       (       d  [        S5      eSS KnSSKJn  SSKJn  SSKJn  SSK	J
n  SSKJn  SS	KJn  SS
KJn  Ub  [#        U5        [$        R&                  " S5      nUR)                  [$        R*                  UR-                  5          5        U" UU	U
UUUUS9U l        SSKJn   UUUUUS.R5                  5        V!V"s0 s H  u  n!n"U"c  M  U!U"_M     n#n!n"U#(       a  U " S0 U#D6OS n$U" UUU$S9U l        U(       aB  [8        R:                  " U5      n[=        U[>        5      (       d  [@        RB                  " S5      eO0 nU[D        RF                  :X  a  SUS'   OU[D        RH                  :X  a  SUS'   U" U R.                  U R6                  US9U l%        U" U R.                  U R6                  S9U l&        U" U R.                  U R6                  US9U l'        U" U R.                  U R6                  5      U l(        U" U R.                  U RJ                  U RL                  U RN                  U RP                  U R6                  US9n%URS                  U%UUSS9n&URU                  U&5      U l+        U(       a  U RY                  5         g U RV                  R[                  5         g s  sn"n!f )NzRMissing dependencies for serving. Install with `pip install transformers[serving]`r   r   )ÚChatCompletionHandler)ÚCompletionHandler)ÚModelManager)ÚResponseHandler)Úbuild_server)ÚTranscriptionHandler)ÚGenerationStateÚtransformers)r&   r'   r(   r!   r#   r)   r   )ÚContinuousBatchingConfig)Ú
block_sizeÚ
num_blocksÚmax_batch_tokensÚmax_memory_percentÚuse_cuda_graph)r    r"   Ú	cb_configz,--chat-template-kwargs must be a JSON objectTÚenable_thinkingF)Úmodel_managerÚgeneration_stater%   )rH   rI   )Úcompletion_handlerÚresponse_handlerÚtranscription_handlerrI   r1   Úinfo)r/   r0   r2   r   ).r   ÚImportErrorÚuvicornÚserving.chat_completionr8   Úserving.completionr9   Úserving.model_managerr:   Úserving.responser;   Úserving.serverr<   Úserving.transcriptionr=   Úserving.utilsr>   r   r   Ú
get_loggerÚsetLevelÚ
log_levelsÚlowerÚ_model_managerr?   r@   ÚitemsÚ_generation_stateÚjsonÚloadsÚ
isinstanceÚdictÚtyperÚBadParameterr	   r   r   Ú_chat_handlerÚ_completion_handlerÚ_response_handlerÚ_transcription_handlerÚConfigÚServerÚserverÚstart_serverÚrun)'Úselfr   r    r!   r"   r#   r$   r%   r&   r'   r(   r)   r*   r+   r,   r-   r.   r/   r0   r1   r2   r3   r4   rO   r8   r9   r:   r;   r<   r=   r>   Útransformers_loggerr@   ÚkÚvÚ	cb_kwargsrF   ÚappÚconfigs'                                          r   Ú__init__ÚServe.__init__*   s¤  € ôT "×#Ñ#ÜÐrÓsÐsãåBÝ9Ý7Ý5Ý0Ý?Ý2ð Ñ#Ü˜<Ô(ô &×0Ò0°Ó@ÐØ×$Ñ$¤W×%7Ñ%7¸	¿¹Ó8IÑ%JÔKá*ØØØ/Ø 3Ø%Ø'Ø#ñ
ˆÔõ 	:ð
 ,Ø+Ø$7Ø&;Ø"3ñ÷ ‰e‹gðô

ò‘��1ð ó ˆAˆqŠDñð 	ñ 

ö >GÑ,Ñ9¨yÒ9ÈDˆ	Ù!0Ø 3ØØñ"
ˆÔö  Ü#'§:¢:Ð.BÓ#CÐ ÜÐ2´D×9Ñ9Ü×(Ò(Ð)WÓXÐXð :ð $&Ð àœ×(Ñ(Ó(Ø6:Ð Ð!2Ò3Øœ-×+Ñ+Ó+Ø6;Ð Ð!2Ñ3á2Ø×-Ñ-Ø!×3Ñ3Ø!5ñ
ˆÔñ $5Ø×-Ñ-Ø!×3Ñ3ñ$
ˆÔ ñ
 "1Ø×-Ñ-Ø!×3Ñ3Ø!5ñ"
ˆÔñ ';¸4×;NÑ;NÐPT×PfÑPfÓ&gˆÔ#áØ×ÑØ×ÑØ#×7Ñ7Ø!×3Ñ3Ø"&×"=Ñ"=Ø!×3Ñ3Ø#ñ
ˆð —‘ ¨$°TÀV�ÐLˆØ—n‘n VÓ,ˆŒæØ×ÑÕà�K‰K�O‰OÕùóC

s   Ã
J(ÃJ(c                 ó|   ^ • U 4S jn[         R                  " USSS9T l        T R                  R                  5         g )Nc                  ó®   >• [         R                  " 5       n [         R                  " U 5        U R                  TR                  R                  5       5        g )N)ÚasyncioÚnew_event_loopÚset_event_loopÚrun_until_completerj   Úserve)Úlooprm   s    €r   Ú_runÚ Serve.start_server.<locals>._runØ   s:   ø€ Ü×)Ò)Ó+ˆDÜ×"Ò" 4Ô(Ø×#Ñ# D§K¡K×$5Ñ$5Ó$7Õ8r   zuvicorn-threadF)ÚtargetÚnameÚdaemon)Ú	threadingÚThreadÚ_threadÚstart)rm   r~   s   ` r   rk   ÚServe.start_server×   s2   ø€ õ	9ô
 !×'Ò'¨tÐ:JÐSXÑYˆŒØ�‰×ÑÕr   c                 ó8   • U R                   R                  5         g)z$Clear all loaded models from memory.N)r[   Úshutdown©rm   s    r   Úreset_loaded_modelsÚServe.reset_loaded_modelsà   s   € à×Ñ×$Ñ$Õ&r   c                 ó"  • U R                   R                  5         U R                  R                  5         U R                  (       a  U R                  R	                  5       (       d  g SU R
                  l        U R                  R                  SS9  g )NTé   )Útimeout)r]   r‰   r[   r…   Úis_aliverj   Úshould_exitÚjoinrŠ   s    r   Úkill_serverÚServe.kill_serverä   sc   € Ø×Ñ×'Ñ'Ô)Ø×Ñ×$Ñ$Ô&Ø�|�| 4§<¡<×#8Ñ#8×#:Ñ#:ØØ"&ˆ�‰ÔØ�‰×Ñ !ÐÒ$r   )rd   re   r]   r[   rf   r…   rg   rj   )r   r   r   r   r	   r   Úvaluer   Ústrrb   ÚArgumentÚboolÚOptionÚintÚfloatrt   rk   r‹   r“   r   r   r   r   r   r   )   sÆ  † ð ð
 ð Ødið ð ×Ñ×$Ñ$ð ØlrØpvØchð ð ð ð ð ð àLWØIMØUZØbkØX\ð ñQkàØ�$‰JØ�NŠNðPñðñ
ðkð 'ØØ�LŠLÐmÑnðpñ
ðkð  'Ø�$‰J˜ŸšÐ*^Ñ_Ð_ñ
ð!kð& ˜4 §¢Ð3_Ñ!`Ð`Ñað'kð(  Ø�$‰J˜ŸšÐ*ZÑ[Ð[ñ
ð)kð. ØØ�LŠLð/ñðñ	
ð/kðB (Ø�$‰JØ�LŠLðmñðñ
ðCkðT ˜#˜uŸ|š|Ð1gÑhÐhÑiðUkðV ˜˜t™ U§\¢\Ð7kÑ%lÐlÑmðWkðX % T¨5¯<ª<Ð=^Ñ+_Ð%_Ñ`ðYkðZ !Ø�—’Ð#lÑmÐmñ
ð[kðb !Ø�$‰J˜ŸšÐ*bÑcÐcñ
ðckðh !Ø�$‰J˜ŸšÐ*^Ñ_Ð_ñ
ðikðn 'Ø�$‰J˜ŸšÐ*]Ñ^Ð^ñ
ðokðt  )Ø�D‰L˜%Ÿ,š,Ð,]Ñ^Ð^ñ 
ðukðz %Ø�4‰K˜ŸšÐ+XÑYÐYñ
ð{kðB ˜˜UŸ\š\Ð/GÑHÐHÑIðCkðD ˜˜UŸ\š\Ð/DÑEÐEÑFðEkðF ˜t U§\¢\Ð7PÑ%QÐQÑRðGkðH ˜S %§,¢,Ð4]Ñ"^Ð^Ñ_ðIkðJ    d¡
¨E¯LªLÐ>SÑ,TÐ TÑUðKkðL  Ø�%—,’, dÐ1dÑeÐeñ
ðMkðR 
õSkòZò'õ%r   r   uð  
Run a FastAPI server to serve models on-demand with an OpenAI compatible API.
Models will be loaded and unloaded automatically based on usage and a timeout.


Endpoints:
    POST /v1/chat/completions â€” Chat completions (streaming + non-streaming).
    POST /v1/completions      â€” Legacy text completions from a prompt.
    GET  /v1/models           â€” Lists available models.
    GET  /health              â€” Health check.

Requires FastAPI and Uvicorn: pip install transformers[serving]
)Ú__doc__rx   Úenumr^   rƒ   Útypingr   rb   Útransformers.utilsr   Útransformers.utils.import_utilsr   rV   r   rW   r   Úloggerr–   ÚEnumr	   r   r   r   r   Ú<module>r£      sa   ðñó Û Û Û Ý ã å &Ý >å )ð 
×	Ò	˜HÓ	%€ô�C˜Ÿ™ô ÷A%ñ A%ðH€…r   