ó
    æ“,jb  ã                   óh   • S r SSKJr  SSKrSSKrSSKJr  SSKJ	r	J
r
   " S S\5      rS	 rS
 rS rg)zV
Learnt-Stepsize quantizer from [Esser et al. 2019] https://arxiv.org/abs/1902.08153.
é    )Ú	dataclassNé   )ÚBaseQuantizer)Úcapture_initÚsimple_reprc                   óÖ  ^ • \ rS rSrSr\ " S S\R                  5      5       r\SSSS/ S	4S
\	R                  R                  S\S\S\S\4
U 4S jjj5       rS rS\	R$                  R&                  4S jrS\	R$                  R&                  4S jrS rS#U 4S jjrS rS rS rS\S\R8                  4S jrS\S\R8                  S\	R<                  4S jrS r S r!U 4S  jr"S! r#S"r$U =r%$ )$ÚLSQé   zcImplements weight only quantization based on [Esser et al. 2019].
https://arxiv.org/abs/1902.08153
c                   óH   • \ rS rSr% \R
                  R                  \S'   Srg)ÚLSQ._QuantizedParamé   Úscale© N)	Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__ÚtorchÚnnÚ	ParameterÚ__annotations__Ú__static_attributes__r   ó    Ú—C:\Users\johnn\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.13_qbz5n2kfra8p0\LocalCache\local-packages\Python313\site-packages\diffq/lsq.pyÚ_QuantizedParamr      s   ‡ à�x‰x×!Ñ!Ö!r   r   é   g{®Gáz„?FÚ_lsqTÚmodelÚbitsÚmin_sizeÚfloat16Úsuffixc                 óæ   >• SUs=:  a  S::  d   e   eXPl         SU l        X l        UR                  5        H&  u  p‰UR	                  U5      (       d  M  [        S5      e   [        T
U ]  XXFU5        g )Nr   é   FzbThe model already has some noise scales parameters, maybe you used twice a LSQ on the same model?.)r"   Ú_optimizer_setupr   Únamed_parametersÚendswithÚRuntimeErrorÚsuperÚ__init__)Úselfr   r   r    r!   r"   ÚexcludeÚdetect_boundÚnameÚ_Ú	__class__s             €r   r*   ÚLSQ.__init__   sx   ø€ ð �4�~˜2‹~Ð‰~Ðˆ~ØŒØ %ˆÔØŒ	à×-Ñ-Ö/‰GˆDØ�}‰}˜V×$Ó$Ü"ð $Tó Uð Uñ 0ô
 	‰Ñ˜¨'¸LÕIr   c                 ó„  • Ub%  U R                   R                  XX4R                  US9$ SUR                  R	                  5       R                  5       -  SU R                  S-
  -  S-  -  n[        R                  R                  U5      nUR                  XR                  -   U5        U R                   R                  XX5S S9$ )N)r.   ÚparamÚmoduler   Úotheré   r   ç      à?)r0   r   r   ÚdataÚabsÚmeanr   r   r   r   Úregister_parameterr"   )r+   r.   r3   r4   r5   r   s         r   Ú_register_paramÚLSQ._register_param*   s¸   € ØÑØ—>‘>×1Ñ1Ø¨f¿K¹KÈuð 2ð Vð Vð �E—J‘J—N‘NÓ$×)Ñ)Ó+Ñ+¨q°T·Y±YÀ±]Ñ/CÀcÑ.IÑIˆÜ—‘×"Ñ" 5Ó)ˆØ×!Ñ! $¯©Ñ"4°eÔ<Ø�~‰~×-Ñ-Ø¨&ÀTð .ð Kð 	Kr   Ú	optimizerc                 ó  • U R                    Vs/ s H  o"R                  PM     nnUR                   HN  n/ n[        US   5       H0  nSnU H  nX†L d  M	  SnM     U(       a  M  UR	                  U5        M2     XTS   S S & MP     g s  snf )NÚparamsFT)Ú_qparamsr   Úparam_groupsÚlistÚappend)	r+   r>   Úqpr@   ÚgroupÚ
new_paramsÚqÚmatchedÚps	            r   Úclear_optimizerÚLSQ.clear_optimizer5   s…   € Ø%)§]¢]Ó3¢]˜r—(”(¡]ˆÐ3à×+Ô+ˆEØˆJÜ˜% ™/Ö*�Ø�Û�AØ”vØ"&šñ  ÷ �wØ×%Ñ% aÖ(ñ +ð ",�(‰O™AÒò ,ùò 4s   �Bc                 óX  • U R                   (       a   eSU l         U R                   Vs/ s H  o3R                  PM     nnUR                   H0  n[	        US   5       H  nU H  nXvL d  M	  [        S5      e   M     M2     SU0nUR                  U5        UR                  U5        gs  snf )a  
Setup the optimizer to tune the scale parameter.
Following [Esser et al. 2019], we use the same LR and weight decay
as the base optimizer, unless specified otherwise.

Args:
    optimizer (torch.Optimizer): optimizer to use.
    kwargs (dict): overrides for optimization parameters
Tr@   z5You should create the optimizer before the quantizer!N)r%   rA   r   rB   rC   r(   ÚupdateÚadd_param_group)r+   r>   ÚkwargsrE   r@   rF   rH   rJ   s           r   Úsetup_optimizerÚLSQ.setup_optimizerC   s§   € ð ×(×(Ð(Ð(Ø $ˆÔà%)§]¢]Ó3¢]˜r—(”(¡]ˆÐ3à×+Ô+ˆEÜ˜% ™/Ö*�Û�AØ”vÜ*ð ,Có Dð Dó  ó +ñ ,ð ˜6Ð"ˆØ�‰�VÔØ×!Ñ! %Õ(ùò 4s   ©B'c                 ó   • SU l         g)z3
Call this if you do not want to use an optimizer.
TN)r%   ©r+   s    r   Úno_optimizerÚLSQ.no_optimizer]   s   € ð !%ˆÕr   c                 óÚ   >• [         TU ]  5       nSnU R                   HB  nUR                  b  M  UR                  R                  5       U R                  -  nX5-  nUS-  nMD     US-  nX#-   $ )zù
Differentiable estimate of the model size.
The size is returned in MB.

If `exact` is True, then the output is no longer differentiable but
reflect exactly an achievable size, even without compression,
i.e.same as returned by `naive_model_size()`.
r   é    i  € )r)   Ú
model_sizerA   r5   r3   Únumelr   )r+   ÚexactÚtotalÚsubtotalÚqparamr   r0   s         €r   rY   ÚLSQ.model_sizec   st   ø€ ô ‘Ñ"Ó$ˆØˆØ—m”mˆFà�|‰|Ñ'ÙØ—<‘<×%Ñ%Ó'¨$¯)©)Ñ3ˆDØÑˆHØ˜ÑŠHñ $ð 	�KÑˆØÑÐr   c                 ó<   • U R                  SS9R                  5       $ )z,
Naive model size without zlib compression.
T)r[   )rY   ÚitemrT   s    r   Útrue_model_sizeÚLSQ.true_model_sizey   s   € ð �‰ TˆÐ*×/Ñ/Ó1Ð1r   c                 ó  • U R                   (       d  [        S5      eU R                   HS  nUR                  n[	        UR
                  X R                  5      u  p4X1R                  R                  UR                  '   MU     g)NzMYou must call `setup_optimizer()` on your optimizer before starting training.T)
r%   r(   rA   r   Úquantizer3   r   r4   Ú_parametersr.   )r+   r^   r   Úquantr/   s        r   Ú_pre_forward_trainÚLSQ._pre_forward_train   se   € Ø×$×$Üð  ;ó <ð <à—m”mˆFØ—L‘LˆEÜ §¡¨e·Y±YÓ?‰HˆEà5:�M‰M×%Ñ% f§k¡kÓ2ñ	 $ð
 r   c                 ó„   • U R                    H0  nUR                  UR                  R                  UR                  '   M2     g)NT)rA   r3   r4   rf   r.   )r+   r^   s     r   Ú_post_forward_trainÚLSQ._post_forward_trainŠ   s/   € Ø—m”mˆFØ5;·\±\ˆF�M‰M×%Ñ% f§k¡kÓ2ñ $àr   r^   Úreturnc                 óÈ  • [        UR                  UR                  U R                  5      u  p#USU R                  S-
  -  S-
  :*  R	                  5       (       d   UR                  5       5       eUSU R                  S-
  -  * :¬  R	                  5       (       d   UR                  5       5       eUR                  5       R                  5       UR                  R                  5       4$ ©Nr6   r   )	re   r3   r   r   ÚallÚmaxÚminÚdetachÚshort)r+   r^   r/   Úindexs       r   Ú_quantize_paramÚLSQ._quantize_param�   s±   € Ü˜FŸL™L¨&¯,©,¸¿	¹	ÓB‰ˆØ˜! §	¡	¨A¡Ñ.°Ñ2Ñ3×8Ñ8×:Ñ:ÐG¸E¿I¹I»KÓGÐ:Ø˜1 §¡¨Q¡Ñ/Ð/Ñ0×5Ñ5×7Ñ7ÐD¸¿¹»ÓDÐ7Ø�|‰|‹~×#Ñ#Ó% v§|¡|×':Ñ':Ó'<Ð<Ð<r   Ú	quantizedc                 ó0   • Uu  p4UR                  5       U-  $ ©N)Úfloat)r+   r^   rx   ru   r   s        r   Ú_unquantize_paramÚLSQ._unquantize_param•   s   € Ø ‰ˆØ�{‰{‹}˜uÑ$Ð$r   c                 óF   • Uu  pEU" USU R                   S-
  -  -   5      nXe4$ ro   )r   )r+   r^   rx   Úpack_fnÚlevelsr   Úpackeds          r   Ú_bit_pack_paramÚLSQ._bit_pack_param™   s-   € Ø!‰ˆÙ˜ !¨¯	©	°A©Ñ"6Ñ6Ó7ˆØˆÐr   c                 óò   • Uu  pEU" XAR                   R                  5       5      R                  UR                   R                  5      R	                  UR                   5      nUSU R
                  S-
  -  -  nXe4$ )z=Unpack bitpacked representation. Should be overriden
        r6   r   )r3   rZ   ÚtoÚdeviceÚview_asr   )r+   r^   r�   Ú	unpack_fnÚpacked_levelsr   r€   s          r   Ú_bit_unpack_paramÚLSQ._bit_unpack_paramž   sl   € ð  &ÑˆÙØŸ<™<×-Ñ-Ó/ó1ß13±°F·L±L×4GÑ4GÓ1HÏÉÐQW×Q]ÑQ]ÓI^ð 	à�!˜Ÿ	™	 A™Ñ&Ñ&ˆØˆÐr   c                 ó¢   >• [         TU ]  5         U R                   H0  n[        UR                  UR
                  U R                  -   5        M2     g rz   )r)   rs   rA   Údelattrr4   r.   r"   )r+   r^   r0   s     €r   rs   Ú
LSQ.detach§   s6   ø€ Ü‰‰ÔØ—m”mˆFÜ�F—M‘M 6§;¡;°·±Ñ#<Ö=ò $r   c                 ó   • [        U 5      $ rz   )r   rT   s    r   Ú__repr__ÚLSQ.__repr__¬   s   € Ü˜4Ó Ð r   )r%   r   r"   )F)&r   r   r   r   Ú__doc__r   r   r   r   r   r   ÚModuleÚintr{   ÚboolÚstrr*   r<   ÚoptimÚ	OptimizerrK   rQ   rU   rY   rb   rh   rk   ÚtpÚAnyrv   ÚTensorr|   r‚   rŠ   rs   r�   r   Ú__classcell__)r0   s   @r   r	   r	      s  ø† ñð ô"˜-×7Ñ7ó "ó ð"ð Ø;<ÐPTØ!&°fÀbÐW[ñJ˜eŸh™hŸo™oð J°Sð JÈð JØðJØ03÷Jó ðJò	Kð,¨¯©×)>Ñ)>ô ,ð)¨¯©×)>Ñ)>ô )ò4%÷ ò,2ò	òð
= oð =¸"¿&¹&ô =ð%¨ð %ÀBÇFÁFð %ÈuÏ|É|ô %òò
õ>÷
!ð !r   r	   c                 óJ   • U R                  5       U -
  R                  5       U -   $ rz   )Úroundrs   )Úxs    r   Ú	roundpassr    °   s    € Ø�G‰G‹I˜‰M×!Ñ!Ó# aÑ'Ð'r   c                 ó6   • X U-  -
  R                  5       X-  -   $ rz   )rs   )rŸ   r   s     r   Ú	gradscaler¢   ´   s   € Ø�E‘	‰M×!Ñ!Ó# a¡iÑ/Ð/r   c                 ó¼   • SUS-
  -  * nSUS-
  -  S-
  n[        USU R                  5       U-  S-  -  5      nX-  nUR                  X45      n[        U5      nXQ-  U4$ )Nr6   r   r7   )r¢   rZ   Úclampr    )Útensorr   r   ÚlowÚhighru   s         r   re   re   ¸   sq   € Ø�$˜‘(‰OÐ
€CØ�˜‘‰?˜QÑ€DÜ�e˜Q &§,¡,£.°4Ñ"7¸#Ñ!=Ñ=Ó>€Eà‰N€EØ�K‰K˜Ó"€EÜ�eÓ€EØ‰=˜%ÐÐr   )r’   Údataclassesr   Útypingr™   r   Úbaser   Úutilsr   r   r	   r    r¢   re   r   r   r   Ú<module>r¬      s9   ðñõ "Û ã å ß ,ôZ!ˆ-ô Z!òz(ò0ó r   