ó
    æ“,jú5  ã                   ón   • S r SSKJr  SSKrSSKrSSKrSSKJr  SSK	J
r
Jr  SSKJrJr   " S S	\5      rg)
z;
Differentiable quantizer based on scaled noise injection.
é    )Ú	dataclassNé   )ÚBaseQuantizer)Úuniform_quantizeÚuniform_unquantize)Úcapture_initÚsimple_reprc                   ód  ^ • \ rS rSr\ " S S\R                  5      5       r\SSSSSS	S
SSS/ S4S\R                  R                  S\S\S\S\S\S\S\S\S\R                   \   S\4U 4S jjj5       rS\R$                  4S jrS\R$                  4S jrS rS\R,                  R.                  4S  jr S3S\R,                  R.                  S!\4S" jjrS# rS$ rS4U 4S% jjrS& rS' rS( rS)\S*\R@                  4S+ jr!S)\S,\R@                  S*\R$                  4S- jr"S. r#S/ r$U 4S0 jr%S1 r&S2r'U =r($ )5ÚDiffQuantizeré   c                   óH   • \ rS rSr% \R
                  R                  \S'   Srg)ÚDiffQuantizer._QuantizedParamé   Úlogit© N)	Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__ÚtorchÚnnÚ	ParameterÚ__annotations__Ú__static_attributes__r   ó    Ú™C:\Users\johnn\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.13_qbz5n2kfra8p0\LocalCache\local-packages\Python313\site-packages\diffq/diffq.pyÚ_QuantizedParamr      s   ‡ à�x‰x×!Ñ!Ö!r   r   g{®Gáz„?Fr   é   é   ÚbitsÚgaussiané   r   Ú_diffqTÚmodelÚmin_sizeÚfloat16Ú
group_sizeÚmin_bitsÚmax_bitsÚ	init_bitsÚ
extra_bitsÚsuffixÚexcludeÚdetect_boundc                 óÈ  >• X@l         XPl        X`l        X�l        X l        X°l        Xpl        X€l        US;   d   eSU l        SSU R                  -  S-
  -  U l	        SSU R                  -  S-
  -  U l
        US:¼  d   eXYs=:  a  U:  d   S5       e   S5       eUR                  5        H&  u  pïUR                  U5      (       d  M  [        S5      e   [        TU ]=  XX<U5        g)	a°  
Differentiable quantizer based on scaled noise injection.
For every parameter `p` in the model, this introduces a number of bits parameter
`b` with the same dimensions (when group_size = 1).
Before each forward, `p` is replaced by `p + U`
with U uniform iid noise with range [-d/2, d/2], with `d` the uniform quantization
step for `b` bits.
This noise approximates the quantization noise in a differentiable manner, both
with respect to the unquantized parameter `p` and the number of bits `b`.

At eveluation (as detected with `model.eval()`), the model is replaced
by its true quantized version, and restored when going back to training.

When doing actual quantization (for serialization, or evaluation),
the number of bits is rounded to the nearest integer, and needs to be stored along.
This will cost a few bits per dimension. To reduce this cost, one can use `group_size`,
which will use a single noise level for multiple weight entries.

You can use the `DiffQuantizer.model_size` method to get a differentiable estimate of the
model size in MB. You can then use this estimate as a penalty in your training loss.

Args:
    model (torch.nn.Module): model to quantize
    min_size (float): minimum size in MB of a parameter to be quantized.
    float16 (bool): if a layer is smaller than min_size, should we still do float16?
    group_size (int): weight entries are groupped together to reduce the number
        of noise scales to store. This should divide the size of all parameters
        bigger than min_size.
    min_bits (float): minimal number of bits.
    max_bits (float): maximal number of bits.
    init_bits (float): initial number of bits.
    extra_bits (float): extra bits to add for actual quantization (before roundoff).
    suffix (str): suffix used for the name of the extra noise scale parameters.
    exclude (list[str]): list of patterns used to match parameters to exclude.
        For instance `['bias']` to exclude all bias terms.
    detect_bound (bool): if True, will detect bound parameters and reuse
        the same quantized tensor for both, as well as the same number of bits.

..Warning::
    You must call `model.training()` and `model.eval()` for `DiffQuantizer` work properly.

)r!   ÚuniformFr   r   r   z9init_bits must be between min_bits and max_bits excluded3zlThe model already has some noise scales parameters, maybe you used twice a DiffQuantizer on the same model?.N)r'   r(   r)   r*   r+   r,   ÚparamÚnoiseÚ_optimizer_setupÚ
_min_noiseÚ
_max_noiseÚnamed_parametersÚendswithÚRuntimeErrorÚsuperÚ__init__)Úselfr$   r%   r&   r'   r(   r)   r1   r2   r*   r+   r,   r-   r.   ÚnameÚ_Ú	__class__s                   €r   r:   ÚDiffQuantizer.__init__   s  ø€ ð` %ŒØ ŒØ ŒØ"ŒØ$ŒØŒØŒ
ØŒ
ØÐ/Ó/Ð/Ð/Ø %ˆÔà˜q D§M¡MÑ1°AÑ5Ñ6ˆŒØ˜q D§M¡MÑ1°AÑ5Ñ6ˆŒà˜Q‹ÐˆØÕ. hÓ.ð 	KØJó	KÑ.ð 	KØJó	KÐ.ð ×-Ñ-Ö/‰GˆDØ�}‰}˜V×$Ó$Ü"ð $^ó _ð _ñ 0ô
 	‰Ñ˜¨'¸LÕIr   r   c                 óè   • U R                   S:X  a+  [        R                  " SSU R                  U5      -  -   5      $ [        R                  " U5      nU R
                  U-  SU-
  U R                  -  -   $ )Nr2   r   )r1   r   Úlog2Ú_get_noise_scaleÚsigmoidr)   r(   ©r;   r   Úts      r   Ú	_get_bitsÚDiffQuantizer._get_bitsc   sa   € Ø�:‰:˜Ó Ü—:’:˜a ! d×&;Ñ&;¸EÓ&BÑ"BÑBÓCÐCä—’˜eÓ$ˆAØ—=‘= 1Ñ$¨¨A©°·±Ñ'>Ñ>Ð>r   c                 ó>  • U R                   S:X  at  [        R                  " U5      n[        R                  " U[        R
                  " U R                  5      -  SU-
  [        R
                  " U R                  5      -  -   5      $ SSU R                  U5      -  S-
  -  $ )Nr2   r   r   )	r1   r   rC   ÚexpÚmathÚlogr4   r5   rF   rD   s      r   rB   ÚDiffQuantizer._get_noise_scalej   sy   € Ø�:‰:˜Ó Ü—’˜eÓ$ˆAÜ—9’9˜Q¤§¢¨$¯/©/Ó!:Ñ:¸aÀ!¹eÄtÇxÂxÐPT×P_ÑP_ÓG`Ñ=`Ñ`ÓaÐaà˜˜TŸ^™^¨EÓ2Ñ2°QÑ6Ñ7Ð7r   c                 óŽ  • Ub%  U R                   R                  XX4R                  US9$ U R                  S:X  d#  UR	                  5       U R                  -  S:X  d   eU R
                  S:X  a�  SSU R                  -  S-
  -  n[        R                  " U5      [        R                  " U R                  5      -
  [        R                  " U R                  5      [        R                  " U R                  5      -
  -  nO3U R                  U R                  -
  U R                  U R                  -
  -  nSUs=:  a  S:  d   e   e[        R                  " [        R                  " [        U5      5      5      n[!        U R#                  U5      U R                  -
  5      S:  d   eU R                  S:”  a  UR	                  5       U R                  -  nOSn[        R$                  R'                  [        R(                  " U4UUR*                  S95      nUR-                  XR.                  -   U5        U R                   R                  XX7S S9$ )N)r<   r1   Úmoduler   Úotherr   r2   r   r   gñhãˆµøä>)Údevice)r>   r   r   r'   Únumelr1   r*   rJ   rK   r5   r4   r(   r)   r   ÚtensorÚfloatÚabsrF   r   r   ÚfullrP   Úregister_parameterr,   )	r;   r<   r1   rN   rO   Únoise_scalerE   r   Únparams	            r   Ú_register_paramÚDiffQuantizer._register_paramq   sÒ  € ØÑØ—>‘>×1Ñ1Ø¨f¿K¹KÈuð 2ð Vð Và�‰ !Ó# u§{¡{£}°t·±Ñ'FÈ!Ó'KÐKÐKà�:‰:˜Ó Ø˜q D§N¡NÑ2°QÑ6Ñ7ˆKÜ—’˜+Ó&¬¯ª°$·/±/Ó)BÑBÜ—’˜Ÿ™Ó)¬D¯HªH°T·_±_Ó,EÑEñG‰Að —‘ $§-¡-Ñ/°D·M±MÀDÇMÁMÑ4QÑRˆAØ�1�y�q‹yÐ‰yÐˆyÜ—’œEŸLšL¬¨q«Ó2Ó3ˆÜ�4—>‘> %Ó(¨4¯>©>Ñ9Ó:¸TÓAÐAÐAØ�?‰?˜QÓØ—[‘[“] d§o¡oÑ5‰FàˆFÜ—‘×"Ñ"Ü�JŠJØ�	ØØ—|‘|ñ%ó&ˆð
 	×!Ñ! $¯©Ñ"4°eÔ<Ø�~‰~×-Ñ-Ø¨&ÀTð .ð Kð 	Kr   Ú	optimizerc                 ó  • U R                    Vs/ s H  o"R                  PM     nnUR                   HN  n/ n[        US   5       H0  nSnU H  nX†L d  M	  SnM     U(       a  M  UR	                  U5        M2     XTS   S S & MP     g s  snf )NÚparamsFT)Ú_qparamsr   Úparam_groupsÚlistÚappend)	r;   r[   Úqpr]   ÚgroupÚ
new_paramsÚqÚmatchedÚps	            r   Úclear_optimizerÚDiffQuantizer.clear_optimizer�   s…   € Ø%)§]¢]Ó3¢]˜r—(”(¡]ˆÐ3à×+Ô+ˆEØˆJÜ˜% ™/Ö*�Ø�Û�AØ”vØ"&šñ  ÷ �wØ×%Ñ% aÖ(ñ +ð ",�(‰O™AÒò ,ùò 4s   �BÚlrc                 óZ  • U R                   (       a   eSU l         U R                   Vs/ s H  oDR                  PM     nnUR                   H0  n[	        US   5       H  nU H  nX‡L d  M	  [        S5      e   M     M2     XRSS.nUR                  U5        UR                  U5        gs  snf )af  
Setup the optimizer to tune the number of bits. In particular, this will deactivate
weight decay for the bits parameters.

Args:
    optimizer (torch.Optimizer): optimizer to use.
    lr (float): specific learning rate for the bits parameters. 1e-3
        is perfect for Adam.,w
    kwargs (dict): overrides for other optimization parameters for the bits.
Tr]   z5You should create the optimizer before the quantizer!r   )r]   rj   Úweight_decayN)r3   r^   r   r_   r`   r8   ÚupdateÚadd_param_group)	r;   r[   rj   Úkwargsrb   r]   rc   re   rg   s	            r   Úsetup_optimizerÚDiffQuantizer.setup_optimizer›   s§   € ð ×(×(Ð(Ð(Ø $ˆÔà%)§]¢]Ó3¢]˜r—(”(¡]ˆÐ3à×+Ô+ˆEÜ˜% ™/Ö*�Û�AØ”vÜ*ð ,Có Dð Dó  ó +ñ ,ð "¸QÑ?ˆØ�‰�VÔØ×!Ñ! %Õ(ùò 4s   ©B(c                 ó   • SU l         g)z3
Call this if you do not want to use an optimizer.
TN)r3   ©r;   s    r   Úno_optimizerÚDiffQuantizer.no_optimizer·   s   € ð !%ˆÕr   c                 ó<  • U R                    HŒ  nUR                  b  M  UR                  R                  nUb  US:H  R	                  5       (       d  ME  UR
                  R                  c  M^  UR
                  R                  R                  R                  5         MŽ     g )Nr   )r^   rO   r1   ÚgradÚallr   ÚdataÚzero_)r;   Úqparamrw   s      r   Úcheck_unusedÚDiffQuantizer.check_unused½   sm   € Ø—m”mˆFØ�|‰|Ñ'ÙØ—<‘<×$Ñ$ˆDØ‰| ¨¡	Ÿ™×0Ó0Ø—<‘<×$Ñ$Ó0Ø—L‘L×%Ñ%×*Ñ*×0Ñ0Ö2ò $r   c                 óº  >• [         TU ]  5       nSnU R                   GH1  nUR                  b  M  U R                  U R                  UR                  5      -   nU(       a   UR                  5       R                  SS5      nU R                  S:X  a  UR                  R                  5       nOU R                  nX6UR                  5       -  -  nUS-  n[        R                  " [        R                  " SUR!                  5       R                  5       R#                  5       U R$                  -
  -   5      5      nUS-  nX7UR                  5       -  -  nGM4     US-  nX#-   $ )zù
Differentiable estimate of the model size.
The size is returned in MB.

If `exact` is True, then the output is no longer differentiable but
reflect exactly an achievable size, even without compression,
i.e.same as returned by `naive_model_size()`.
r   r   r   é@   r"   i  € )r9   Ú
model_sizer^   rO   r+   rF   r   ÚroundÚclampr'   r1   rQ   ÚsumrJ   ÚceilrA   ÚmaxÚitemr(   )	r;   ÚexactÚtotalÚsubtotalr{   r    r'   Ú	bits_bitsr>   s	           €r   r€   ÚDiffQuantizer.model_sizeÆ   s  ø€ ô ‘Ñ"Ó$ˆØˆØ—m•mˆFà�|‰|Ñ'ÙØ—?‘? T§^¡^°F·L±LÓ%AÑAˆDÞØ—z‘z“|×)Ñ)¨!¨RÓ0�Ø�‰ !Ó#Ø#Ÿ\™\×/Ñ/Ó1‘
à!Ÿ_™_�
Ø T§X¡X£ZÑ/Ñ/ˆHØ˜ÑˆHô Ÿ	š	¤$§)¢)¨A°·±³×1AÑ1AÓ1C×1HÑ1HÓ1JÈTÏ]É]Ñ1ZÑ,[Ó"\Ó]ˆIØ˜‰MˆHØ D§J¡J£LÑ0Ñ0‹Hñ# $ð& 	�KÑˆØÑÐr   c                 ó<   • U R                  SS9R                  5       $ )z,
Naive model size without zlib compression.
T)r‡   )r€   r†   rs   s    r   Útrue_model_sizeÚDiffQuantizer.true_model_sizeç   s   € ð �‰ TˆÐ*×/Ñ/Ó1Ð1r   c                 ód  • U R                   (       d  [        S5      eU R                   GHƒ  nUR                  b8  UR                  R                  R
                  UR                  R                     nOþU R                  UR                  5      S S 2S 4   nU R                  S:X  a  UR                  R                  S5      nO&UR                  R                  SU R                  5      nUR                  5       UR                  5       -
  nSSU-  S-
  -  nU R                  S:X  a  [        R                   " U5      S-
  nO)U R                  S:X  a  [        R"                  " U5      S-  nXV-  W-  nXH-   nUR%                  UR                  5      UR                  R
                  UR                  '   GM†     g	)
NzMYou must call `setup_optimizer()` on your optimizer before starting training.r   éÿÿÿÿr   r   r0   g      à?r!   T)r3   r8   r^   rO   rN   Ú_parametersr<   rF   r   r'   r1   Úviewr…   Úminr2   r   Ú	rand_likeÚ
randn_likeÚview_as)	r;   r{   Únoisyr    Úp_flatÚscaleÚunitÚnoise_sourcer2   s	            r   Ú_pre_forward_trainÚ DiffQuantizer._pre_forward_trainí   sS  € Ø×$×$Üð  ;ó <ð <à—m•mˆFØ�|‰|Ñ'ØŸ™×+Ñ+×7Ñ7¸¿¹×8IÑ8IÑJ‘à—~‘~ f§l¡lÓ3²A°t°GÑ<�Ø—?‘? aÓ'Ø#Ÿ\™\×.Ñ.¨rÓ2‘Fà#Ÿ\™\×.Ñ.¨r°4·?±?ÓC�FØŸ
™
› v§z¡z£|Ñ3�Ø˜A˜t™G a™KÑ(�Ø—:‘: Ó*Ü$)§O¢O°FÓ$;¸cÑ$A‘LØ—Z‘Z :Ó-Ü#(×#3Ò#3°FÓ#;¸aÑ#?�LØ™ |Ñ3�Ø™�à5:·]±]À6Ç<Á<Ó5PˆF�M‰M×%Ñ% f§k¡kÔ2ñ% $ð& r   c                 ó„   • U R                    H0  nUR                  UR                  R                  UR                  '   M2     g)NT)r^   r1   rN   r‘   r<   )r;   r{   s     r   Ú_post_forward_trainÚ!DiffQuantizer._post_forward_train  s/   € Ø—m”mˆFØ5;·\±\ˆF�M‰M×%Ñ% f§k¡kÓ2ñ $àr   r{   Úreturnc                 óº  • U R                   U R                  UR                  5      -   nUR                  5       R	                  SS5      S S 2S 4   R                  5       nU R                  S:X  a'  UR                  R                  R                  SS5      nO0UR                  R                  R                  SU R                  5      n[        X25      u  pEXEUS S 2S4   4$ )Nr   r   r   r�   )r+   rF   r   r�   r‚   Úbyter'   r1   ry   r’   r   )r;   r{   r    rg   ÚlevelsÚscaless         r   Ú_quantize_paramÚDiffQuantizer._quantize_param  s®   € Ø�‰ §¡°·±Ó!=Ñ=ˆØ�z‰z‹|×!Ñ! ! RÓ(ª¨D¨Ñ1×6Ñ6Ó8ˆØ�?‰?˜aÓØ—‘×!Ñ!×&Ñ& q¨"Ó-‰Aà—‘×!Ñ!×&Ñ& r¨4¯?©?Ó;ˆAÜ)¨!Ó2‰ˆØ˜t¢A q D™zÐ)Ð)r   Ú	quantizedc                 ó¤   • Uu  p4nUR                  5       S:X  a	  US S 2S 4   n[        X4U5      R                  UR                  R                  5      $ )Nr   )Údimr   r–   r1   ry   )r;   r{   r¨   r¤   Úparam_scaler    s         r   Ú_unquantize_paramÚDiffQuantizer._unquantize_param  sH   € Ø$-Ñ!ˆ˜TØ�8‰8‹:˜‹?Øš˜4˜‘=ˆDÜ! &°tÓ<×DÑDÀVÇ\Á\×EVÑEVÓWÐWr   c                 óö   • Uu  pEn/ n[        SS5       HK  nXFU:H     n	U	R                  5       (       d  UR                  S 5        M2  U" X˜5      n
UR                  U
5        MM     U" X`R                  -
  5      nXuU4$ )Nr   r   )ÚrangerQ   ra   r(   )r;   r{   r¨   Úpack_fnr¤   r¥   r    Ú
all_packedÚbitÚ
sub_levelsÚpackedÚpacked_bitss               r   Ú_bit_pack_paramÚDiffQuantizer._bit_pack_param  s€   € Ø(Ñˆ˜Øˆ
Ü˜˜B–<ˆCØ¨¡Ñ,ˆJØ×#Ñ#×%Ñ%Ø×!Ñ! $Ö'á  Ó1�Ø×!Ñ! &Ö)ñ  ñ ˜d§]¡]Ñ2Ó3ˆØ KÐ0Ð0r   c                 ó*  • Uu  pEnU" XaR                   R                  5       5      U R                  -   nUR                  UR                  R
                  5      n[        R                  " UR                   R                  5       U R                  [        R                  UR                  R
                  S9n[        U5       HO  u  pšU	S-   nU
c  M  X‡U:H     nU" X¬R                  5       5      R                  U5      R                  U5      X‡U:H  '   MQ     X…U4$ )z>Unpack bitpacked representation. Should be overriden.
        )ÚdtyperP   r   )r   rQ   r(   Útor1   rP   r   Úemptyr'   ÚshortÚ	enumerater–   )r;   r{   r´   Ú	unpack_fnÚpacked_all_levelsr¥   rµ   r    r¤   ÚidxÚpacked_levelsr²   r³   s                r   Ú_bit_unpack_paramÚDiffQuantizer._bit_unpack_param(  sî   € ð 28Ñ.Ð ;Ù˜§l¡l×&8Ñ&8Ó&:Ó;¸d¿m¹mÑKˆØ�w‰w�v—|‘|×*Ñ*Ó+ˆÜ—’˜VŸ\™\×/Ñ/Ó1°4·?±?Ü#(§;¡;°v·|±|×7JÑ7JñLˆä"+Ð,=Ö">ÑˆCØ˜‘'ˆCØÑ$ÙØ¨¡Ñ,ˆJÙ"+Ø×/Ñ/Ó1ó#3ß3:±7¸:Ó3FÇrÁrÈ*Ã~ð ˜3‘;Óñ #?ð  Ð%Ð%r   c                 ó¢   >• [         TU ]  5         U R                   H0  n[        UR                  UR
                  U R                  -   5        M2     g ©N)r9   Údetachr^   ÚdelattrrN   r<   r,   )r;   r{   r>   s     €r   rÆ   ÚDiffQuantizer.detach9  s6   ø€ Ü‰‰ÔØ—m”mˆFÜ�F—M‘M 6§;¡;°·±Ñ#<Ö=ò $r   c                 ó   • [        U 5      $ rÅ   )r	   rs   s    r   Ú__repr__ÚDiffQuantizer.__repr__>  s   € Ü˜4Ó Ð r   )r5   r4   r3   r+   r'   r*   r)   r(   r2   r1   r,   )gü©ñÒMbP?)F))r   r   r   r   r   r   r   r   r   r   ÚModulerS   ÚboolÚintÚstrÚtpÚListr:   ÚTensorrF   rB   rY   ÚoptimÚ	Optimizerrh   rp   rt   r|   r€   r�   rœ   rŸ   ÚAnyr¦   r¬   r¶   rÂ   rÆ   rÊ   r   Ú__classcell__)r>   s   @r   r   r      s¶  ø† Øô"˜-×7Ñ7ó "ó ð"ð ØAEÐW\Ø#$¸ÈRØ ZØ$%¸1ÈHØ)+À$ñ	FJ˜eŸh™hŸo™oð FJ¸ð FJÐPTð FJØ ðFJØ05ðFJØEJðFJð "ðFJð 49ðFJð GJðFJð Ÿ'™' #™,ð	FJð <@÷	FJó ðFJðP?˜uŸ|™|ô ?ð8 e§l¡lô 8òKð8,¨¯©×)>Ñ)>ô ,ð %)ñ)¨¯©×)>Ñ)>ð )Ø!õ)ò8%ò3÷ òB2òò2ð
* oð *¸"¿&¹&ô *ðX¨ð XÀBÇFÁFð XÈuÏ|É|ô Xò1ò&õ">÷
!ð !r   r   )Ú__doc__Údataclassesr   rJ   ÚtypingrÐ   r   Úbaser   r0   r   r   Úutilsr   r	   r   r   r   r   Ú<module>rÜ      s/   ðñõ "Û Û ã å ß 9ß ,ôj!�Mõ j!r   