§
    9†Æj^,  ã                   ó`  — d dl Z d dlZd dlmc mZ d dlZd dlmZm	Z	m
Z
 d dlZd dlmZ d dlmZmZmZ d dlmZmZmZmZ d dlmZ d dlZddlmZ d dlmZ ej          !                    ¦   «         rej          "                    ¦   «         nd	Z#d
ej$        dej$        fd„Z%d„ Z& G d„ d¦  «        Z'e(dk    �r{ ej)        d¦  «        Z* ej+        e*ej,        ¦  «        Z*dZ-dZ.dZ/ e'e-e.e/¬¦  «        Z0e0 1                    e*¦  «         dZ2 ej3        ddgddgg¦  «         ej3        ddg¦  «        dœZ4e0 5                    e4e2d	¬¦  «        \  Z6Z7Z8 ee*e6 ej9        e7¦  «                  :                    d¦  «        d¬¦  «        Z; ej+        e;ej<        ¦  «        Z; ej=        d e;¦  «         d!Z2e8 ej9        e7¦  «        dd…dd…f         Z>d"e>ddd…dd…f         iZ4 ej3        ddgddgg¦  «         ej3        dd g¦  «        e>ddd…dd…f         d#œZ4e0 5                    e4e2d$¬¦  «        \  Z6Z7Z8 ee*e6 ej9        e7¦  «                  :                    d¦  «        d¬¦  «        Z; ej+        e;ej<        ¦  «        Z; ej=        d%e;¦  «         d&Z2e8 ej9        e7¦  «        dd…dd…f         Z>d"e>ddd…dd…f         iZ4e0 5                    e4e2d$¬¦  «        \  Z6Z7Z8 ee*e6 ej9        e7¦  «                  :                    d¦  «        d¬¦  «        Z; ej+        e;ej<        ¦  «        Z; ej=        d'e;¦  «         dS dS )(é    N)ÚImageÚ	ImageDrawÚImageOps)ÚUnion)Úsam_model_registryÚSamPredictorÚSamAutomaticMaskGenerator)Úwindow_partitionÚwindow_unpartitionÚget_rel_posÚBlocké   )Úmask_painter)Úfiles_locatorFÚxÚreturnc                 óÈ  ‡— dd„}dt           j        fd„Šddt           j        dt           j        fˆfd„}|}|                      |¦  «        }| j        dk    r2|j        d         |j        d	         }}t          || j        ¦  «        \  }}|j        }|d         d
k    r_t          |j        d         dz  ¦  «        dz   }	t          j        ||	¦  «        }
t          |
¦  «        D ]\  }} || j	        |¦  «        |d<   Œn || j	        |d¦  «        }| j        dk    rt          || j        |||f¦  «        }||z  }|                      |¦  «        |d<   | || j        |¦  «        z  }|S )zfVRAM-optimized forward pass for SAM image encoder blocks.

    Optimizations made by DeepBeepMeep
    é   c                 óŽ  — |j         }|                     d|j         d         ¦  «        }t          |j         d         |z  ¦  «        }t          j        ||¦  «        }t          |¦  «        D ]G\  }}|                      |¦  «        }|                      |¦  «        }|                      |¦  «        |d<   ŒH| 	                    |¦  «        S )Néÿÿÿÿr   .)
ÚshapeÚviewÚintÚtorchÚsplitÚ	enumerateÚlin1ÚactÚlin2Úreshape)	Úmlpr   ÚdivideÚx_shapeÚ
chunk_sizeÚx_chunksÚiÚx_chunkÚ	mlp_chunks	            úTC:\Users\johnn\OneDrive\Documents\AI\preprocessing\matanyone\tools\base_segmenter.pyÚ	split_mlpz#_patched_forward.<locals>.split_mlp   s±   € Ø”'ˆØ�FŠF�2�q”w˜r”{Ñ#Ô#ˆÝ˜œ œ fÑ,Ñ-Ô-ˆ
Ý”;˜q *Ñ-Ô-ˆÝ# HÑ-Ô-ð 	/ð 	/‰JˆAˆwØŸš Ñ)Ô)ˆIØŸš 	Ñ*Ô*ˆIØŸ8š8 IÑ.Ô.ˆG�C‰LˆLØ�yŠy˜Ñ!Ô!Ð!ó    r   c           	      óÜ  — |\  }}|\  }}t          |||¦  «        }	t          |||¦  «        }
| j        \  }}}|                      ||||¦  «        }t          j        d||	¦  «        }t          j        d||
¦  «        }t          j        |||||| j        | j        ¬¦  «        }||d d …d d …d d …d d …d f         z  }||d d …d d …d d …d d d …f         z  }|                     |||z  ||z  ¦  «        S )Nzbhwc,hkc->bhwkzbhwc,wkc->bhwk)ÚdtypeÚdevice)	r   r   r    r   ÚeinsumÚzerosr-   r.   r   )ÚqÚ	rel_pos_hÚ	rel_pos_wÚq_sizeÚk_sizeÚq_hÚq_wÚk_hÚk_wÚRhÚRwÚBÚ_ÚdimÚr_qÚrel_hÚrel_wÚattns                     r)   Úget_decomposed_rel_posz0_patched_forward.<locals>.get_decomposed_rel_pos#   s  € Ø‰ˆˆSØ‰ˆˆSÝ˜˜c 9Ñ-Ô-ˆÝ˜˜c 9Ñ-Ô-ˆØ”G‰	ˆˆ1ˆcØ�iŠi˜˜3  SÑ)Ô)ˆÝ”Ð-¨s°BÑ7Ô7ˆÝ”Ð-¨s°BÑ7Ô7ˆÝŒ{˜1˜c 3¨¨S¸¼ÈÌÐQÑQÔQˆØ��a�a�a˜˜˜˜A˜A˜A˜q˜q˜q $Ð&Ô'Ñ'ˆØ��a�a�a˜˜˜˜A˜A˜A˜t Q Q QÐ&Ô'Ñ'ˆØ�yŠy˜˜C #™I s¨S¡yÑ1Ô1Ð1r+   r   r   c           	      ób  •— |j         \  }}}}|                      |¦  «                             |||z  d| j        d¦  «                             ddddd¦  «        }t
          s|                     t          j        ¦  «        }|                     d|| j        z  ||z  d¦  «         	                    d¦  «        \  }}	}
|dk    rkd }| j
        rC ‰|| j                             |¦  «        | j                             |¦  «        ||f||f¦  «        }t          j        ||	|
|| j        ¬¦  «        }ný| j        |z  }t          j        |¦  «        }t          j        ||¦  «        }t          j        |	|¦  «        }t          j        |
|¦  «        }t          j        ||¦  «        }t%          ||||¦  «        D ]t\  }}}}d }| j
        rC ‰|| j                             |¦  «        | j                             |¦  «        ||f||f¦  «        }t          j        ||||| j        ¬¦  «        |d<   Œu~~~~~~	~
~|                     || j        ||d¦  «                             ddddd¦  «                             |||d¦  «        }t
          s|                     t          j        ¦  «        }|                      |¦  «        S )	Né   r   é   r   r   r   )Ú	attn_maskÚscale.)r   Úqkvr    Ú	num_headsÚpermuteÚ_bfloat16_supportedÚtor   Úfloat16ÚunbindÚuse_rel_posr2   r3   ÚFÚscaled_dot_product_attentionrH   Ú
empty_liker   Úzipr   Úbfloat16Úproj)Úselfr   Úsplit_headsr<   ÚHÚWr=   rI   r1   ÚkÚvrG   r$   Úq_chunksÚk_chunksÚv_chunksr%   r'   Úq_chunkÚk_chunkÚv_chunkrC   s                        €r)   Úpay_attentionz'_patched_forward.<locals>.pay_attention1   s½  ø€ Ø”W‰
ˆˆ1ˆa�à�hŠh�q‰kŒk×!Ò! ! Q¨¡U¨A¨t¬~¸rÑBÔB×JÒJÈ1ÈaÐQRÐTUÐWXÑYÔYˆå"ð 	(Ø—&’&�œÑ'Ô'ˆCð —+’+˜a  T¤^Ñ!3°Q¸±U¸BÑ?Ô?×FÒFÀqÑIÔI‰ˆˆ1ˆaØ˜!ÒÐØˆIØÔð rØ2Ð2°1°d´n×6GÒ6GÈÑ6JÔ6JÈDÌN×L]ÒL]Ð^_ÑL`ÔL`ÐcdÐfgÐbhÐklÐnoÐjpÑqÔq�	ÝÔ.¨q°!°QÀ)ÐSWÔS]Ð^Ñ^Ô^ˆAˆAàœ¨;Ñ6ˆJÝÔ  Ñ#Ô#ˆAÝ”{ 1 jÑ1Ô1ˆHÝ”{ 1 jÑ1Ô1ˆHÝ”{ 1 jÑ1Ô1ˆHÝ”{ 1 jÑ1Ô1ˆHÝ69¸(ÀHÈhÐX`Ñ6aÔ6að @ð @Ñ2�˜ '¨7Ø �	ØÔ#ð |Ø 6Ð 6°wÀÄ×@QÒ@QÐRSÑ@TÔ@TÐVZÔVd×VgÒVgÐhiÑVjÔVjÐmnÐpqÐlrÐuvÐxyÐtzÑ {Ô {�IÝ Ô=¸gÀwÐPWÐclÐtxÔt~ÐÑÔ�˜‘�Ø˜ '¨7Øˆq�!�YØ�FŠF�1�d”n a¨¨BÑ/Ô/×7Ò7¸¸1¸aÀÀAÑFÔF×NÒNÈqÐRSÐUVÐXZÑ[Ô[ˆÝ"ð 	%Ø—’•U”^Ñ$Ô$ˆAà�yŠy˜‰|Œ|Ðr+   r   rF   é
   .)r   )r   )r   ÚTensorÚnorm1Úwindow_sizer   r
   r   r   r   rB   r   Únorm2r!   )rW   r   r*   rc   ÚshortcutrY   rZ   Úpad_hwr#   r$   r%   r&   r'   rC   s                @r)   Ú_patched_forwardrk      s­  ø€ ð
	"ð 	"ð 	"ð 	"ð2Í5Ì<ð 2ð 2ð 2ð 2ð!ð !�uœ|ð !½u¼|ð !ð !ð !ð !ð !ð !ðF €HØ�
Š
�1‰Œ€AàÔ˜!ÒÐØŒw�qŒz˜1œ7 1œ:ˆ1ˆÝ$ Q¨Ô(8Ñ9Ô9‰	ˆˆ6ØŒg€Gàˆq„z�B‚€Ý˜œ œ a™Ñ(Ô(¨1Ñ,ˆ
Ý”;˜q *Ñ-Ô-ˆÝ# HÑ-Ô-ð 	=ð 	=‰JˆAˆwØ(˜=¨¬°GÑ<Ô<ˆG�C‰LˆLð	=ð ˆM˜$œ) Q¨Ñ*Ô*ˆð Ô˜!ÒÐÝ˜q $Ô"2°F¸QÀ¸FÑCÔCˆØˆ�M€AØ—J’J˜q‘M”M€HˆS�MØˆˆ�4”8˜XÑ	&Ô	&Ñ&€Aà€Hr+   c                  ón   — t          t          d¦  «        st          t          _        dt          _        dS dS )z5Apply VRAM optimizations to SAM image encoder blocks.ÚpatchedTN)ÚhasattrÚimage_encoder_blockrk   Úforwardrm   © r+   r)   Úset_image_encoder_patchrr   n   s7   € åÕ&¨	Ñ2Ô2ð +Ý&6ÕÔ#Ø&*ÕÔ#Ð#Ð#ð+ð +r+   c                   óŽ   — e Zd Zd	d„Z ej        ¦   «         dej        fd„¦   «         Z ej        ¦   «         d„ ¦   «         Z	d
d„Z
dS )ÚBaseSegmenterúcuda:0c                 óæ  — ddl m} ddlm}  | |¦   «         ¦  «         t	          d|› �¦  «         |dv s
J d¦   «         ‚t          ¦   «          t          j        |¦  «        | _        | j        j        dk    rt          j	        nt          j
        | _        ddlm}  |¦   «         5  t          |         |¬	¦  «        | _        d
d
d
¦  «         n# 1 swxY w Y   ddlm} |                     | j        t'          j        d¦  «        dd
¬¦  «         | j                             t          j
        ¦  «         | j                             | j        ¬¦  «         t-          | j        ¦  «        | _        d| _        d
S )z}
        device: model device
        SAM_checkpoint: path of SAM checkpoint
        model_type: vit_b, vit_l, vit_h
        r   )Úquery_sam_download_def)Úprocess_files_def_if_neededzInitializing BaseSegmenter to )Úvit_bÚvit_lÚvit_hz)model_type must be vit_b, vit_l, or vit_hÚcuda)Úinit_empty_weights)Ú
checkpointN)Úoffloadz&mask/sam_vit_h_4b8939_fp16.safetensorsF)Úwritable_tensorsÚdefault_dtype)r.   )Ú*preprocessing.matanyone.utils.model_assetsrw   Úshared.utils.downloadrx   Úprintrr   r   r.   ÚtyperN   Úfloat32Útorch_dtypeÚ
accelerater}   r   ÚmodelÚmmgpr   Úload_model_dataÚflÚlocate_filerM   r   Ú	predictorÚembedded)rW   ÚSAM_checkpointÚ
model_typer.   rw   rx   r}   r   s           r)   Ú__init__zBaseSegmenter.__init__v   së  € ð 	VÐUÐUÐUÐUÐUØEÐEÐEÐEÐEÐEà#Ð#Ð$:Ð$:Ñ$<Ô$<Ñ=Ô=Ð=ÝÐ7¨vÐ7Ð7Ñ8Ô8Ð8ØÐ8Ð8Ð8Ð8Ð:eÑ8Ô8Ð8õ 	 Ñ!Ô!Ð!å”l 6Ñ*Ô*ˆŒà,0¬KÔ,<ÀÒ,FÐ,F�5œ=˜=ÍEÌMˆÔØ1Ð1Ð1Ð1Ð1Ð1ð  ÐÑ!Ô!ð 	Sð 	SÝ+¨JÔ7À>ÐRÑRÔRˆDŒJð	Sð 	Sð 	Sñ 	Sô 	Sð 	Sð 	Sð 	Sð 	Sð 	Sð 	Søøøð 	Sð 	Sð 	Sð 	Sà Ð Ð Ð Ð Ð ð 	×Ò ¤
­B¬NÐ;cÑ,dÔ,dÐw|ð  MQÐñ  	Rô  	Rð  	RØŒ
�Š•e”mÑ$Ô$Ð$ØŒ
�Š˜Tœ[ˆÑ)Ô)Ð)Ý% d¤jÑ1Ô1ˆŒØˆŒˆˆs   Â$CÃCÃCÚimagec                 ó†   — || _         | j        rt          d¦  «         d S | j                             |¦  «         d| _        d S )Nz%repeat embedding, please reset_image.T)Úorignal_imager�   r„   rŽ   Ú	set_image)rW   r“   s     r)   r–   zBaseSegmenter.set_image˜   sL   € ð #ˆÔØŒ=ð 	ÝÐ9Ñ:Ô:Ð:ØˆFØŒ× Ò  Ñ'Ô'Ð'ØˆŒØˆr+   c                 óF   — | j                              ¦   «          d| _        d S )NF)rŽ   Úreset_imager�   )rW   s    r)   r˜   zBaseSegmenter.reset_image¤   s#   € ð 	Œ×"Ò"Ñ$Ô$Ð$ØˆŒˆˆr+   Tc                 ó,  — | j         s
J d¦   «         ‚|dv s
J d¦   «         ‚t          j        | j        j        t          j        | j        j        dk    ¬¦  «        5  |dk    r.| j                             |d         |d         |¬	¦  «        \  }}}nj|d
k    r'| j                             |d         |¬¦  «        \  }}}n=|dk    r5| j                             |d         |d         |d         |¬¦  «        \  }}}nd‚|||fcddd¦  «         S # 1 swxY w Y   dS )a  
        image: numpy array, h, w, 3
        prompts: dictionary, 3 keys: 'point_coords', 'point_labels', 'mask_input'
        prompts['point_coords']: numpy array [N,2]
        prompts['point_labels']: numpy array [1,N]
        prompts['mask_input']: numpy array [1,256,256]
        mode: 'point' (points only), 'mask' (mask only), 'both' (consider both)
        mask_outputs: True (return 3 masks), False (return 1 mask only)
        whem mask_outputs=True, mask_input=logits[np.argmax(scores), :, :][None, :, :]
        z:prediction is called before set_image (feature embedding).)ÚpointÚmaskÚbothz!mode must be point, mask, or bothr|   )Údevice_typer-   Úenabledrš   Úpoint_coordsÚpoint_labels)rŸ   r    Úmultimask_outputr›   Ú
mask_input)r¢   r¡   rœ   )rŸ   r    r¢   r¡   zNot implement now!N)r�   r   Úautocastr.   r…   rN   rŽ   Úpredict)rW   ÚpromptsÚmodeÚ	multimaskÚmasksÚscoresÚlogitss          r)   r¤   zBaseSegmenter.predictª   s¡  € ð Œ}ÐZÐZÐZÑZÔZˆ}ØÐ0Ð0Ð0Ð0Ð2UÑ0Ô0Ð0åŒ^¨¬Ô(8ÅÄÐW[ÔWbÔWgÐkqÒWqÐrÑrÔrð 	)ð 	)Ø�wŠˆØ(,¬×(>Ò(>ÈGÐTbÔLcØ18¸Ô1HØ5>ð )?ñ )@ô )@Ñ%��v˜v˜vð ˜’�Ø(,¬×(>Ò(>È'ÐR^ÔJ_Ø5>ð )?ñ )@ô )@Ñ%��v˜v˜và˜’�Ø(,¬×(>Ò(>ÈGÐTbÔLcØ18¸Ô1HØ/6°|Ô/DØ5>ð )?ñ )@ô )@Ñ%��v˜v˜vð
 +Ð+à˜& &Ð(ð!	)ð 	)ð 	)ð 	)ñ 	)ô 	)ð 	)ð 	)ð 	)ð 	)ð 	)ð 	)øøøð 	)ð 	)ð 	)ð 	)ð 	)ð 	)s   ÁB#D	Ä	DÄDN)ru   )T)Ú__name__Ú
__module__Ú__qualname__r’   r   Úno_gradÚnpÚndarrayr–   r˜   r¤   rq   r+   r)   rt   rt   u   s‘   € € € € € ð ð  ð  ð  ðD €U„]�_„_ð	˜rœzð 	ð 	ð 	ñ „_ð	ð €U„]�_„_ðð ñ „_ðð
)ð )ð )ð )ð )ð )r+   rt   Ú__main__z/hhd3/gaoshang/truck.jpgz0/ssd1/gaomingqi/checkpoints/sam_vit_h_4b8939.pthr{   zcuda:4)r�   r‘   r.   rš   iô  iw  ie  iq  )rŸ   r    )r§   Úuint8gš™™™™™é?)Úbackground_alphaz/hhd3/gaoshang/truck_point.jpgrœ   r¢   )rŸ   r    r¢   Tz/hhd3/gaoshang/truck_both.jpgr›   z/hhd3/gaoshang/truck_mask.jpg)?Útimer   Útorch.nn.functionalÚnnÚ
functionalrQ   Úcv2ÚPILr   r   r   Únumpyr¯   Útypingr   Úsegment_anythingr   r   r	   Ú'segment_anything.modeling.image_encoderr
   r   r   r   ro   Úmatplotlib.pyplotÚpyplotÚpltr   Úshared.utilsr   rŒ   r|   Úis_availableÚis_bf16_supportedrL   re   rk   rr   rt   r«   Úimreadr“   ÚcvtColorÚCOLOR_BGR2RGBr�   r‘   r.   Úbase_segmenterr–   r¦   Úarrayr¥   r¤   r¨   r©   rª   ÚargmaxÚastypeÚpainted_imageÚCOLOR_RGB2BGRÚimwriter¢   rq   r+   r)   ú<module>rÎ      sê  ðØ €€€Ø €€€Ø Ð Ð Ð Ð Ð Ð Ð Ð Ø 
€
€
€
Ø *Ð *Ð *Ð *Ð *Ð *Ð *Ð *Ð *Ð *Ø Ð Ð Ð Ø Ð Ð Ð Ð Ð Ø XÐ XÐ XÐ XÐ XÐ XÐ XÐ XÐ XÐ Xð Dð  Dð  Dð  Dð  Dð  Dð  Dð  Dð  Dð  Dð  Dð  DØ Ð Ð Ð Ð Ð Ø 
€
€
€
Ø &Ð &Ð &Ð &Ð &Ð &Ø ,Ð ,Ð ,Ð ,Ð ,Ð ,ð 9>¼
×8OÒ8OÑ8QÔ8QÐ\�e”j×2Ò2Ñ4Ô4Ð4ÐW\Ð ðX˜eœlð X¨u¬|ð Xð Xð Xð Xðv+ð +ð +ðS)ð S)ð S)ð S)ð S)ñ S)ô S)ð S)ðl ˆzÒÑàˆCŒJÐ1Ñ2Ô2€EØˆCŒL˜ Ô 1Ñ2Ô2€Eð G€NØ€JØ€FØ"�]°.ÈZÐ`fÐgÑgÔg€Nð ×Ò˜UÑ#Ô#Ð#ð €Dà ˜œ 3¨ *¨t°S¨kÐ!:Ñ;Ô;Ø ˜œ ! Q Ñ(Ô(ðð €Gð +×2Ò2°7¸DÈEÐ2ÑRÔRÑ€Eˆ6�6Ø �L ¨¨i¨b¬i¸Ñ.?Ô.?Ô(@×(GÒ(GÈÑ(PÔ(PÐcfÐgÑgÔg€MØ �C”L °Ô0AÑBÔB€MØ€C„KÐ0°-Ñ@Ô@Ð@ð €DØ˜˜œ 6Ñ*Ô*¨A¨A¨A¨q¨q¨qÐ0Ô1€JØ˜Z¨¨q¨q¨q°!°!°!¨Ô4Ð5€Gà ˜œ 3¨ *¨t°S¨kÐ!:Ñ;Ô;Ø ˜œ ! Q Ñ(Ô(Ø   q q q¨!¨!¨! Ô,ðð €Gð
 +×2Ò2°7¸DÈDÐ2ÑQÔQÑ€Eˆ6�6Ø �L ¨¨i¨b¬i¸Ñ.?Ô.?Ô(@×(GÒ(GÈÑ(PÔ(PÐcfÐgÑgÔg€MØ �C”L °Ô0AÑBÔB€MØ€C„KÐ/°Ñ?Ô?Ð?ð €DØ˜˜œ 6Ñ*Ô*¨A¨A¨A¨q¨q¨qÐ0Ô1€Jà˜Z¨¨a¨a¨a°°°¨
Ô3Ð4€Gà*×2Ò2°7¸DÈDÐ2ÑQÔQÑ€Eˆ6�6Ø �L ¨¨i¨b¬i¸Ñ.?Ô.?Ô(@×(GÒ(GÈÑ(PÔ(PÐcfÐgÑgÔg€MØ �C”L °Ô0AÑBÔB€MØ€C„KÐ/°Ñ?Ô?Ð?Ð?Ð?ðc Ðr+   