bi' d dl Z d dlZd dlmZ ddlmZ ddlmZ ddlm Z ddl
m Z m Z ddl
mZmZ dd lmZ d
d lmZ e r
d dlZd dlmZ d Zd
ZdZdZdZ ddZ G d d Z dS ) N)Image )create_causal_mask)_get_model_class)
AutoConfig) MODEL_FOR_PRETRAINING_MAPPING
MODEL_MAPPING)PROCESSOR_MAPPING_NAMES
AutoProcessor)
AutoTokenizer )is_torch_availablez[92mz[93mz[0mu ■u ⬚![]()
c
^ j dk rdddddf j dk rddddddf t t d D }d}g }t D ]F\ } | k r |s }df< |dk r*| k s dz
k rdz
k rdz
d||f< d}Gd fdt D d fd
t D }
||dk } t j | d d z | z
| d d t j d|dz | } t j |
t t t dt t t d}
| d |
z d |d z z d t |
dz z }|dz
}| | g }t D ]\ }||f dk rg| d t' t) | t t) D z| t' t) | t t) t' t- t& t/ | }|D ]P}| 7|d z d z d | z dz d | z nd Qt D ]\ t1 | }v rt | t n|}d fdt D }d}3d fdt D }| | dt) d d | d| d | S )z
Generates an attention matrix from a given attention mask.
Optionally applies a sliding window mask (e.g., for Gemma2/3) and
marks regions where image tokens occur based on the specified `img_token`.
r N c 3 N K | ] }t t | V !d S N)lenrepr).0words b/root/projects/butler/venv/lib/python3.11/site-packages/transformers/utils/attention_visualizer.py z6generate_attention_matrix_from_mask..8 s. < g | ]}d |z
cxk rk rn ndnd S )r r
)r jisliding_windows r
zBgenerate_attention_matrix_from_mask...I sB \ \ \aa1q5&A&A&A&A>&A&A&A&A&A \ \ \r )range)r r nr! s @r r" z7generate_attention_matrix_from_mask..I s= o o oab \ \ \ \ \SXYZS[S[ \ \ \ o o or c 3 K | ]Z}d |f rt t t n4|d k rt t t nd |f rt nt V [dS r NYELLOW BLACK_SQUARERESETGREEN WHITE_SQUARE)r r masks r r z6generate_attention_matrix_from_mask..K s
1: 6(<((((
66
,|
,U
,
,
, 1:\\
r )
boundariesz: i == j (diagonal) z: token_type_idszAttention MatrixzSliding Window Maskc 2 g | ]}t | t S r )r) r+ )r ks r r" z7generate_attention_matrix_from_mask..i s( #e#e#eav$9q$9%$9$9#e#e#er z | c 3 K | ]h}| v r%|f rv rt t t n4|k rt t t n|f rt nt V id S r r( )r r r img_tokenr. r wordss r r z6generate_attention_matrix_from_mask..x s
E!H$$ad$ T8I8I ,| ,U , , , Avv 0<0000 AqDz
r c 3 K | ]x}| v r1v r-d f d |f k rt t t n8|k rt t t n | rt nt V ydS r' r( )r r r r6 sliding_window_masktoken_type_bucketsr r7 s r r z6generate_attention_matrix_from_mask.. s * *
a((Y$->->CUVWYZVZC[_qrsuvrv_wCwCw 0<0000 66 4|4U444 'q)!,"\\! * * * * * *r z:
)intndimr max enumerater# jointorchwherecumsumboolarange bucketizer, r* r+ r) appendljustliststrrjustmapzipr )r7 r. r6 r! token_type_idsimage_seq_lengthmax_word_length
first_img_idxoutputr3 row_dummy
is_specialr1 legendf_stringvertical_headeridxrow word_repr colored_word row_displaysliding_window_rowr r$ r9 r: r s ```` @@@@@r #generate_attention_matrix_from_maskr^ ( sv 88::D yA~~Aqqq!!!G} yA~~Aq!!!QQQJ E
A<>-> MDAJ 1 !y..AQJJAEzzQ56D q -/1 2 M ! o o o o ofklmfnfn o o o q I !#q(
"[
"
"2
&
&
*Z
7 = = ? ? AVAVWYAZAZ\]
\!%5%9;KLL
"_-?J W W W
o|
oU
o
o&
o,
oX]
o
o
oF
MM#, o)*-?-E-Ec)nnXYFY-Z-ZZH!))
MM(O u%% F F T S>Q " "#e#e4CWZ[^_`[a[aWbWbHcHcCdCd#e#e#e f f f f " "4Cs3q66{{(C(C#D#D E E E E3tS/%:;;<._ModelWrapperc t j | t j dd | _ || _ d S )Nr
)nnModule__init__Linear dummy_moduleconfig)selfrk ra s r rh z@AttentionMaskVisualizer.__init__.._ModelWrapper.__init__ s5 ""4((($&IaOO! $ r N)__name__
__module__ __qualname__rh r r r
_ModelWrapperrd s#
%
%
%
%
%r rp )r from_pretrained image_tokenhasattrget_text_configgetattrr! r r Exceptionr
ValueError
mapped_clsrf rg modeltodtyperepo_idrk )rl ra rk rx rp s r rh z AttentionMaskVisualizer.__init__ s: +J77" 6))++-= > > \")&*@*@*B*BDTVZ"["[D Q)&-@@JJ Q Q Q)&2OPPJJJ Q d: d d dee e$ % % % % %J % % % #]6:66
fl###! s 'A= = B B r4 input_sentencec 4 | || d S )N)suffix)visualize_attention_mask)rl r} r s r __call__z AttentionMaskVisualizer.__call__ s! %%nV%DDDDDr c
| j }i }d }| j j t v r,d}t j t
j t j |d j
}d}t j | j
| }t |d r|j }n&|j |j g d }|r| d| } ||||d
} |j |j g d | _ | d }
d | v r | d |d < |j | d
d } n`t) j | j
} | | } | |d d }
|
t- d| j j d d|j _ | |
j \ }
}t5 j |
||j j f| j j }t5 j | }t? |j ||
|d }|| }
n>|
! d ! d " |
d|| }
dtG d| j j d| j$ dz z }d}tK d| tK dd| j j d| j
d | j$ j& ' tG | z d z |z tK | tQ | |
| j tS | j d d | d | }tK | tK | d S ) Nzchttps://huggingface.co/datasets/huggingface/documentation-images/resolve/main/bee.jpg?download=trueT)follow_redirectsr0 )rO rr r r pt)imagestextr return_tensorsattention_maskrN input_ids)r z Model type z) does not support attention visualizationeager)r{ )rk
inputs_embedsr cache_positionpast_key_valuesr
z##z Attention visualization for z | r r; z" Attention visualization for [1m:z[0m z r! )r6 r! rN rO )*ry rk
model_typer
r openioBytesIOhttpxgetcontentr rq r| rs rr tokenizerconvert_ids_to_tokensimage_token_idreplacer tokenizerw _attn_implementationtrainshaperA zeros hidden_sizer{ rE r rD unsqueezeexpandr rx printrm centerr^ ru )rl r} r ry kwargsrO img processorrr inputsr tokensr
batch_size
seq_lengthr r causal_masktop_bottom_border side_borderrV s r r z0AttentionMaskVisualizer.visualize_attention_mask s
; !%< < <wC*RZ #(M(M(M(U V VWWC %5dlUefffIy-00
g ' 3 ' 1 G GIaHb c cde f
N!/!7!7!M!M Ycv^bcccF ( 2 H H)JbIc d def gD #$4 5N 6))+12B+C'( (>>vk?RST?UVVFF%5dlCCI ''77F &Y~d K K KL\ ]N % !pt{/E!p!p!pqqq,3 )
!/!5
J ZU\=U$V^b^h^nooo
j11(<'))
" ) . . 0 0 0NN + 5 5a 8 8 B B1 E E L LZYZ\fhr s sN [t{/E[[$/[[ \ \_` `
&$&&'''
dk6Lt|eiete} G G%&&
"$%%%6 &"4;0@$GG!::&677 -
h
"$%%%%%r N)r4 )rm rn ro rJ rh r r r r r r` r` s} 3 2E Es E E E EJ& J&s J& J& J& J& J& J&r r` )r NNN) r r PILr
masking_utilsr models.auto.auto_factoryr models.auto.configuration_autor models.auto.modeling_autor r models.auto.processing_autor
r models.auto.tokenization_autor import_utilsr rA torch.nnrf r, r) r+ r* r- r^ r` r r r r sZ
. . . . . . 7 7 7 7 7 7 7 7 7 7 7 7 T T T T T T T T P P P P P P P P 9 9 9 9 9 9 , , , , , , LLL `di i i i Xg& g& g& g& g& g& g& g& g& g&r