§
    ÷óÉjí   ã                   óX   — d dl Z d dlmZ d dlmZmZmZ d dlZ G d„ d¦  «        ZdZ	dZ
dS )é    N)Úcached_property)ÚListÚOptionalÚTuplec            
       óâ  — e Zd ZdZ	 	 ddej        dedee         dee         fd„Z	e
defd	„¦   «         Ze
defd
„¦   «         Ze
defd„¦   «         Ze
defd„¦   «         Ze
defd„¦   «         Ze
defd„¦   «         Ze
defd„¦   «         Ze
defd„¦   «         Zedefd„¦   «         Zedee         fd„¦   «         Zdedee         fd„Zdee         defd„Zdee         defd„Ze
dee         fd„¦   «         Zdee         deee         eee                  f         fd„Zdee         deee         eee                  f         fd„Zdee         deee         eee                  f         fd„ZdS )Ú	Tokenizerz-Simple wrapper around a tokenizers.Tokenizer.NÚ	tokenizerÚmultilingualÚtaskÚlanguagec                 ó¸  — || _         |r»|t          vr.t          d|›dd                     t          ¦  «        ›d�¦  «        ‚|t          vr.t          d|›dd                     t          ¦  «        ›d�¦  «        ‚| j                              d|z  ¦  «        | _        | j                              d|z  ¦  «        | _        || _        d S d | _        d | _        d| _        d S )Nú'z'' is not a valid task (accepted tasks: z, ú)z9' is not a valid language code (accepted language codes: z<|%s|>Úen)	r	   Ú_TASKSÚ
ValueErrorÚjoinÚ_LANGUAGE_CODESÚtoken_to_idr   r   Úlanguage_code)Úselfr	   r
   r   r   s        ú{/home/lighthouse/badge-speaker-benchmark/.benchmark-venv-community/lib/python3.11/site-packages/faster_whisper/tokenizer.pyÚ__init__zTokenizer.__init__   sö   € ð #ˆŒàð 	&Ø�6Ð!Ð!Ý �jà�t�t˜TŸYšY¥vÑ.Ô.Ð.Ð.ð0ñô ð ð
 �Ð.Ð.Ý �jà�x�x §¢­?Ñ!;Ô!;Ð!;Ð!;ð=ñô ð ð
 œ×2Ò2°8¸d±?ÑCÔCˆDŒIØ œN×6Ò6°xÀ(Ñ7JÑKÔKˆDŒMØ!)ˆDÔÐÐàˆDŒIØ ˆDŒMØ!%ˆDÔÐÐó    Úreturnc                 ó6   — | j                              d¦  «        S )Nz<|transcribe|>©r	   r   ©r   s    r   Ú
transcribezTokenizer.transcribe*   s   € àŒ~×)Ò)Ð*:Ñ;Ô;Ð;r   c                 ó6   — | j                              d¦  «        S )Nz<|translate|>r   r   s    r   Ú	translatezTokenizer.translate.   ó   € àŒ~×)Ò)¨/Ñ:Ô:Ð:r   c                 ó6   — | j                              d¦  «        S )Nz<|startoftranscript|>r   r   s    r   ÚsotzTokenizer.sot2   s   € àŒ~×)Ò)Ð*AÑBÔBÐBr   c                 ó6   — | j                              d¦  «        S )Nz<|startoflm|>r   r   s    r   Úsot_lmzTokenizer.sot_lm6   r"   r   c                 ó6   — | j                              d¦  «        S )Nz<|startofprev|>r   r   s    r   Úsot_prevzTokenizer.sot_prev:   s   € àŒ~×)Ò)Ð*;Ñ<Ô<Ð<r   c                 ó6   — | j                              d¦  «        S )Nz<|endoftext|>r   r   s    r   ÚeotzTokenizer.eot>   r"   r   c                 ó6   — | j                              d¦  «        S )Nz<|notimestamps|>r   r   s    r   Úno_timestampszTokenizer.no_timestampsB   s   € àŒ~×)Ò)Ð*<Ñ=Ô=Ð=r   c                 ój   — | j                              d¦  «        p| j                              d¦  «        S )Nz<|nospeech|>z<|nocaptions|>r   r   s    r   Ú	no_speechzTokenizer.no_speechF   s6   € àŒ~×)Ò)¨.Ñ9Ô9ð 
¸T¼^×=WÒ=WØñ>
ô >
ð 	
r   c                 ó   — | j         dz   S )Né   )r,   r   s    r   Útimestamp_beginzTokenizer.timestamp_beginL   s   € àÔ! AÑ%Ð%r   c                 óš   — | j         g}| j        �|                     | j        ¦  «         | j        �|                     | j        ¦  «         |S )N)r$   r   Úappendr   )r   Úsequences     r   Úsot_sequencezTokenizer.sot_sequenceP   sH   € à”H�:ˆàŒ=Ð$Ø�OŠO˜DœMÑ*Ô*Ð*àŒ9Ð Ø�OŠO˜DœIÑ&Ô&Ð&àˆr   Útextc                 óD   — | j                              |d¬¦  «        j        S )NF)Úadd_special_tokens)r	   ÚencodeÚids)r   r6   s     r   r9   zTokenizer.encode\   s    € ØŒ~×$Ò$ T¸eÐ$ÑDÔDÔHÐHr   Útokensc                 óT   ‡ — ˆ fd„|D ¦   «         }‰ j                              |¦  «        S )Nc                 ó*   •— g | ]}|‰j         k     ¯|‘ŒS © )r*   )Ú.0Útokenr   s     €r   ú
<listcomp>z$Tokenizer.decode.<locals>.<listcomp>`   s%   ø€ ÐEÐEÐE °E¸D¼HÒ4DÐ4D�uÐ4DÐ4DÐ4Dr   )r	   Údecode)r   r;   Útext_tokenss   `  r   rB   zTokenizer.decode_   s2   ø€ ØEÐEÐEÐE¨&ÐEÑEÔEˆØŒ~×$Ò$ [Ñ1Ô1Ð1r   c                 ó  ‡ — g g}|D ]e}|‰ j         k    r=d|‰ j         z
  dz  d›d�}|                     |¦  «         |                     g ¦  «         ŒJ|d                              |¦  «         Œfd                     ˆ fd„|D ¦   «         ¦  «        S )Nz<|g{®Gáz”?z.2fz|>éÿÿÿÿÚ c                 ór   •— g | ]3}t          |t          ¦  «        r|n‰j                             |¦  «        ‘Œ4S r>   )Ú
isinstanceÚstrr	   rB   )r?   Úsr   s     €r   rA   z4Tokenizer.decode_with_timestamps.<locals>.<listcomp>o   s<   ø€ ÐTÐTÐTÀq•*˜Q¥Ñ$Ô$ÐBˆQˆQ¨$¬.×*?Ò*?ÀÑ*BÔ*BÐTÐTÐTr   )r1   r3   r   )r   r;   Úoutputsr@   Ú	timestamps   `    r   Údecode_with_timestampsz Tokenizer.decode_with_timestampsc   s¶   ø€ Ø�$ˆàð 	*ð 	*ˆEØ˜Ô,Ò,Ð,ØN %¨$Ô*>Ñ">À$Ñ!FÐNÐNÐNÐN�	Ø—’˜yÑ)Ô)Ð)Ø—’˜rÑ"Ô"Ð"Ð"à˜”×"Ò" 5Ñ)Ô)Ð)Ð)à�wŠwØTÐTÐTÐTÈGÐTÑTÔTñ
ô 
ð 	
r   c                 ó4  — t          d¦  «        }|d                     ¦   «         z  }t          d¦  «        }t          d„ |D ¦   «         ¦  «        sJ ‚|                      d¦  «        d         |                      d¦  «        d         h}|t          |¦  «        z   D ]d}|                      |¦  «        |                      d|z   ¦  «        fD ]4}t          |¦  «        d	k    s||v r|                     |d         ¦  «         Œ5Œet          t          |¦  «        ¦  «        S )
u…  
        Returns the list of tokens to suppress in order to avoid any speaker tags or non-speech
        annotations, to prevent sampling texts that are not actually spoken in the audio, e.g.

        - â™ªâ™ªâ™ª
        - ( SPEAKING FOREIGN LANGUAGE )
        - [DAVID] Hey there,

        keeping basic punctuations like commas, periods, question marks, exclamation points, etc.
        u#   "#()*+/:;<=>@[\]^_`{|}~ã€Œã€�ã€Žã€�uK   << >> <<< >>> -- --- -( -[ (' (" (( )) ((( ))) [[ ]] {{ }} â™ªâ™ª â™ªâ™ªâ™ªu   â™©â™ªâ™«â™¬â™­â™®â™¯c              3   óP   K  — | ]!}d t          |¦  «        cxk    odk    nc V — Œ"dS )i@&  i&  N)Úord)r?   Úcs     r   ú	<genexpr>z.Tokenizer.non_speech_tokens.<locals>.<genexpr>ˆ   sD   è è € ÐEÐE°!�6�S ™VœVÐ-Ð-Ò-Ð- vÒ-Ð-Ð-Ð-ÐEÐEÐEÐEÐEÐEr   z -r   z 'ú r0   )	ÚlistÚsplitÚsetÚallr9   ÚlenÚaddÚtupleÚsorted)r   ÚsymbolsÚmiscellaneousÚresultÚsymbolr;   s         r   Únon_speech_tokenszTokenizer.non_speech_tokensr   s*  € õ Ð=Ñ>Ô>ˆØØZ×`Ò`ÑbÔbñ	
ˆõ Ð3Ñ4Ô4ˆÝÐEÐE°}ÐEÑEÔEÑEÔEÐEÐEÐEð —+’+˜dÑ#Ô# AÔ&¨¯ª°DÑ(9Ô(9¸!Ô(<Ð=ˆØ¥ ]Ñ 3Ô 3Ñ3ð 	*ð 	*ˆFà—’˜FÑ#Ô#Ø—’˜C &™LÑ)Ô)ðð *ð *�õ �v‘;”; !Ò#Ð# v°Ð'>Ð'>Ø—J’J˜v aœyÑ)Ô)Ð)øð*õ •V˜F‘^”^Ñ$Ô$Ð$r   c                 óh   — | j         dv r|                      |¦  «        S |                      |¦  «        S )N>   ÚjaÚloÚmyÚthÚzhÚyue)r   Úsplit_tokens_on_unicodeÚsplit_tokens_on_spaces)r   r;   s     r   Úsplit_to_word_tokenszTokenizer.split_to_word_tokens–   s=   € ð ÔÐ!FÐFÐFð ×/Ò/°Ñ7Ô7Ð7à×*Ò*¨6Ñ2Ô2Ð2r   c                 óÀ  — |                       |¦  «        }d}g }g }g }d}|D ]¹}|                     |¦  «         |                       |¦  «        }		 |	                     |¦  «        }
|
|z  }
n# t          $ r d }
Y nw xY w|
�|
t	          |¦  «        k     rJ||
         |k    r>|                     |	¦  «         |                     |¦  «         g }|t	          |	¦  «        z  }Œº||fS )Nu   ï¿½r   )rM   r3   Úindexr   rX   )r   r;   Údecoded_fullÚreplacement_charÚwordsÚword_tokensÚcurrent_tokensÚunicode_offsetr@   ÚdecodedÚreplacement_char_indexs              r   rh   z!Tokenizer.split_tokens_on_unicode¡   s(  € ð ×2Ò2°6Ñ:Ô:ˆØ#ÐàˆØˆØˆØˆàð 	/ð 	/ˆEØ×!Ò! %Ñ(Ô(Ð(Ø×1Ò1°.ÑAÔAˆGð.Ø)0¯ªÐ7GÑ)HÔ)HÐ&Ø&¨.Ñ8Ð&Ð&øÝð .ð .ð .Ø)-Ð&Ð&Ð&ð.øøøð &Ð-Ø&­¨\Ñ):Ô):Ò:Ð:Ø Ð!7Ô8Ð<LÒLÐLà—’˜WÑ%Ô%Ð%Ø×"Ò" >Ñ2Ô2Ð2Ø!#�Ø¥# g¡,¤,Ñ.�øà�kÐ!Ð!s   ÁA*Á*A9Á8A9c                 óÔ  — |                       |¦  «        \  }}g }g }t          ||¦  «        D ]¸\  }}|d         | j        k    }|                     d¦  «        }	|                     ¦   «         t
          j        v }
|s|	s|
st          |¦  «        dk    r+|                     |¦  «         |                     |¦  «         Œ�|d         |z   |d<   |d          	                    |¦  «         Œ¹||fS )Nr   rS   rE   )
rh   Úzipr*   Ú
startswithÚstripÚstringÚpunctuationrX   r3   Úextend)r   r;   ÚsubwordsÚsubword_tokens_listro   rp   ÚsubwordÚsubword_tokensÚspecialÚ
with_spacerz   s              r   ri   z Tokenizer.split_tokens_on_spacesÁ   s  € ð )-×(DÒ(DÀVÑ(LÔ(LÑ%ˆÐ%ØˆØˆå'*¨8Ð5HÑ'IÔ'Ið 		7ð 		7Ñ#ˆG�^Ø$ QÔ'¨4¬8Ò3ˆGØ ×+Ò+¨CÑ0Ô0ˆJØ!Ÿ-š-™/œ/­VÔ-?Ð?ˆKØð 7˜*ð 7¨ð 7µs¸5±z´zÀQ²°Ø—’˜WÑ%Ô%Ð%Ø×"Ò" >Ñ2Ô2Ð2Ð2à! "œI¨Ñ/��b‘	Ø˜B”×&Ò& ~Ñ6Ô6Ð6Ð6à�kÐ!Ð!r   )NN) Ú__name__Ú
__module__Ú__qualname__Ú__doc__Ú
tokenizersr   Úboolr   rI   r   r   Úintr   r!   r$   r&   r(   r*   r,   r.   Úpropertyr1   r   r5   r9   rB   rM   r   r`   rj   rh   ri   r>   r   r   r   r   	   sZ  € € € € € Ø7Ð7ð #Ø"&ð&ð &àÔ'ð&ð ð&ð �sŒmð	&ð
 ˜3”-ð&ð &ð &ð &ð< ð<˜Cð <ð <ð <ñ „_ð<ð ð;˜3ð ;ð ;ð ;ñ „_ð;ð ðC�Sð Cð Cð Cñ „_ðCð ð;˜ð ;ð ;ð ;ñ „_ð;ð ð=˜#ð =ð =ð =ñ „_ð=ð ð;�Sð ;ð ;ð ;ñ „_ð;ð ð>˜sð >ð >ð >ñ „_ð>ð ð
˜3ð 
ð 
ð 
ñ „_ð
ð
 ð& ð &ð &ð &ñ „Xð&ð ð	˜d 3œið 	ð 	ð 	ñ „Xð	ðI˜3ð I 4¨¤9ð Ið Ið Ið Ið2˜T #œYð 2¨3ð 2ð 2ð 2ð 2ð
¨T°#¬Yð 
¸3ð 
ð 
ð 
ð 
ð ð!% 5¨¤:ð !%ð !%ð !%ñ „_ð!%ðF	3Ø˜3”ið	3à	ˆt�CŒy˜$˜t Cœyœ/Ð)Ô	*ð	3ð 	3ð 	3ð 	3ð"Ø˜3”ið"à	ˆt�CŒy˜$˜t Cœyœ/Ð)Ô	*ð"ð "ð "ð "ð@"Ø˜3”ið"à	ˆt�CŒy˜$˜t Cœyœ/Ð)Ô	*ð"ð "ð "ð "ð "ð "r   r   )r   r!   )dÚafÚamÚarÚasÚazÚbaÚbeÚbgÚbnÚboÚbrÚbsÚcaÚcsÚcyÚdaÚdeÚelr   ÚesÚetÚeuÚfaÚfiÚfoÚfrÚglÚguÚhaÚhawÚheÚhiÚhrÚhtÚhuÚhyÚidÚisÚitrb   ÚjwÚkaÚkkÚkmÚknÚkoÚlaÚlbÚlnrc   ÚltÚlvÚmgÚmiÚmkÚmlÚmnÚmrÚmsÚmtrd   ÚneÚnlÚnnÚnoÚocÚpaÚplÚpsÚptÚroÚruÚsaÚsdÚsiÚskÚslÚsnÚsoÚsqÚsrÚsuÚsvÚswÚtaÚteÚtgre   ÚtkÚtlÚtrÚttÚukÚurÚuzÚviÚyiÚyorf   rg   )ry   Ú	functoolsr   Útypingr   r   r   r†   r   r   r   r>   r   r   ú<module>ré      sŽ   ðØ €€€à %Ð %Ð %Ð %Ð %Ð %Ø (Ð (Ð (Ð (Ð (Ð (Ð (Ð (Ð (Ð (à Ð Ð Ð ðJ"ð J"ð J"ð J"ð J"ñ J"ô J"ð J"ðZ
€ð
e€€€r   