§
    ãóÉjÏ*  ã                   óZ  — d dl Z d dlZd dlmZ d dlZd dlZd dlmZ d dl	m
Z
 d dlmZmZ ej        j        ej        j        ej        j        dœZh d£Z G d„ d	e
¦  «        Zd
„ Zd„ Zd„ Zd„ Zd„ Zd„ Zd„ Zd„ Zdd„Zd„ Zd„ Zdd„Z d d„Z!d„ Z"d!d„Z#d"d„Z$d„ Z%e&dk    r e%¦   «          dS dS )#é    N)ÚList)Úutils)Ú	Converter)Úcommon_specÚtransformer_spec)ÚgeluÚreluÚswish>   Ú ÚdÚnÚndc                   ó4   — e Zd ZdZdedee         fd„Zd„ ZdS )ÚMarianConverterz$Converts models trained with Marian.Ú
model_pathÚvocab_pathsc                 ó"   — || _         || _        dS )z¹Initializes the Marian converter.

        Arguments:
          model_path: Path to the Marian model (.npz file).
          vocab_paths: Paths to the vocabularies (.yml files).
        N)Ú_model_pathÚ_vocab_paths)Úselfr   r   s      ú‚/home/lighthouse/badge-speaker-benchmark/.benchmark-venv-community/lib64/python3.11/site-packages/ctranslate2/converters/marian.pyÚ__init__zMarianConverter.__init__   s   € ð &ˆÔØ'ˆÔÐÐó    c           
      ó   — t          j        | j        ¦  «        }t          |¦  «        }t	          t          t          | j        ¦  «        ¦  «        }|d         }d|d         v }|d         }t          j	        ¦   «         } ||d         dk    d¦  «          ||d         d	k    d
¦  «          ||d          d¦  «          ||t          v d|›dd                     t                               ¦   «         ¦  «        ›d�¦  «          ||t          v d|›dd                     t          ¦  «        ›d�¦  «         |r= ||d         dk    o%|d         dk    o|                     dd¦  «        dk    d¦  «         n< ||d         dk    o%|d         dk    o|                     dd¦  «        dk    d¦  «         |                     ¦   «          |d         }|dk    rdnt!          |¦  «        dz
  }d|v }	t"          j                             |d         |d         f|d          |t          |         |d|	¬!¦  «        }
t)          |
|¦  «         |
                     |d"         ¦  «         |
                     |d         ¦  «         d#|
j        _        |
S )$Nztransformer-ffn-activationr   ztransformer-preprocessztransformer-postprocess-embÚtypeÚtransformerz#Option --type must be 'transformer'ztransformer-decoder-autoregzself-attentionz=Option --transformer-decoder-autoreg must be 'self-attention'ztransformer-no-projectionz3Option --transformer-no-projection is not supportedz$Option --transformer-ffn-activation z. is not supported (supported activations are: z, ú)z%Option --transformer-postprocess-emb z) is not supported (supported values are: ztransformer-postprocessÚdaztransformer-postprocess-topr   zµUnsupported pre-norm Transformer architecture, expected the following combination of options: --transformer-preprocess n --transformer-postprocess da --transformer-postprocess-top nÚdanz¹Unsupported post-norm Transformer architecture, excepted the following combination of options: --transformer-preprocess '' --transformer-postprocess dan --transformer-postprocess-top ''z"transformer-guided-alignment-layerÚlastéÿÿÿÿé   z	enc-depthz	dec-depthztransformer-heads)Úpre_normÚ
activationÚalignment_layerÚalignment_headsÚlayernorm_embeddingr   T)ÚnpÚloadr   Ú_get_model_configÚlistÚmapÚ
load_vocabr   r   ÚConfigurationCheckerÚ_SUPPORTED_ACTIVATIONSÚjoinÚkeysÚ_SUPPORTED_POSTPROCESS_EMBÚgetÚvalidateÚintr   ÚTransformerSpecÚfrom_configÚset_transformer_specÚregister_source_vocabularyÚregister_target_vocabularyÚconfigÚadd_source_eos)r   Úmodelr;   Úvocabsr$   r#   Úpostprocess_embÚcheckr%   r'   Ú
model_specs              r   Ú_loadzMarianConverter._load#   s  € Ý”˜Ô(Ñ)Ô)ˆÝ" 5Ñ)Ô)ˆÝ•c�* dÔ&7Ñ8Ô8Ñ9Ô9ˆàÐ8Ô9ˆ
Ø˜&Ð!9Ô:Ð:ˆØ Ð!>Ô?ˆåÔ*Ñ,Ô,ˆØˆˆf�VŒn Ò-Ð/TÑUÔUÐUØˆØÐ0Ô1Ð5EÒEØKñ	
ô 	
ð 	
ð 	ˆØÐ2Ô3Ð3ØAñ	
ô 	
ð 	
ð 	ˆØÕ0Ð0Ð0ð ˆzˆz˜4Ÿ9š9Õ%;×%@Ò%@Ñ%BÔ%BÑCÔCÐCÐCðEñ	
ô 	
ð 	
ð 	ˆØÕ9Ð9Ð9àˆˆ §	¢	Õ*DÑ EÔ EÐ EÐ EðGñ	
ô 	
ð 	
ð ð 	ØˆEØÐ/Ô0°CÒ7ð IØÐ4Ô5¸Ò=ðIà—J’JÐ<¸bÑAÔAÀSÒHð2ñ		ô 	ð 	ð 	ð ˆEØÐ/Ô0°BÒ6ð HØÐ4Ô5¸Ò>ðHà—J’JÐ<¸bÑAÔAÀRÒGð3ñ		ô 	ð 	ð 	�ŠÑÔÐà Ð!EÔFˆØ /°6Ò 9Ð 9˜"˜"½sÀ?Ñ?SÔ?SÐVWÑ?WˆØ! _Ð4Ðå%Ô5×AÒAØ�KÔ  &¨Ô"5Ð6ØÐ&Ô'ØÝ-¨jÔ9Ø+ØØ 3ð Bñ 
ô 
ˆ
õ 	˜Z¨Ñ/Ô/Ð/Ø×-Ò-¨f°Q¬iÑ8Ô8Ð8Ø×-Ò-¨f°R¬jÑ9Ô9Ð9Ø+/ˆ
ÔÔ(ØÐr   N)Ú__name__Ú
__module__Ú__qualname__Ú__doc__Ústrr   r   rB   © r   r   r   r      sT   € € € € € Ø.Ð.ð( 3ð (°T¸#´Yð (ð (ð (ð (ðIð Ið Ið Ið Ir   r   c                 óv   — | d         }|d d…                               ¦   «         }t          j        |¦  «        }|S )Nzspecial:model.ymlr!   )ÚtobytesÚyamlÚ	safe_load)r=   r;   s     r   r*   r*   o   s:   € ØÐ&Ô'€FØ�C�R�CŒ[× Ò Ñ"Ô"€FÝŒ^˜FÑ#Ô#€FØ€Mr   c           	      óü  — t          | d¬¦  «        5 }g }d }d }t          |¦  «        D �] \  }}|                     d¦  «        }|sŒ|                     d¦  «        r|dd …         }n&|�|dd …         }n|                     dd¦  «        \  }}|�Ð|                     d¦  «        rq|                     d¦  «        r\t          j        d	d
|¦  «        }|dd…         }|                     d¦  «        r&t          t          |dd …         d¬¦  «        ¦  «        }nJ|                     d¦  «        r5|                     d¦  «        r |dd…         }| 
                    dd¦  «        }|�h	 t          |                     ¦   «         ¦  «        }n*# t          $ r}t          d|dz   |fz  ¦  «        |‚d }~ww xY w|                     ||f¦  «         d }d }�Œ¢	 d d d ¦  «         n# 1 swxY w Y   d„ t          |d„ ¬¦  «        D ¦   «         S )Nzutf-8)Úencodingz
z? é   ú:r"   ú"z\\([^x])z\1r!   z\xé   )Úbaseú'z''z"Unexpected format at line %d: '%s'c                 ó   — g | ]\  }}|‘ŒS rH   rH   )Ú.0Ú_Útokens      r   ú
<listcomp>zload_vocab.<locals>.<listcomp>¡   s   € ÐKÐKÐK‘h�a˜ˆEÐKÐKÐKr   c                 ó   — | d         S )Nr   rH   )Úitems    r   ú<lambda>zload_vocab.<locals>.<lambda>¡   s
   € À$ÀqÄ'€ r   )Úkey)ÚopenÚ	enumerateÚrstripÚ
startswithÚrsplitÚendswithÚreÚsubÚchrr5   ÚreplaceÚstripÚ
ValueErrorÚappendÚsorted)ÚpathÚvocabÚtokensrX   ÚidxÚiÚlineÚes           r   r-   r-   v   s„  € å	ˆd˜WÐ	%Ñ	%Ô	%ð '¨ØˆØˆØˆÝ  Ñ'Ô'ð #	ñ #	‰GˆAˆtØ—;’;˜vÑ&Ô&ˆDØð Øà�Š˜tÑ$Ô$ð 1Ø˜Q˜R˜Rœ��ØÐ"Ø˜1˜2˜2”h��à!Ÿ[š[¨¨aÑ0Ô0‘
��sàÐ Ø×#Ò# CÑ(Ô(ð 	5¨U¯^ª^¸CÑ-@Ô-@ð 	5åœF ;°°uÑ=Ô=�EØ! ! B $œK�EØ×'Ò'¨Ñ.Ô.ð =å #¥C¨¨a¨b¨b¬	¸Ð$;Ñ$;Ô$;Ñ <Ô <˜øØ×%Ò% cÑ*Ô*ð 5¨u¯~ª~¸cÑ/BÔ/Bð 5Ø! ! B $œK�EØ!ŸMšM¨$°Ñ4Ô4�EàˆðÝ˜cŸiši™kœkÑ*Ô*�C�CøÝ!ð ð ð Ý$Ø<ÀÀAÁÀt¸}ÑLñô àðøøøøðøøøð
 —’˜s E˜lÑ+Ô+Ð+à�Ø�ùðG#	ð	'ð 'ð 'ñ 'ô 'ð 'ð 'ð 'ð 'ð 'ð 'øøøð 'ð 'ð 'ð 'ðR LÐK¥&¨Ð5IÐ5IÐ"JÑ"JÔ"JÐKÑKÔKÐKs6   ’EGÅ!!FÆGÆ
F*ÆF%Æ%F*Æ* GÇGÇGc                 ó^   — t          | j        |d¦  «         t          | j        |d¦  «         d S )NÚencoderÚdecoder)Úset_transformer_encoderrt   Úset_transformer_decoderru   )ÚspecÚweightss     r   r8   r8   ¤   s0   € Ý˜DœL¨'°9Ñ=Ô=Ð=Ý˜DœL¨'°9Ñ=Ô=Ð=Ð=Ð=r   c           	      óŽ   — t          | ||¦  «         t          | j        ¦  «        D ]\  }}t          ||d||dz   fz  ¦  «         Œd S )Nú%s_l%dr"   )Úset_common_layersr_   ÚlayerÚset_transformer_encoder_layer©rx   ry   Úscoperp   Ú
layer_specs        r   rv   rv   ©   sb   € Ý�d˜G UÑ+Ô+Ð+Ý" 4¤:Ñ.Ô.ð Vð V‰ˆˆ:Ý% j°'¸8ÀuÈaÐRSÉeÀnÑ;TÑUÔUÐUÐUðVð Vr   c           	      óæ   — d| _         t          | ||¦  «         t          | j        ¦  «        D ]\  }}t	          ||d||dz   fz  ¦  «         Œt          | j        |d|z  | j        j        ¬¦  «         d S )NTr{   r"   z%s_ff_logit_out)Úreuse_weight)	Ústart_from_zero_embeddingr|   r_   r}   Úset_transformer_decoder_layerÚ
set_linearÚ
projectionÚ
embeddingsÚweightr   s        r   rw   rw   ¯   s—   € Ø%)€DÔ"Ý�d˜G UÑ+Ô+Ð+Ý" 4¤:Ñ.Ô.ð Vð V‰ˆˆ:Ý% j°'¸8ÀuÈaÐRSÉeÀnÑ;TÑUÔUÐUÐUåØŒØØ˜EÑ!Ø”_Ô+ð	ñ ô ð ð ð r   c                 óx  — | j         }t          |t          ¦  «        s|g}t          |d         ||¦  «         t	          | j        ||d         j        j        d         ¬¦  «         t          | d¦  «        rt          | j
        |d|z  d¬¦  «         t          | d¦  «        rt          | j        |d	|z  ¦  «         d S d S )
Nr   r"   )Údimr'   z%s_embT©r#   Ú
layer_normz%s_top)rˆ   Ú
isinstancer+   Úset_embeddingsÚset_position_encodingsÚposition_encodingsr‰   ÚshapeÚhasattrÚset_layer_normr'   r�   )rx   ry   r€   Úembeddings_specss       r   r|   r|   ½   sð   € Ø”ÐÝÐ&­Ñ-Ô-ð .Ø,Ð-ÐåÐ# AÔ&¨°Ñ7Ô7Ð7ÝØÔ Ð.>¸qÔ.AÔ.HÔ.NÈqÔ.Qðñ ô ð õ ˆtÐ*Ñ+Ô+ð 
ÝØÔ$ØØ�uÑØð		
ñ 	
ô 	
ð 	
õ ˆt�\Ñ"Ô"ð CÝ�t”¨°¸EÑ1AÑBÔBÐBÐBÐBðCð Cr   c                 ón   — t          | j        |d|z  ¦  «         t          | j        |d|z  d¬¦  «         d S )Nú%s_ffnú%s_selfT©Úself_attention)Úset_ffnÚffnÚset_multi_head_attentionrš   ©rx   ry   r€   s      r   r~   r~   Ñ   sM   € ÝˆDŒH�g˜x¨%Ñ/Ñ0Ô0Ð0ÝØÔ˜W i°%Ñ&7Èðñ ô ð ð ð r   c                 ó    — t          | j        |d|z  ¦  «         t          | j        |d|z  d¬¦  «         t          | j        |d|z  ¦  «         d S )Nr—   r˜   Tr™   z
%s_context)r›   rœ   r�   rš   Ú	attentionrž   s      r   r…   r…   Ø   sd   € ÝˆDŒH�g˜x¨%Ñ/Ñ0Ô0Ð0ÝØÔ˜W i°%Ñ&7Èðñ ô ð õ ˜Tœ^¨W°lÀUÑ6JÑKÔKÐKÐKÐKr   Fc                 ó>  — d„ t          d¦  «        D ¦   «         }t          |d         ||d¦  «         t          |d         ||d¦  «         t          |d         ||d¦  «         |r!t          j        | j        d         |¦  «         nb|d         j        | j        d         _        |d         j        | j        d         _        t          j        | j        d         |dd …         ¦  «         t          | j        d	         ||d
¦  «         t          | j        |d|z  ¦  «         d S )Nc                 ó4   — g | ]}t          j        ¦   «         ‘ŒS rH   )r   Ú
LinearSpec)rV   rW   s     r   rY   z,set_multi_head_attention.<locals>.<listcomp>á   s!   € Ð?Ð?Ð?°•KÔ*Ñ,Ô,Ð?Ð?Ð?r   é   r   Úqr"   ÚkrO   Úvr!   Úoz%s_Wo)	Úranger†   r   Úfuse_linearÚlinearr‰   ÚbiasÚset_layer_norm_autor�   )rx   ry   r€   rš   Úsplit_layerss        r   r�   r�   à   s  € Ø?Ð?µe¸A±h´hÐ?Ñ?Ô?€LÝˆ|˜AŒ ¨°Ñ4Ô4Ð4Ýˆ|˜AŒ ¨°Ñ4Ô4Ð4Ýˆ|˜AŒ ¨°Ñ4Ô4Ð4àð <ÝÔ˜$œ+ aœ.¨,Ñ7Ô7Ð7Ð7à ,¨Q¤Ô 6ˆŒ�AŒÔØ*¨1œoÔ2ˆŒ�AŒÔÝÔ˜$œ+ aœ.¨,°q°r°rÔ*:Ñ;Ô;Ð;åˆtŒ{˜2Œ ¨°Ñ4Ô4Ð4Ý˜œ¨°'¸E±/ÑBÔBÐBÐBÐBr   c                 ó”   — t          | j        |d|z  ¦  «         t          | j        ||d¦  «         t          | j        ||d¦  «         d S )Nr—   Ú1Ú2)r­   r�   r†   Úlinear_0Úlinear_1rž   s      r   r›   r›   ñ   sM   € Ý˜œ¨°(¸UÑ2BÑCÔCÐCÝˆtŒ}˜g u¨cÑ2Ô2Ð2ÝˆtŒ}˜g u¨cÑ2Ô2Ð2Ð2Ð2r   c                 ór   — 	 t          | ||d¬¦  «         d S # t          $ r t          | ||¦  «         Y d S w xY w)NTrŒ   )r”   ÚKeyErrorrž   s      r   r­   r­   ÷   sZ   € ð-Ý�t˜W e°dÐ;Ñ;Ô;Ð;Ð;Ð;øÝð -ð -ð -Ý�t˜W eÑ,Ô,Ð,Ð,Ð,Ð,ð-øøøs   ‚ —6µ6c                 ó¢   — |rdnd}||›d|›�                               ¦   «         | _        ||›d|›�                               ¦   «         | _        d S )NÚ_prer   Ú	_ln_scaleÚ_ln_bias)ÚsqueezeÚgammaÚbeta)rx   ry   r€   r#   Úsuffixs        r   r”   r”   þ   sZ   € ØÐ'ˆVˆV R€FØ¨E¨E¨E°6°6Ð:Ô;×CÒCÑEÔE€D„JØ¨%¨%¨%°°Ð8Ô9×AÒAÑCÔC€D„I€I€Ir   r   c                 ó  — |                      |›d|›�¦  «        }|€|                      |›d|›�|¦  «        }n|                     ¦   «         }|| _        |                      |›d|›�¦  «        }|�|                     ¦   «         | _        d S d S )NÚ_WÚ_WtÚ_b)r3   Ú	transposer‰   rº   r¬   )rx   ry   r€   r½   rƒ   r‰   r¬   s          r   r†   r†     s˜   € Ø�[Š[ U U U¨F¨FÐ3Ñ4Ô4€Fà€~Ø—’¨%¨%¨%°°Ð8¸,ÑGÔGˆˆà×!Ò!Ñ#Ô#ˆà€D„Kà�;Š; 5 5 5¨&¨&Ð1Ñ2Ô2€DØÐØ—L’L‘N”NˆŒ	ˆ	ˆ	ð Ðr   c                 ó†   — |                      d|z  ¦  «        | _        | j        €|                      d¦  «        | _        d S d S )Nz%s_WembÚWemb)r3   r‰   rž   s      r   r�   r�     sA   € Ø—+’+˜i¨%Ñ/Ñ0Ô0€D„KØ„{ÐØ—k’k &Ñ)Ô)ˆŒˆˆð Ðr   c                 óV   — |                      dt          |¦  «        ¦  «        | _        d S )NÚWpos)r3   Ú#_make_sinusoidal_position_encodingsÚ	encodings)rx   ry   r‹   s      r   r�   r�     s$   € Ø—[’[ Õ)LÈSÑ)QÔ)QÑRÔR€D„N€N€Nr   é   c                 óÀ  — t          j        |¦  «        }t          j        ddt          j        | ¦  «        dz  z  | z  ¦  «        }t          j        |d¦  «        t          j        |d¦  «        z  }t          j        |¦  «        }t          j        |d d …dd d…f         ¦  «        |d d …d | dz  …f<   t          j        |d d …dd d…f         ¦  «        |d d …| dz  d …f<   |S )Ni'  rO   r"   r   )r(   ÚarangeÚpowerÚexpand_dimsÚ
zeros_likeÚsinÚcos)r‹   Únum_positionsÚ	positionsÚ
timescalesÚposition_encÚtables         r   rÇ   rÇ     sÝ   € Ý”	˜-Ñ(Ô(€IÝ”˜% ¥b¤i°¡n¤n¸Ñ&9Ñ!:¸SÑ!@ÑAÔA€JÝ”> )¨QÑ/Ô/µ"´.ÀÈQÑ2OÔ2OÑO€LÝŒM˜,Ñ'Ô'€EÝœ6 ,¨q¨q¨q°!°$°Q°$¨wÔ"7Ñ8Ô8€Eˆ!ˆ!ˆ!ˆZˆs�a‰xˆZˆ-ÑÝœ6 ,¨q¨q¨q°!°$°Q°$¨wÔ"7Ñ8Ô8€Eˆ!ˆ!ˆ!ˆS�A‰XˆZˆZˆ-ÑØ€Lr   c                  óT  — t          j        t           j        ¬¦  «        } |                      ddd¬¦  «         |                      dddd¬	¦  «         t	          j        | ¦  «         |                      ¦   «         }t          |j        |j	        ¦  «        }| 
                    |¦  «         d S )
N)Úformatter_classz--model_pathTzPath to the model .npz file.)ÚrequiredÚhelpz--vocab_pathsú+z'List of paths to the YAML vocabularies.)rØ   ÚnargsrÙ   )ÚargparseÚArgumentParserÚArgumentDefaultsHelpFormatterÚadd_argumentr   Údeclare_argumentsÚ
parse_argsr   r   r   Úconvert_from_args)ÚparserÚargsÚ	converters      r   Úmainræ   '  sÃ   € ÝÔ$Ý Ô>ðñ ô €Fð ×ÒØ Ð,Jð ñ ô ð ð ×ÒØØØØ6ð	 ñ ô ð õ Ô Ñ'Ô'Ð'Ø×ÒÑÔ€DÝ ¤°Ô1AÑBÔB€IØ×Ò Ñ%Ô%Ð%Ð%Ð%r   Ú__main__)F)r   N)N)rÉ   )'rÜ   rd   Útypingr   Únumpyr(   rK   Úctranslate2.convertersr   Ú ctranslate2.converters.converterr   Úctranslate2.specsr   r   Ú
ActivationÚGELUSigmoidÚRELUÚSWISHr/   r2   r   r*   r-   r8   rv   rw   r|   r~   r…   r�   r›   r­   r”   r†   r�   r�   rÇ   ræ   rC   rH   r   r   ú<module>rñ      s:  ðØ €€€Ø 	€	€	€	à Ð Ð Ð Ð Ð à Ð Ð Ð Ø €€€à (Ð (Ð (Ð (Ð (Ð (Ø 6Ð 6Ð 6Ð 6Ð 6Ð 6Ø ;Ð ;Ð ;Ð ;Ð ;Ð ;Ð ;Ð ;ð Ô"Ô.ØÔ"Ô'ØÔ#Ô)ðð Ð ð 2Ð1Ð1Ð ðVð Vð Vð Vð V�iñ Vô Vð Vðrð ð ð+Lð +Lð +Lð\>ð >ð >ð
Vð Vð Vðð ð ðCð Cð Cð(ð ð ðLð Lð LðCð Cð Cð Cð"3ð 3ð 3ð-ð -ð -ðDð Dð Dð Dð#ð #ð #ð #ð*ð *ð *ðSð Sð Sð Sðð ð ð ð&ð &ð &ð& ˆzÒÐØ€D�F„F€F€F€Fð Ðr   