
    bi                       d dl mZ d dlZd dlZd dlZd dlZd dlmZmZ d dl	m	Z	 d dl
mZmZmZ d dlmZ d dlmZ d dlmZ d d	lmZ d d
lmZmZmZ d dlZd dlZd dlmZmZ d dlmZ  d dlm!Z" d dl#m$Z$m%Z% d dl&m'Z' d dlm(Z( d dl)m*Z* d dlm+Z+ d dl,m-Z- d dl.m/Z/ d dl0m1Z1m2Z2 d dl3m4Z4 d dl5m6Z7 d dl8m9Z9m:Z: d dl;m<Z< d dl=m>Z>m?Z?m@Z@  e@            rd dlAmBZBmCZCmDZDmEZEmFZF  ejG        eH          ZIerd dlJmKZK d dlLmMZM d dlNmOZO  G d d e+          ZP e4d!           G d" d#eP                      ZQg d$ZRg d%ZSd7d(ZTd8d,ZUd9d0ZVe G d1 d2e                      ZWd:d6ZXdS );    )annotationsN)Counterdefaultdict)copy)	dataclassfieldfields)Path)python_version)pformatindent)TYPE_CHECKINGAnyLiteral)CardData	ModelCard)dataset_info)
model_info)
EvalResulteval_results_to_model_index)	yaml_dump)nn)tqdm)TrainerCallback)CodeCarbonCallback)make_markdown_table)TrainerControlTrainerState)
deprecated__version__)RouterStaticEmbedding)$SentenceTransformerTrainingArguments)fullnameis_accelerate_availableis_datasets_available)DatasetDatasetDictIterableDatasetIterableDatasetDictValue)SentenceEvaluator)SentenceTransformer)SentenceTransformerTrainerc                  @     e Zd Zd fdZddZddZddZddZ xZS )$SentenceTransformerModelCardCallbackdefault_args_dictdict[str, Any]returnNonec                V    t                                                       || _        d S N)super__init__r3   )selfr3   	__class__s     [/root/projects/butler/venv/lib/python3.11/site-packages/sentence_transformers/model_card.pyr:   z-SentenceTransformerModelCardCallback.__init__0   s'    !2    argsr%   stater   controlr   modelr/   trainerr0   c                f   |j                             d           d |j        j        D             }|r|d         |j         _        |j        r;|j                             |j        |j         j        |j        d          |j         _        |j	        r;|j                             |j	        |j         j
        |j        d          |j         _
        t          |j                  }|j                             |           |j         j        s,|j	        p|j        x}	r|j                             |	           d S d S d S )Ngenerated_from_trainerc                <    g | ]}t          |t                    |S  )
isinstancer   ).0callbacks     r=   
<listcomp>zDSentenceTransformerModelCardCallback.on_init_end.<locals>.<listcomp>@   s9     
 
 
!:V^`rKsKs

 
 
r>   r   traineval)model_card_dataadd_tagscallback_handler	callbackscode_carbon_callbacktrain_datasetextract_dataset_metadatatrain_datasetslosseval_dataseteval_datasets
get_losses
set_losseswidgetset_widget_examples)
r;   r?   r@   rA   rB   rC   kwargsrQ   lossesdatasets
             r=   on_init_endz0SentenceTransformerModelCardCallback.on_init_end4   s[    	&&'?@@@
 
%,%=%G
 
 
	  	F9B1E!6   	383H3a3a%u'<'KW\[b4 4E!0  	272G2`2`$e&;&I7<Y_3 3E!/ GL))((000 $+ 	?G<P<iT[Ti1i 	?!55g>>>>>	? 	? 	? 	?r>   c                     h d|                                 }fd|                                D             |j        _         fd|                                D             |j        _        d S )N>   do_evaldo_testdo_trainrun_name	hub_token	report_to
eval_delay
eval_steps
output_dir
save_stepslogging_dirlogging_stepssave_strategylogging_strategysave_total_limitgreater_is_betterpush_to_hub_tokensamples_per_labelshow_progress_barlogging_first_stepevaluation_strategymetric_for_best_modelc                $    i | ]\  }}|v	||S rG   rG   )rI   keyvalueignore_keyss      r=   
<dictcomp>zGSentenceTransformerModelCardCallback.on_train_begin.<locals>.<dictcomp>z   s0     5
 5
 5
%3c>T>TC>T>T>Tr>   c                X    i | ]&\  }}|v	|j         v |j         |         k    #||'S rG   )r3   )rI   ry   rz   r{   r;   s      r=   r|   zGSentenceTransformerModelCardCallback.on_train_begin.<locals>.<dictcomp>}   sU     =
 =
 =
U+%%#1G*G*GEUYUkloUpLpLp LpLpLpr>   )to_dictitemsrN   all_hyperparametersnon_default_hyperparameters)r;   r?   r@   rA   rB   r]   	args_dictr{   s   `      @r=   on_train_beginz3SentenceTransformerModelCardCallback.on_train_beginY   s    
 
 
0 LLNN	5
 5
 5
 5
)2):):5
 5
 5
1=
 =
 =
 =
 =
'oo//=
 =
 =
999r>   metricsdict[str, float]c                j   fdD             }t          |          dk    rd|v r
d|d         i}|j        j        rH|j        j        d         d         |j        k    r'|j        j        d                             |           d S |j        j                            |j        |j        d|           d S )Nc                    i | ]c}|                     d           |                    d          ,d                    |                    d          dd                   |         dS )eval__loss _   N)
startswithendswithjoinsplit)rI   ry   r   s     r=   r|   zDSentenceTransformerModelCardCallback.on_evaluate.<locals>.<dictcomp>   sv     
 
 
~~g&&
 ,/<<+@+@
HHSYYs^^ABB'(('#,
 
 
r>   r   rV   Validation LossStepEpochr   )lenrN   training_logsglobal_stepupdateappendepoch)r;   r?   r@   rA   rB   r   r]   	loss_dicts        `  r=   on_evaluatez0SentenceTransformerModelCardCallback.on_evaluate   s    
 
 
 

 
 
	
 y>>Q6Y#6#6*If,=>I!/	%3B7?5CTTT!/3::9EEEEE!/66"[!-       r>   logsc                   dht          |          z  }|r|j        j        rO|j        j        d         d         |j        k    r.||                                         |j        j        d         d<   nE|j        j                            |j        |j        ||                                         d           |j        j        |D ]}d|v rd|j        _        d S d S )NrV   r   r   Training Loss)r   r   r   ndcgT)setrN   r   r   popr   r   ir_model)	r;   r?   r@   rA   rB   r   r]   keysry   s	            r=   on_logz+SentenceTransformerModelCardCallback.on_log   s     x#d))# 	%3)7;FCuGXXXKOPTPXPXPZPZK[%3B7HH%3::!& % 1)-dhhjj)9     )1 : :S==59E)2 21: :r>   )r3   r4   r5   r6   )r?   r%   r@   r   rA   r   rB   r/   rC   r0   r5   r6   )
r?   r%   r@   r   rA   r   rB   r/   r5   r6   )r?   r%   r@   r   rA   r   rB   r/   r   r   r5   r6   )r?   r%   r@   r   rA   r   rB   r/   r   r   r5   r6   )	__name__
__module____qualname__r:   r`   r   r   r   __classcell__r<   s   @r=   r2   r2   /   s        3 3 3 3 3 3#? #? #? #?J(
 (
 (
 (
T   <: : : : : : : :r>   r2   zThe `ModelCardCallback` has been renamed to `SentenceTransformerModelCardCallback` and the former is now deprecated. Please use `SentenceTransformerModelCardCallback` instead.c                       e Zd Z fdZ xZS )ModelCardCallbackc                :     t                      j        |i | d S r8   )r9   r:   )r;   r?   r]   r<   s      r=   r:   zModelCardCallback.__init__   s%    $)&)))))r>   )r   r   r   r:   r   r   s   @r=   r   r      s8        * * * * * * * * *r>   r   )languagelicenselibrary_nametagsdatasetsr   pipeline_tagr[   model-indexco2_eq_emissions
base_model)rB   rC   eval_results_dictr5   r4   c                     t                      t          t          j        t          j        d} t                      rddlm} || d<   t                      rddlm} || d<   ddl	m} || d<   | S )N)pythonsentence_transformerstransformerstorchr   r!   
accelerater   
tokenizers)
r   sentence_transformers_versionr   r"   r   r'   r   r(   r   r   )versionsaccelerate_versiondatasets_versiontokenizers_versions       r=   get_versionsr      s     ""!>$0"	 H    4@@@@@@!3 0<<<<<</<<<<<</H\Or>   rz   float | int | strr   c                P    t          | t                    rt          | d          S | S )N   )rH   floatroundrz   s    r=   
format_logr      s'    % UALr>   rV   nn.Module | dict[nn.Module]list[nn.Module]c                   t          | t                    r"t          |                                           }n| g}d}|t	          |          k     r||         } t          | d          r#| j        |vr|                    | j                   t          | d          r#| j        |vr|                    | j                   t          | d          r#| j	        |vr|                    | j	                   |dz  }|t	          |          k     |S )Nr   rV   document_regularizerquery_regularizerr   )
rH   dictlistvaluesr   hasattrrV   r   r   r   )rV   r^   loss_idxs      r=   rY   rY      s   $ dkkmm$$ H
S[[
 
 h4   	%TYf%<%<MM$)$$$4/00 	5T5NV\5\5\MM$34444,-- 	2$2HPV2V2VMM$0111A S[[
 
  Mr>   c                  &   e Zd ZU dZ ee          Zded<   dZded<   dZ	ded<   dZ
ded	<    ee          Zd
ed<    ee          Zd
ed<   dZded<    ed           Zded<   dZded<    ed          Zded<    edd          Zded<    edd          Zded<    eed          Zded<    eed          Zded<    eed          Zd ed!<    eed          Zd"ed#<    eed          Zd
ed$<    edd          Zded%<    eed          Zd
ed&<    edd          Zd'ed(<    eed          Zd)ed*<    edd          Zd+ed,<    eedd-          Zd.ed/<    eddd0          Zd1ed2<    eddd0          Z ded3<    edd          Z!ded4<    ed5d          Z"d6ed7<    ed8d          Z#ded9<    ed:d          Z$ded;<    ee%d          Z&d)ed<<    e e'e(          j)        d=z  dd0          Z*d>ed?<    eddd0          Z+d@edA<   dzdDZ,	 d{d|dHZ-d}dKZ.d~dMZ/ddPZ0	 dddVZ1ddXZ2d{ddZZ3dd]Z4ddbZ5ddgZ6ddiZ7ddjZ8d{ddlZ9ddmZ:ddnZ;ddoZ<dzdpZ=ddqZ>dr Z?ddsZ@dduZAddvZBdzdwZCddxZDd{ddyZEdS ) SentenceTransformerModelCardDataa	  A dataclass storing data used in the model card.

    Args:
        language (`Optional[Union[str, List[str]]]`): The model language, either a string or a list,
            e.g. "en" or ["en", "de", "nl"]
        license (`Optional[str]`): The license of the model, e.g. "apache-2.0", "mit",
            or "cc-by-nc-sa-4.0"
        model_name (`Optional[str]`): The pretty name of the model, e.g. "SentenceTransformer based on microsoft/mpnet-base".
        model_id (`Optional[str]`): The model ID when pushing the model to the Hub,
            e.g. "tomaarsen/sbert-mpnet-base-allnli".
        train_datasets (`List[Dict[str, str]]`): A list of the names and/or Hugging Face dataset IDs of the training datasets.
            e.g. [{"name": "SNLI", "id": "stanfordnlp/snli"}, {"name": "MultiNLI", "id": "nyu-mll/multi_nli"}, {"name": "STSB"}]
        eval_datasets (`List[Dict[str, str]]`): A list of the names and/or Hugging Face dataset IDs of the evaluation datasets.
            e.g. [{"name": "SNLI", "id": "stanfordnlp/snli"}, {"id": "mteb/stsbenchmark-sts"}]
        task_name (`str`): The human-readable task the model is trained on,
            e.g. "semantic textual similarity, semantic search, paraphrase mining, text classification, clustering, and more".
        tags (`Optional[List[str]]`): A list of tags for the model,
            e.g. ["sentence-transformers", "sentence-similarity", "feature-extraction"].
        local_files_only (`bool`): If True, don't attempt to find dataset or base model information on the Hub.
            Defaults to False.
        generate_widget_examples (`bool`): If True, generate widget examples from the evaluation or training dataset,
            and compute their similarities. Defaults to True.

    .. tip::

        Install `codecarbon <https://github.com/mlco2/codecarbon>`_ to automatically track carbon emission usage and
        include it in your model cards.

    Example::

        >>> model = SentenceTransformer(
        ...     "microsoft/mpnet-base",
        ...     model_card_data=SentenceTransformerModelCardData(
        ...         model_id="tomaarsen/sbert-mpnet-base-allnli",
        ...         train_datasets=[{"name": "SNLI", "id": "stanfordnlp/snli"}, {"name": "MultiNLI", "id": "nyu-mll/multi_nli"}],
        ...         eval_datasets=[{"name": "SNLI", "id": "stanfordnlp/snli"}, {"name": "MultiNLI", "id": "nyu-mll/multi_nli"}],
        ...         license="apache-2.0",
        ...         language="en",
        ...     ),
        ... )
    )default_factoryzstr | list[str] | Noner   N
str | Noner   
model_namemodel_idlist[dict[str, str]]rU   rX   zjsemantic textual similarity, semantic search, paraphrase mining, text classification, clustering, and morestr	task_namec                 
    g dS )N)sentence-transformerssentence-similarityzfeature-extractiondenserG   rG   r>   r=   <lambda>z)SentenceTransformerModelCardData.<lambda>@  s     !
 !
 !
 r>   zlist[str] | Noner   Fboollocal_files_onlyT)defaultgenerate_widget_examples)r   initr   base_model_revision)r   r   r4   r   r   z.dict[SentenceEvaluator, dict[str, Any]] | Noner   zlist[dict[str, float]]r   r[   predict_examplelabel_example_listzCodeCarbonCallback | NonerR   dict[str, str]	citationsz
int | Nonebest_model_step)r   r   repr	list[str]r   )r   r   r   bool | Noner   similarities
first_saver   intwidget_stepr   r   r   r   versionzmodel_card_template.mdr
   template_pathzSentenceTransformer | NonerB   r5   r6   c                v   t          | j        t                    r| j        g| _        |                     | j                  | _        |                     | j                  | _        | j        rJ| j                            d          dk    r.t          	                    d| j        d           d | _        d S d S d S )N/r   zThe provided z} model ID should include the organization or user, such as "tomaarsen/mpnet-base-nli-matryoshka". Setting `model_id` to None.)
rH   r   r   validate_datasetsrU   rX   r   countloggerwarningr;   s    r=   __post_init__z.SentenceTransformerModelCardData.__post_init__h  s    dmS)) 	,!]ODM"44T5HII!33D4FGG= 	!T]0055::NN^ ^ ^ ^   !DMMM	! 	!::r>   dataset_listlist[dict[str, Any]]infer_languagesc                d   || j          }g }|D ]}d|vrd|v r|d         |d<   d|v r| j        s	 t          |d                   }|j        rg|red|j        v r\|j                            d          }|@t          |t                    r|g}|D ]%}|| j         vr| j                             |           &|j        | j	        vr| j	                            |j                   n7# t          $ r* t                              d|d         d           |d= Y nw xY w|                    |           !|S )a!  
        Validate (i.e. check if the dataset IDs exist on the Hub) and process a list of dataset dictionaries.

        Args:
            dataset_list (list[dict[str, Any]]): List of dataset metadata dictionaries.
            infer_languages (bool | None, optional): Whether to infer languages from the dataset information.
                If None (default), languages will be inferred only if `self.language` is empty.

        Returns:
            list[dict[str, Any]]: The validated and possibly updated list of dataset dictionaries.
        Nnameidr   zThe dataset `id` z5 does not exist on the Hub. Setting the `id` to None.)r   r   get_dataset_infocardDatagetrH   r   r   r  r   	Exceptionr   r   )r;   r  r  output_dataset_listr_   infodataset_languager   s           r=   r   z2SentenceTransformerModelCardData.validate_datasetsw  s    ""&-/O # 	0 	0GW$$7??&-dmGFOwt'<6+GDM::D } C CZ4==X=X+/=+<+<Z+H+H(+7)*:C@@ F4D3E 0,< C C#+4=#@#@$(M$8$8$B$B$B wdm33,,TW555# ! & & &NNrGDMrrr    	&&  &&w////""s   C""1DDr^   r   c                   ddi}|D ]'}	 |j         ||j        j        <   # t          $ r Y $w xY wt	          t
                    }|                                D ] \  }}||                             |           !ddfd|                                D             | _        | 	                    d	 d
 |D             D                        d S )NzSentence Transformersa  
@inproceedings{reimers-2019-sentence-bert,
    title = "Sentence-BERT: Sentence Embeddings using Siamese BERT-Networks",
    author = "Reimers, Nils and Gurevych, Iryna",
    booktitle = "Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing",
    month = "11",
    year = "2019",
    publisher = "Association for Computational Linguistics",
    url = "https://arxiv.org/abs/1908.10084",
}
r^   r   r5   r   c                    t          |           dk    r)d                    | d d                   dz   | d         z   S | d         S )Nr   , r   z and r   )r   r   )r^   s    r=   	join_listz>SentenceTransformerModelCardData.set_losses.<locals>.join_list  sC    6{{Qyy--7&*DD!9r>   c                .    i | ]\  }} |          |S rG   rG   )rI   citationr^   r  s      r=   r|   z?SentenceTransformerModelCardData.set_losses.<locals>.<dictcomp>  s*    iii:J(F))F++Xiiir>   c                    g | ]}d | S )zloss:rG   rI   rV   s     r=   rK   z?SentenceTransformerModelCardData.set_losses.<locals>.<listcomp>  s    eee$~t~~eeer>   c                (    i | ]}|j         j        |S rG   )r<   r   r  s     r=   r|   z?SentenceTransformerModelCardData.set_losses.<locals>.<dictcomp>  s     2d2d2dUY4>3JD2d2d2dr>   )r^   r   r5   r   )
r  r<   r   r
  r   r   r   r   r   rO   )r;   r^   r   rV   inverted_citationsr  r  s         @r=   rZ   z+SentenceTransformerModelCardData.set_losses  s   # 
&
	  	 	D59]	$.122   (..'oo// 	6 	6ND(x(//5555	 	 	 	
 jiiiN`NfNfNhNhiiiee2d2d]c2d2d2deeefffffs    
--stepc                    || _         d S r8   )r   )r;   r  s     r=   set_best_model_stepz4SentenceTransformerModelCardData.set_best_model_step  s    #r>   r_   Dataset | DatasetDictc                8   t          |t          t          f          rd S t          |t                    rt	          |          }g | _        t          t          j        t          |
                                          d                    }d}t          |                                ddd          D ]\  }}t          ||         t                    r"d	 ||         j                                        D             }||                             |          }t          |          }|d
k    r|i }	t!          |                    t          j        t'          |          t)          ||                                        D ]3\  }
}t+          d |                                D                       |	|
<   4t-          t/          |	                                d            \  }}|d |         t          ||d          d d d                   }}|D ]}
d ||
                                         D             }t          |          dk     r|r|                                }d ||                                         D             }t          |          dk    r|                    |           n|                    |d                    t          |          dk     r|t          |          dk     rd |D             }| j        dk    rP| j                            |d
         t          j        |dd          t          |          dz
            d           n.| j                            dt          j        |          i           |d d         | _        d S )N)r_      )k  zComputing widget examplesexampleF)descunitleavec                ~    g | ]:\  }}t          |t                    st          |t                    r|j        d v 8|;S )>   stringlarge_string)rH   r   r-   dtype)rI   columnfeatures      r=   rK   zHSentenceTransformerModelCardData.set_widget_examples.<locals>.<listcomp>  s_       #FGgt,, w..	 4;=D^3^3^  4_3^3^r>   r   c              3  F   K   | ]\  }}|d k    t          |          V  dS )dataset_nameNr   rI   ry   rz   s      r=   	<genexpr>zGSentenceTransformerModelCardData.set_widget_examples.<locals>.<genexpr>  s8      "h"h*#uRUYgRgRg3u::RgRgRgRg"h"hr>   c                    | d         S )Nr   rG   )xs    r=   r   zFSentenceTransformerModelCardData.set_widget_examples.<locals>.<lambda>  s
    AaD r>   ry   r   c                $    g | ]\  }}|d k    |S r+  rG   rI   ry   sentences      r=   rK   zHSentenceTransformerModelCardData.set_widget_examples.<locals>.<listcomp>  s(    mmm-#xWZ^lWlWlXWlWlWlr>   r   c                $    g | ]\  }}|d k    |S r3  rG   r4  s      r=   rK   zHSentenceTransformerModelCardData.set_widget_examples.<locals>.<listcomp>  s/     % % %%2S(Y\`nYnYnYnYnYnr>   r   c                    g | ]@}t          |t                    r't          |                                          d          n|AS )r   )rH   r   r   r   rI   r5  s     r=   rK   zHSentenceTransformerModelCardData.set_widget_examples.<locals>.<listcomp>  sQ       _g*Xt2L2LZD**++A..RZ  r>   r   )source_sentence	sentencestext)rH   r+   r,   r)   r*   r[   r   randomchoicesr   r   r   r   featuresselect_columnsr   	enumerateselectsamplerangeminsumzipsortedr   extendr   r   choicer   )r;   r_   dataset_namesnum_samples_to_checkr+  num_samplescolumnsstr_datasetdataset_sizelengthsidxrB  indicesr   target_indicesbackup_indicesr:  
backup_idxbackup_samples                      r=   r\   z4SentenceTransformerModelCardData.set_widget_examples  s   g1DEFF 	Fgw'' 	3!'222GtGLLNN/C/Cq I I IJJ#)-!!(C)[`*
 *
 *
 @	5 @	5%L+ ',/AA  '.|'<'E'K'K'M'M  G ",/>>wGGK{++Lq  G(""6=|1D1DL`bnHoHo#p#p#pqq    i iV  #"h"h"h"h"hhhfW]]__..IIIJJGQ-4\k\-BDQ\Q]Q]I^_c_cac_cIdDeDeNN & "5 "5mm;s;K;Q;Q;S;Smmm	)nnq((^(!/!3!3!5!5J% %6A*6M6S6S6U6U% % %M =))Q..!((7777 "((q)9::: )nnq((^( y>>A%% kt  	 $(===K&&/8|)/y}IYZHZ)[)[)[     K&&i0H0H'IJJJ'0!}$$E"5=@	5 @	5r>   r   	evaluatorr.   r   r   c                   ddl m} t          |          | j        |<   t	          |d          r|j        xrt          ||          rd |j        D             nt          t                    rgfd|	                                D             }| j
        r9| j
        d         d         |k    r"| j
        d                             |           d S | j
                            ||d|           d S d S d S )	Nr   )SequentialEvaluatorprimary_metricc                    g | ]	}|j         
S rG   )rZ  )rI   sub_evaluators     r=   rK   zKSentenceTransformerModelCardData.set_evaluation_metrics.<locals>.<listcomp>"  s    "j"j"jM=#?"j"j"jr>   c                $    i | ]\  }}|v 	||S rG   rG   )rI   ry   rz   primary_metricss      r=   r|   zKSentenceTransformerModelCardData.set_evaluation_metrics.<locals>.<dictcomp>&  s+    #k#k#k:3TW[jTjTjCTjTjTjr>   r   r   r   ) sentence_transformers.evaluationrY  r   r   r   rZ  rH   
evaluatorsr   r   r   r   r   )r;   rW  r   r   r  rY  training_log_metricsr^  s          @r=   set_evaluation_metricsz7SentenceTransformerModelCardData.set_evaluation_metrics  sO    	IHHHHH,0MMy) 9.// 		H`5`_ 	)%899 4"j"jU^Ui"j"j"jOS11 4#2"3#k#k#k#k#k#k#k ! 	d&8&<V&D&L&L"2&--.BCCCCC"))!& $  /    	 	 	 	r>   r)   c                    d}t          t                    }t                      }|D ]}|d         }|d         }||vrZ||                             dt	          |           d           t          ||                   |k    r|                    |           t          |           j        k    r n fd|                                D              _	        d S )N   r;  labelz<li>z</li>c                    g | ]V\  }}j         j        r't          |t                    rj         j        |         n|d d                    |          z   dz   dWS )z<ul> z</ul>)LabelExamples)rB   labelsrH   r   r   )rI   re  example_setr;   s      r=   rK   zGSentenceTransformerModelCardData.set_label_examples.<locals>.<listcomp>@  s}     #
 #
 #

 #{ 6:Z5Fl:V[]`KaKal*511gl"RWW[%9%99GC #
 #
 #
r>   )
r   r   r   r   r   r   addnum_classesr   r   )r;   r_   num_examples_per_labelexamplesfinished_labelsrB  r;  re  s   `       r=   set_label_examplesz3SentenceTransformerModelCardData.set_label_examples3  s    !"t$$%% 	 	F&>D7OEO++&&'?d4jj'?'?'?@@@x''+AAA#''...?##t'777 8#
 #
 #
 #

 '/nn&6&6#
 #
 #
r>   r+  c                    t          |t                    r  fd|                                D             S |rt          j        d|          rd }|p|j        j        t          |j                  d}|j        j	        r2|j        |j        j	        v r|j        j	        |j                 j
        |d<   |j        x}rt          |                                          d         }|                    d          rtd|v rp|t          d          d                              d          }|d         |d<   |d	                             d
          d         x}rt          |          dk    r||d<   |gS )Nc                N    g | ]!\  }}                     ||           D ]}|"S )r3  )infer_datasets)rI   r+  sub_datasetr_   r;   s       r=   rK   zCSentenceTransformerModelCardData.infer_datasets.<locals>.<listcomp>J  sW       -L+#22;\2ZZ      r>   z_dataset_\d+)r  r   sizer   zhf://datasets/@r  r   r   (   revision)rH   r*   r   rematchr  r+  r   r   splitsnum_examplesdownload_checksumsr   r   r   r   )r;   r_   r+  dataset_output	checksumssourcesource_partsry  s   `       r=   rt  z/SentenceTransformerModelCardData.infer_datasetsH  s   g{++ 	   18     	 BH_lCC 	 L !=GL$=''
 
 < 	U7=GL4G#G#G%,\%8%G%TN6"  229 	:)..**++A.F  !122 :sf}}%c*:&;&;&=&=>DDSII'3At$ ,Q 5 5c : :1 ==H :3x==TVCVCV19N:.r>   r;  str | list[str]c                ~    	  | j         j        |fi |S # t          $ r | j                             |          cY S w xY wr8   )rB   tokenize	TypeError)r;   r;  r]   s      r=   r  z)SentenceTransformerModelCardData.tokenizeg  s[    	-&4:&t66v666 	- 	- 	-:&&t,,,,,	-s    $<< Dataset | IterableDataset | Noner   rV   'dict[str, nn.Module] | nn.Module | Nonec           	        |si S t          |t                    rt          |          |d<   d |j        D             |d<   i |d<   t          |t                    r|j        D ]}|dd         |         d         }t          |t                    r|                     d	          }t          |t                    r5d
|v r1|d
                             d                                          }d}nd D             }d}dt          t          |          d           d| t          t          |          t          |          z  d           d| t          t          |          d           d| dd|d         |<   #t          |t          t          f          r9t                    dfdt                    D             d|d         |<   xt          |t                     rudt          t                    d          t          t                    t                    z  d          t          t                    d          dd|d         |<   t          |t"                    rt          d D                       t                    dk    r"ddt          |           did|d         |<   edt                     dt                    t                    z  ddt                     ddd|d         |<   t%          |          i d|d         |<   dCddd id! |d                                         D             dd"ifd#|d                                         D             g}	t)          t+          |	                              d$d%          d&          |d'<   |dd(         |d)<   t          |d)         t#          |d)                   d                            }
g }t/          |
          D ]}i }|j        D ]}|d)         |         |         }t          |t"                    r5t          |          d*k    r"t	          |dd*                   dd+         d,z   }t          |t                    r t          |          dk    r|dd         d-z   }t	          |                              d.d/                              d0d1          }d2| d3||<   |                    |           t)          t+          |                              d$d%          d&          |d4<   d5t%          |          i|d6<   t3          |d7          r|                                }dDd<fd=|                                D             }	 t7          j        |d>?          }n!# t:          $ r t=          |d>?          }Y nw xY wt)          d@| dAd&          |d6         dB<   |S )Ea  
        Given a dataset, compute the following:
        * Dataset Size
        * Dataset Columns
        * Dataset Stats
            - Strings: min, mean, max word count/token length
            - Integers: Counter() instance
            - Floats: min, mean, max range
            - List: number of elements or min, mean, max number of elements
        * 3 Example samples
        * Loss function name
            - Loss function config
        rv  c                    g | ]}d | d	S )<code></code>rG   )rI   r(  s     r=   rK   zLSentenceTransformerModelCardData.compute_dataset_metrics.<locals>.<listcomp>  s$    "_"_"_#;F#;#;#;"_"_"_r>   rM  statsNr  r   document)taskattention_maskr   )dimtokensc                ,    g | ]}t          |          S rG   r,  r8  s     r=   rK   zLSentenceTransformerModelCardData.compute_dataset_metrics.<locals>.<listcomp>  s    "L"L"LX3x=="L"L"Lr>   
charactersr%     r   )rD  meanmax)r'  datar   c                v    i | ]5}|t                    d k    rdnd |         t                    z  d6S )r   ~rg  z.2%r,  )rI   ry   counter
subsections     r=   r|   zLSentenceTransformerModelCardData.compute_dataset_metrics.<locals>.<dictcomp>  s`     ! ! ! #  3w<<!+;+;CC!iWS\TWXbTcTcEc!i!i!i! ! !r>   r   c                ,    g | ]}t          |          S rG   r,  )rI   lsts     r=   rK   zLSentenceTransformerModelCardData.compute_dataset_metrics.<locals>.<listcomp>  s    &F&F&FCs3xx&F&F&Fr>   r   z	 elementsz.2fr  r   c                p    dd                     d |                                 D                       z   dz   S )Nz<ul><li>z	</li><li>c              3  *   K   | ]\  }}| d | V  dS )z: NrG   r-  s      r=   r.  zaSentenceTransformerModelCardData.compute_dataset_metrics.<locals>.to_html_list.<locals>.<genexpr>  s5      4f4f:3PU5F5Fu5F5F4f4f4f4f4f4fr>   z
</li></ul>)r   r   r  s    r=   to_html_listzNSentenceTransformerModelCardData.compute_dataset_metrics.<locals>.to_html_list  s=    !K$4$44f4fY]YcYcYeYe4f4f4f$f$ffiuuur>   rg  typec                &    i | ]\  }}||d          S )r'  rG   r-  s      r=   r|   zLSentenceTransformerModelCardData.compute_dataset_metrics.<locals>.<dictcomp>  s"    ddd
UU7^dddr>   detailsc                :    i | ]\  }}| |d                    S r  rG   )rI   ry   rz   r  s      r=   r|   zLSentenceTransformerModelCardData.compute_dataset_metrics.<locals>.<dictcomp>  s-    "t"t"t
U3U6](C(C"t"t"tr>   -:|--|  stats_tablerd  ro  r  r   z, ...]z...
z<br>|z\|r  r  examples_tabler&   rV   get_config_dictrz   r   r5   r   c                T   t          | t          j                  s| S | j        j        }g }t          | d          rAt          | j        d          r,|                    t          | j        j	                             t          | d          r| j
        r|                    d           t          | d          rS|                                                                 D ],\  }}|                    | dt          |                      -|r| dd                    |           d	S |S )
NrN   r   trust_remote_codeztrust_remote_code=Truer  =(r  ))rH   r   Moduler<   r   r   rN   r   r   r   r  r  r   r   )rz   module_namemodule_args_strry   vals        r=   format_config_valuezUSentenceTransformerModelCardData.compute_dataset_metrics.<locals>.format_config_value  sJ   !%33 ! L#o6"$ 5"344 SAVXd9e9e S#**40E0P+Q+QRRR5"566 E5;R E#**+CDDD5"344 E$)$9$9$;$;$A$A$C$C E ES'..#/C/CS		/C/CDDDD" J)IIDIIo,F,FIIII""r>   c                .    i | ]\  }}| |          S rG   rG   )rI   ry   rz   r  s      r=   r|   zLSentenceTransformerModelCardData.compute_dataset_metrics.<locals>.<dictcomp>  s+    WWW*#uc..u55WWWr>   r   r   ```json

```config_code)r  r   )rz   r   r5   r   )rH   r)   r   column_namesr   r  r   rE  tolistr   rD  r  r   r   r   rG  r   r   r&   r   r   r   replacerC  r   r   r  jsondumpsr  r   )r;   r_   r   rV   r(  first	tokenizedrP  suffixstats_linesrL  examples_lines
sample_idxrM  rz   config
str_configr  r  r  r  s                    @@@@r=   compute_dataset_metricsz8SentenceTransformerModelCardData.compute_dataset_metricsn  s   &  	Igw'' 	0#&w<<L "_"_'J^"_"_"_Y "Wgw'' U	u!. 8[ 8[$UdU^F3
"1eS)) 5[ $jz J JI!)T22 .7G97T7T"+,<"="A"Aa"A"H"H"O"O"Q"Q!)"L"L"L"L"L!-!)&+CLL!&<&<#G#Gv#G#G',S\\CLL-H!'L'L$W$Wv$W$W&+CLL!&<&<#G#Gv#G#G! !5 5L)&11  T{33 %[%j11G!&! ! ! ! !'-g! ! !5 5L)&11  u-- [!(#(Z!#<#<$)#j//C
OO*KQ$O$O#(Z!#<#<! !5 5L)&11  t,, [%&F&F:&F&F&FGGG7||q((%+ &3u::(@(@(@%9 9W-f55 &,*-g,,'A'A'A+.w<<#g,,+F(U(U(U(U*-g,,'A'A'A% %9 9W-f55 ?GuooWY4Z4ZL)&11v v v v VeddlSZF[FaFaFcFcdddeYu"t"t"t"tVbcjVkVqVqVsVs"t"t"tuK +11D[1Q1Q1Y1YZ_af1g1gim*n*nL''.rr{L$l:6tL<T7U7UVW7XYZZKN#K00 / /
%2 
> 
>F(4V<ZHE!%.. ?3u::>> #E"1"Iss 3h >!%-- 5#e**t2C2C %eteu 4JJ..tV<<DDS%PPE&=u&=&=&=GFOO%%g....-34G4W4W4_4_`egl4m4mos-t-tL)*  
V 4*++ 	^))++F# # # #( XWWWWWWF7!Zq999

 7 7 7$VA666


7289VZ9V9V9VX\2]2]L /s   !V8 8WWdataset_metadata nn.Module | dict[str, nn.Module]dataset_typeLiteral['train', 'eval']c           	         |r|rt          |t                    r t          |          t          |          k    s(t          |t                    r9t          |          dk    r&t                              d| d| d| d           g }|s                     |          }t          |t                    rC fdt          |                                |	                                |          D             }n 
                    ||d                   g}|dk    rt          d	 |D                       }|r                     d
|             j        ct          |t                    r,t          d |	                                D                       }nt          |j                  }ddh|z  rd _                             |          S )Nr   zThe number of `z?_datasets` in the model card data does not match the number of z1 datasets in the Trainer. Removing the provided `z$_datasets` from the model card data.c           
         g | ]:\  }}}                     ||t          t                    r|         n          ;S rG   )r  rH   r   )rI   r+  dataset_valuer   rV   r;   s       r=   rK   zMSentenceTransformerModelCardData.extract_dataset_metadata.<locals>.<listcomp>  sd     	$ 	$ 	$ Bm\ 00%$.8t.D.DN\**$ 	$ 	$ 	$r>   r   rL   c                :    g | ]}|                     d d          S )rv  r   )r	  )rI   metadatas     r=   rK   zMSentenceTransformerModelCardData.extract_dataset_metadata.<locals>.<listcomp>(  s&    'a'a'aHVQ(?(?'a'a'ar>   zdataset_size:c              3  .   K   | ]}|j         D ]}|V  d S r8   )r  )rI   ru  r(  s      r=   r.  zLSentenceTransformerModelCardData.extract_dataset_metadata.<locals>.<genexpr>.  sK       ' '#.Q\Qi' 'GM' ' ' ' ' ' 'r>   queryquestionT)rH   r*   r   r)   r   r   rt  rF  r   r   r  rE  rO   r   r   r   r  r   )r;   r_   r  rV   r  num_training_samplesr  s   `  `   r=   rT   z9SentenceTransformerModelCardData.extract_dataset_metadata  sR     	f &G[11&69:J6K6KsSZ||6[6[w00 7\589I5J5Ja5O5Oal a as a a.:a a a   $& # @#'#6#6w#?#? ';// f	$ 	$ 	$ 	$ 	$ FI(8(8:JF F	$ 	$ 	$   %)$@$@JZ[\J]_c$d$d#e  7""#&'a'aP`'a'a'a#b#b # FD.BDDEEE}$gt,, =#& ' '29..2B2B' ' ' $ $LL $'w';#<#<LZ(<7 )$(DM%%&6777r>   r/   c                    || _         | j        d S t          d |                                D             v r	d| _        d S dD ]3}||j        v r(t          |j        |                   dk    r
d| _         d S 4d S )Nc                    g | ]	}|j         
S rG   r   )rI   modules     r=   rK   zCSentenceTransformerModelCardData.register_model.<locals>.<listcomp>>  s    FFF6f&FFFr>   T)r  r  passagecorpusr   )rB   r   r#   childrenpromptsr   )r;   rB   ir_prompt_names      r=   register_modelz/SentenceTransformerModelCardData.register_model8  s    
=$FFFU^^5E5EFFFFF DMFH 	 	N..3u}^7T3U3UXY3Y3Y $	 	r>   c                    || _         d S r8   )r   )r;   r   s     r=   set_model_idz-SentenceTransformerModelCardData.set_model_idG  s     r>   ry  c                    | j         rdS 	 t          |          }n# t          $ r Y dS w xY w|j        | _        ||dk    r|j        }|| _        dS )NFmainT)r   get_model_infor
  r  r   shar   )r;   r   ry  r   s       r=   set_base_modelz/SentenceTransformerModelCardData.set_base_modelJ  sw      	5	'11JJ 	 	 	55	 %-x611!~H#+ ts    
))c                D    t          |t                    r|g}|| _        d S r8   )rH   r   r   )r;   r   s     r=   set_languagez-SentenceTransformerModelCardData.set_languageZ  s&    h$$ 	" zH r>   c                    || _         d S r8   )r   )r;   r   s     r=   set_licensez,SentenceTransformerModelCardData.set_license_  s    r>   c                    t          |t                    r|g}|D ]%}|| j        vr| j                            |           &d S r8   )rH   r   r   r   )r;   r   tags      r=   rO   z)SentenceTransformerModelCardData.add_tagsb  sX    dC   	6D 	& 	&C$)##	  %%%	& 	&r>   c                    | j         j        x}|j        j        }t	          |          }d                    |j        dd                    g}|j                            d          |fdt          dt                              D             z  }|D ]}|                     |          r d S d S t          | j         d         t                    r9| j         d         j        r)|                     | j         d         j                   d S d S d S )Nr   r   c                    g | ]@}d                      d|                   dz   d                      |d                   z   AS )r   Nr   )r   )rI   rQ  r|  s     r=   rK   zJSentenceTransformerModelCardData.try_to_set_base_model.<locals>.<listcomp>u  sX     $ $ $JM&&,sxxstt/E/EE$ $ $r>   r   r   )rB   transformers_modelr  _name_or_pathr
   r   partsr  r   rC  r   r  rH   r$   r   )r;   r  r   base_model_pathcandidate_model_idsr   r|  s         @r=   try_to_set_base_modelz6SentenceTransformerModelCardData.try_to_set_base_modeli  sL   "&*"??L+2@J":..O $'88O,A"##,F#G#G"H
 %)//44F $ $ $ $QVWXZ]^dZeZeQfQf$ $ $  0  &&x00 EE  
177 	>z!}' >##DJqM$<=====	> 	>> >r>   c           	     |   g }i }g }| j                                         D ]\  }}t          |dd          t          |dd          rt          fd|                                D                       rTfd|                                D             }r2                    dz             rt                    dz   d         d$dfd|                                D             }fd|                                D             }|j        t          |dd          d}t          |d          ra|	                                x}rK	 t          j        |d          }	n# t          $ r t          |          }	Y nw xY wt          d|	 dd          }|                    t!          |          ||d           fd|                    fd|                                D                        |                    |           g }
|D ]'}d |d         D             }t'          |          }|
D ]}t'          d |d         D                       }|d         |d         k    r||k    r|d         |d         k    r|d         |d         k    r|d         D ];}d|v r|                    d          ||d         <   ||d                  ||d         <   <t+          |d         t,                    s|d         g|d<   |d                             |d                     n|
                    |           )|
D ];}t/          |                    d                                        d d!          |d"<   <|
t-          |                                          t3          | j        |          d#S )%au  Format the evaluation metrics for the model card.

        The following keys will be returned:
        - eval_metrics: A list of dictionaries containing the class name, description, dataset name, and a markdown table
          This is used to display the evaluation metrics in the model card.
        - metrics: A list of all metric keys. This is used in the model card metadata.
        - model-index: A list of dictionaries containing the task name, task type, dataset type, dataset name, metric name,
          metric type, and metric value. This is used to display the evaluation metrics in the model card metadata.
        r  NrZ  c              3  H   K   | ]}|                     d z             V  dS )r   N)r   )rI   ry   r  s     r=   r.  zGSentenceTransformerModelCardData.format_eval_metrics.<locals>.<genexpr>  s3      QQ3CNN4#:66QQQQQQr>   c                L    i | ] \  }}|t                    d z   d         |!S )r   Nr,  )rI   ry   rz   r  s      r=   r|   zHSentenceTransformerModelCardData.format_eval_metrics.<locals>.<dictcomp>  s2    YYY:33s4yy1}/YYYr>   r   r   rz   r   r5   c                r    	 t          | d          r|                                 S n# t          $ r Y nw xY w| S )z^Try to convert a value from a Numpy or Torch scalar to pure Python, if not already pure Pythonr'  )r   itemr
  r   s    r=   try_to_pure_pythonzPSentenceTransformerModelCardData.format_eval_metrics.<locals>.try_to_pure_python  sQ    ug.. ,$zz||+,    Ds   #' 
44c                .    i | ]\  }}| |          S rG   rG   )rI   ry   rz   r  s      r=   r|   zHSentenceTransformerModelCardData.format_eval_metrics.<locals>.<dictcomp>  s+    XXX*#us..u55XXXr>   c                    g | ]<\  }}|k    rd | d n||k    rd t          |           d nt          |          d=S )**Metricr-   )r   )rI   
metric_keymetric_valuerZ  s      r=   rK   zHSentenceTransformerModelCardData.format_eval_metrics.<locals>.<listcomp>  s        -J 6@>5Q5Q1:1111Wa!^33 ?*\":":>>>>#L11	   r>   rg  r  r   r   r  r  r  )
class_namedescriptionr+  table_linesr  c                    	 t          |           S # t          $ r Y nw xY wt          | t                    r'd| v r# |                                 d                   S d S )Nr   r   )r   r
  rH   r   r   )r  try_to_floats    r=   r  zJSentenceTransformerModelCardData.format_eval_metrics.<locals>.try_to_float  s{     ...    D lC00 ASL5H5H'<(:(:(<(<Q(?@@@ts    
c                P   g | ]\  }} |          xt                                                              dd          pdr*                    dd                              dd          nd|                    dd                                          |          S )Nr   -unknownr   Unknown)r   	task_typer  r+  metric_namemetric_typer  )r   lowerr  title)rI   r  r  r+  r  metric_value_floatr  s      r=   rK   zHSentenceTransformerModelCardData.format_eval_metrics.<locals>.<listcomp>  s        1
L.:l<.H.HH*U "-"-"3"3"5"5"="=c3"G"G%1%>YYe%t\%9%9#s%C%C%K%KCQT%U%U%Ukt$.$6$6sC$@$@$F$F$H$H$.%7   VUUr>   c                ,    i | ]}|d          |d         S r   rG   rI   lines     r=   r|   zHSentenceTransformerModelCardData.format_eval_metrics.<locals>.<dictcomp>  s"    "h"h"hT4>4="h"h"hr>   r  c              3  &   K   | ]}|d          V  dS )r  NrG   r  s     r=   r.  zGSentenceTransformerModelCardData.format_eval_metrics.<locals>.<genexpr>  s&      1p1pT$x.1p1p1p1p1p1pr>   r  r+  r  r-   r  r  r  table)eval_metricsr   r   )rz   r   r5   r   )r   r   getattrallr   r   r   r  r   r  r  r  r  r   r   r   r&   rH  r   r   r   rH   r   r   r  r   r   )r;   r  all_metricseval_resultsrW  r   r  r  r  r  grouped_eval_metricseval_metriceval_metric_mappingeval_metric_metricsgrouped_eval_metricgrouped_eval_metric_metricsr  r+  r  r  r  rZ  r  r  s                    @@@@@@@r=   format_eval_metricsz4SentenceTransformerModelCardData.format_eval_metrics  s    "&"8">">"@"@ L	( L	(Iw9fd33D$Y0@$GGN EQQQQ',,..QQQQQ EYYYYYYY! En&?&?s
&K&K E%3CIIMOO%DN    YXXXXXXG    18  K $/K"9fd;;LKy"344 JID]D]D_D_:_& J-!%F1!=!=!=JJ  - - -!$VJJJ-$%B%B%B%BDII"*9"5"5#.$0#.#.   	 	 	 	 	        5<MMOO     w''''  "' 	9 	9K"h"h[YfMg"h"h"h"%&9":":'; 9 9#.11p1pM`anMo1p1p1p.p.p+-1D\1RRR+/JJJ#N37J>7ZZZ#M26I-6XXX !4M B ` `"d??HLQXHYHYD!4^!DE<OPTU]P^<_[899%&9.&I4PP d?RSa?b>c+N;'7>>{>?Z[[[E$++K888#7 	 	+>?R?V?VWd?e?e+f+f+n+nu, ,((
 1K,,..//6tUU
 
 	
s   E..F
	F
c                    g  j         D ]2}|                                D ]}|vr                    |           3d	fd}t          |           fd j         D             }t	          |          }|d|v dS )
Nry   r   r5   c                    | dk    rdS | dk    rdS | dk    rdS | dk    rdS |                      d	          rd
S                     |           dz   S )Nr   r   r   r   r   r  r   rd  rV   r   r  )r   index)ry   eval_lines_keyss    r=   sort_metricszKSentenceTransformerModelCardData.format_training_logs.<locals>.sort_metrics	  sr    g~~qf}}qo%%q'''q||F## q"((--11r>   r1  c                0    g | ]fd D             S )c                    i | ]I}|d          j         k    rd|v rt          |                   nd dn                    |d          JS )r   r  r
  )r   r   r	  )rI   ry   r  r;   s     r=   r|   zTSentenceTransformerModelCardData.format_training_logs.<locals>.<listcomp>.<dictcomp>  s{         <4#777 L3$;;*T#Y///CKKKKXXc3''  r>   rG   )rI   r  r;   sorted_eval_lines_keyss    @r=   rK   zISentenceTransformerModelCardData.format_training_logs.<locals>.<listcomp>  sT     
 
 
      2	  
 
 
r>   r  )
eval_linesexplain_bold_in_eval)ry   r   r5   r   )r   r   r   rG  r   )r;   linesry   r(  r   r,  r'  r+  s   `     @@r=   format_training_logsz5SentenceTransformerModelCardData.format_training_logs   s    ' 	0 	0Ezz|| 0 0o--#**3///0
	2 	2 	2 	2 	2 	2 "(\!J!J!J
 
 
 
 
 *
 
 
 )77
$$(J$6
 
 	
r>   c                   | j         | j        r
g d| _         n	g d| _         | j        sd S | j        rn| j                            | j         d         dd          }| j                            | j         dd          dd          }| j                            ||          }nQ| j         d d         | _         | j                            | j         dd          }| j                            ||          }t          j	        
                    d	d
          5  d                    d t          |                                                                          D                       | _        d d d            d S # 1 swxY w Y   d S )N)z(Which planet is known as the Red Planet?zMVenus is often called Earth's twin because of its similar size and proximity.zOMars, known for its reddish appearance, is often referred to as the Red Planet.zGSaturn, famous for its rings, is sometimes mistaken for the Red Planet.)zThe weather is lovely today.zIt's so sunny outside!zHe drove to the stadium.r   TF)convert_to_tensorrt   r   rd  r   )	precisionsci_moder  c              3      K   | ]	}d | V  
dS )z# NrG   r  s     r=   r.  zESentenceTransformerModelCardData.run_usage_snippet.<locals>.<genexpr>G  s(      )e)e$+t++)e)e)e)e)e)er>   )r   r   r   rB   encode_queryencode_document
similarityencoder   _tensor_strprintoptionsr   r   cpu
splitlinesr   )r;   query_embeddingsdocument_embeddingsr7  
embeddingss        r=   run_usage_snippetz2SentenceTransformerModelCardData.run_usage_snippet&  s   '} ( ( ($$( ( ($ , 	F= 	G#z66$Q'4SX  7     #'*"<"<$QRR(DTY #= # # ../?ATUUJJ#'#7#;D **4+?SWkp*qqJ..z:FFJ++a%+HH 	f 	f $		)e)e#jnnFVFVBWBWBbBbBdBd)e)e)e e eD	f 	f 	f 	f 	f 	f 	f 	f 	f 	f 	f 	f 	f 	f 	f 	f 	f 	fs   AE55E9<E91dict[Literal['co2_eq_emissions'], dict[str, Any]]c                *   | j         j                                        }dt          |j                  dz  t          |j                  dd|j        dk    |j        |j        t          |j
        dz  d          di}|j        r|j        |d         d	<   |S )
Nr   r  
codecarbonzfine-tuningYi  rd  )	emissionsenergy_consumedr  training_typeon_cloud	cpu_modelram_total_size
hours_usedhardware_used)rR   tracker_prepare_emissions_datar   rE  rF  rH  rI  rJ  r   duration	gpu_model)r;   emissions_dataresultss      r=   get_codecarbon_dataz4SentenceTransformerModelCardData.get_codecarbon_dataI  s    2:RRTT">#;<<tC#()G#H#H&!.*3s:+5"0"?#N$;d$BAFF
! 
!
 # 	T;I;SG&'8r>   c                V   d}| j         j        rUddddd                    | j         j        | j         j                            dd                                                    }| j                                         | j                                         t          | j                   |dS )	NzCosine SimilarityzDot ProductzEuclidean DistancezManhattan Distance)cosinedot	euclidean	manhattanr   r   )model_max_lengthoutput_dimensionalitymodel_stringsimilarity_fn_name)rB   r\  r	  r  r  get_max_seq_length get_sentence_embedding_dimensionr   )r;   r\  s     r=   get_model_specific_metadataz<SentenceTransformerModelCardData.get_model_specific_metadata\  s    0:( 	j-$11	" "
 c$*/1N1V1VWZ\_1`1`1f1f1h1hii  !%
 = = ? ?%)Z%P%P%R%R
OO"4	
 
 	
r>   c                h    | j         r| j        j        j         d| j          S | j        j        j        S )Nz
 based on )r   rB   r<   r   r   s    r=   get_default_model_namez7SentenceTransformerModelCardData.get_default_model_namel  s8    ? 	1j*3PPtPPP:'00r>   c                l     j         r- j        s&	                                   n# t          $ r Y nw xY w j        s                                  _        	                                   n4# t          $ r'}t                              d|            Y d }~nd }~ww xY w fdt                     D             } j
        rZ	 |                                                                n1# t          $ r$}t                              d|            |d }~ww xY w j        r]	 |                                                                n4# t          $ r'}t                              d|            Y d }~nd }~ww xY wt           j                  dk    |d<    j        rD j        j        r8 j        j        j        '|                                                                |                                                                d _         t*          D ]}|                    |d            |S )Nz,Error while computing usage snippet output: c                F    i | ]}|j         t          |j                   S rG   )r  r  )rI   r   r;   s     r=   r|   z<SentenceTransformerModelCardData.to_dict.<locals>.<dictcomp>  s)    VVVej'$
";";VVVr>   z+Error while formatting evaluation metrics: z&Error while formatting training logs: d   hide_eval_linesF)r   r   r  r
  r   ra  r@  r   r   r	   r   r   r#  r   r/  r   rR   rM  _start_timerS  r_  IGNORED_FIELDSr   )r;   exc
super_dictry   s   `   r=   r~   z(SentenceTransformerModelCardData.to_dictr  s   ? 	4? 	**,,,,     	<"99;;DO	Q""$$$$ 	Q 	Q 	QNNO#OOPPPPPPPP	Q WVVVPTVVV
 ! 	!!$":":"<"<====   RSRRSSS	
  	OO!!$";";"="=>>>> O O OMMMNNNNNNNNO ),D,>(?(?#(E
$% %	:)1	: )1=Id6688999 	$::<<===! 	& 	&CNN3%%%%sT   & 
33A, ,
B6BB'C+ +
D5DD$'E 
E=E88E=c                    t          d |                                                                 D             d|                                          S )Nc                8    i | ]\  }}|t           v |d g fv||S r8   )YAML_FIELDSr-  s      r=   r|   z<SentenceTransformerModelCardData.to_yaml.<locals>.<dictcomp>  s:    sssJCC;DVDV[`imoqhr[r[rS%[r[r[rr>   F)	sort_keys
line_break)r   r~   r   strip)r;   rn  s     r=   to_yamlz(SentenceTransformerModelCardData.to_yaml  sO    ss$,,..*>*>*@*@sss!
 
 
 %''		r>   )r5   r6   r8   )r  r  r  r   r5   r  )r^   r   r5   r6   )r  r   r5   r6   )r_   r  r5   r6   )r   r   )
rW  r.   r   r4   r   r   r  r   r5   r6   )r_   r)   r5   r6   )r_   r  r+  r   r5   r   )r;  r  r5   r4   )r_   r  r   r4   rV   r  r5   r   )
r_   r  r  r  rV   r  r  r  r5   r  )rB   r/   r5   r6   )r   r   r5   r6   )r   r   ry  r   r5   r6   )r   r  r5   r6   )r   r   r5   r6   )r   r  r5   r6   r5   r4   )r5   rA  )r5   r   )Fr   r   r   __doc__r   r   r   __annotations__r   r   r   rU   rX   r   r   r   r   r   r   r   r   r   r   r   r[   r   r   rR   r   r   r   r   r   r   r   r   r   r   r   r
   __file__parentr   rB   r   r   rZ   r  r\   rb  rq  rt  r  r  rT   r  r  r  r  r  rO   r  r#  r/  r@  rS  r_  ra  r~   rp  rG   r>   r=   r   r   	  sl        ( (V (-uT'B'B'BHBBBBG!J!!!!H+05+F+F+FNFFFF*/%*E*E*EMEEEEt      #U
 
  D     #""""%*U4%8%8%88888 #U4e<<<J<<<<&+eDu&E&E&EEEEE27%SX2Y2Y2YYYYY*/%5*Q*Q*QQQQQHM^binHoHoHooooo,1E$U,S,S,SMSSSS#(5E#J#J#JFJJJJ(-d(G(G(GOGGGG/4uTPU/V/V/VVVVV6;eDu6U6U6UUUUU %d G G GIGGGG"'%5"A"A"AOAAAA%5uMMMHMMMM!E$UGGGHGGGG$uTEJJJLJJJJ uT666J6666uRe444K4444 &;%HHHLHHHH&=EJJJLJJJJ#eLuMMMGMMMM%X(=@X(X_dkpqqqMqqqq ).dU(S(S(SESSSS! ! ! !  RV/# /# /# /# /#bg g g g>$ $ $ $L5 L5 L5 L5^ bc    6
 
 
 
*         >- - - -S S S Sj38 38 38 38j   ! ! ! !     ! ! ! !
   & & & &> > > >,
 
 
 
B$
 $
 $
L!f !f !f !fF   &
 
 
 
 1 1 1 13 3 3 3j      r>   r   rB   r/   r   c                \    t          j        | j        | j        j        d          }|j        S )Nu   🤗)	card_datar   hf_emoji)r   from_templaterN   r   content)rB   
model_cards     r=   generate_model_cardr|    s5    ('u7L7Zek  J r>   rq  )rz   r   r5   r   )rV   r   r5   r   )rB   r/   r5   r   )Y
__future__r   r  loggingr<  rz  collectionsr   r   r   dataclassesr   r   r	   pathlibr
   platformr   pprintr   textwrapr   typingr   r   r   r   r   huggingface_hubr   r   r   r  r   r  huggingface_hub.repocard_datar   r   huggingface_hub.utilsr   r   tqdm.autonotebookr   r   transformers.integrationsr   transformers.modelcardr   transformers.trainer_callbackr   r   typing_extensionsr    r   r"   r   sentence_transformers.modelsr#   r$   #sentence_transformers.training_argsr%   sentence_transformers.utilr&   r'   r(   r   r)   r*   r+   r,   r-   	getLoggerr   r   2sentence_transformers.evaluation.SentenceEvaluatorr.   )sentence_transformers.SentenceTransformerr/   sentence_transformers.trainerr0   r2   r   rl  rg  r   r   rY   r   r|  rG   r>   r=   <module>r     s   " " " " " "    				 , , , , , , , ,       0 0 0 0 0 0 0 0 0 0       # # # # # #             . . . . . . . . . .      / / / / / / / / < < < < < < 8 8 8 8 8 8 Q Q Q Q Q Q Q Q + + + + + +       " " " " " " ( ( ( ( ( ( 8 8 8 8 8 8 6 6 6 6 6 6 F F F F F F F F ( ( ( ( ( ( N N N N N N @ @ @ @ @ @ @ @ T T T T T T _ _ _ _ _ _ _ _ _ _ [ZZZZZZZZZZZZZZ		8	$	$ ITTTTTTMMMMMMHHHHHHO: O: O: O: O:? O: O: O:d  v * * * * *< * * *
   ;::   ,      ( b b b b bx b b bJ     r>   