§
    ðóÉjçC ã                   óŽ  — d Z ddlZddlZddlZddlZddlZddlZddlZddl	Z	ddl
Z
ddlZddlZddlZddlZddlZddlZddlZddlmZmZ ddlmZ ddlmZ ddlmZmZmZmZmZ ddlmZ ddl m!Z" ddl#Z$ddl%Z&ddl'Z'ddl(m)Z* dd	l+m,Z,m-Z-m.Z. dd
l/m0Z0 ddl'm1Z1 ddl2m3Z3m4Z4m5Z5m6Z6m7Z7 ddl8m9Z9 ddl:m;Z; ddl<m=Z=m>Z>m?Z? ddl@mAZAmBZB ddlCmDZD ddlEmFZF ddlGmHZHmIZI ddlJmKZK ddlLmMZMmNZNmOZO ddlPmQZQ ddlRmSZS ddlTmUZUmVZVmWZW ddlXmYZYmZZZ ddl[m\Z\m]Z] ddl^m_Z_ ddl`maZa ddlbmcZc ddldmeZemfZfmgZgmhZhmiZimjZjmkZkmlZl ddlmmnZnmoZompZpmqZqmrZrmsZsmtZtmuZumvZvmwZwmxZxmyZymzZzm{Z{m|Z|m}Z}m~Z~mZm€Z€m�Z� dd l‚mƒZƒm„Z„m…Z…m†Z†m‡Z‡mˆZˆm‰Z‰mŠZŠm‹Z‹mŒZŒm�Z�mŽZŽm�Z�m�Z�m‘Z‘m’Z’m“Z“m”Z”m•Z•m–Z–m—Z—m˜Z˜m™Z™ dd!lšm›Z›mœZœm�Z� dd"l!mžZžmŸZŸm Z m¡Z¡m¢Z¢m£Z£m¤Z¤m¥Z¥m¦Z¦m§Z§m¨Z¨m©Z©mªZªm«Z«m¬Z¬m­Z­m®Z®m¯Z¯m°Z°m±Z±m²Z²m³Z³m´Z´mµZµm¶Z¶m·Z·m¸Z¸m¹Z¹mºZºm»Z»m¼Z¼m½Z½m¾Z¾m¿Z¿mÀZÀmÁZÁmÂZÂmÃZÃmÄZÄmÅZÅ dd#lÆmÇZÇ dd$lÈmÉZÉ dd%lÊmËZË efgZÌeiZÍ e³¦   «         rdd&lÎmÏZÏ eÏZÍ e°¦   «         rddlÐZÐ eÁ¦   «         rLddlÑmÒc mÓZÔ ddlÕmÖc m×ZØ ddlÙmÚZÛ ddlÜm9ZÝ  e0jÞ        eÝ¦  «         e0jÞ        e§¦  «        k    Zßeßr	ddlàm)c máZâ nd'Zß e¸¦   «         r>ddlãmäc m'Zå ddlæm9Zç  e0jÞ        eç¦  «         e0jÞ        d(¦  «        k    Zèdd)lmméZémêZêmëZëmìZì nd'Zè e¶¦   «         rdd*límîZî  e­¦   «         rrdd+lïmðZðmñZñ ddlïm9Zò dd,lómôZô dd-lõmöZöm÷Z÷møZømùZùmúZúmûZûmüZü e6gZý e0jÞ        eò¦  «         e0jÞ        d.¦  «        k    rdd/lõmþZþ dd0lÿ�m �Z  eý�e gz  Zý eO¦   «         rdd1lõ�m�Z  e­d2¦  «        rdd3lõ�m�Z d4„ �Zd5„ �Zd6„ �Zerdd�l�Z eÄ�j        �e¦  «        �Z	d7�Z
d8�Zd9�Zd:�Zd;�Zd<�Zd=�Z eÉd>¬?¦  «         G d@„ dA¦  «        ¦   «         �ZdS )Buc   
The Trainer class, to easily train a ðŸ¤— Transformers from scratch or finetune it on a new task.
é    N)ÚIteratorÚMapping©Úpartial)ÚPath)ÚTYPE_CHECKINGÚAnyÚCallableÚOptionalÚUnioné   )Ú#get_reporting_integration_callbacks)Ú	ModelCardÚcreate_repoÚupload_folder)Úversion)Únn)Ú
DataLoaderÚDatasetÚIterableDatasetÚRandomSamplerÚSequentialSampler)Ú__version__)ÚPretrainedConfig)ÚDataCollatorÚDataCollatorWithPaddingÚdefault_data_collator)ÚDebugOptionÚDebugUnderflowOverflow)ÚSequenceFeatureExtractor)ÚFeatureExtractionMixin)Ú"ALL_HYPERPARAMETER_SEARCH_BACKENDSÚdefault_hp_search_backend)ÚBaseImageProcessor)Údeepspeed_initÚdeepspeed_load_checkpointÚis_deepspeed_available)Útpu_spmd_dataloader)ÚTrainingSummary)ÚPreTrainedModelÚload_sharded_checkpointÚunwrap_model)Ú!MODEL_FOR_CAUSAL_LM_MAPPING_NAMESÚMODEL_MAPPING_NAMES)Ú	AdafactorÚget_scheduler)ÚProcessorMixin)Ú"is_torch_greater_or_equal_than_2_3)ÚPreTrainedTokenizerBase)ÚCallbackHandlerÚDefaultFlowCallbackÚExportableStateÚPrinterCallbackÚProgressCallbackÚTrainerCallbackÚTrainerControlÚTrainerState)ÚDistributedTensorGathererÚEvalLoopContainerÚIterableDatasetShardÚLabelSmootherÚLayerWiseDummyOptimizerÚLengthGroupedSamplerÚSequentialDistributedSamplerÚdistributed_broadcast_scalarsÚdistributed_concatÚfind_batch_sizeÚget_model_param_countÚget_module_class_from_nameÚget_parameter_namesÚnested_concatÚnested_detachÚnested_numpifyÚnested_xla_mesh_reduceÚreissue_pt_warningsÚremove_dummy_checkpointÚset_rng_state_for_device)ÚPREFIX_CHECKPOINT_DIRÚBestRunÚEvalLoopOutputÚEvalPredictionÚHPSearchBackendÚHubStrategyÚPredictionOutputÚRemoveColumnsCollatorÚSaveStrategyÚTrainerMemoryTrackerÚTrainOutputÚcheck_target_module_existsÚdefault_compute_objectiveÚdenumpify_detensorizeÚenable_full_determinismÚfind_executable_batch_sizeÚget_last_checkpointÚ
has_lengthÚneftune_post_forward_hookÚnumber_of_argumentsÚseed_workerÚset_seedÚspeed_metrics)ÚOptimizerNamesÚParallelModeÚTrainingArguments)(ÚADAPTER_CONFIG_NAMEÚADAPTER_SAFE_WEIGHTS_NAMEÚADAPTER_WEIGHTS_NAMEÚCONFIG_NAMEÚGENERATION_CONFIG_NAMEÚSAFE_WEIGHTS_INDEX_NAMEÚSAFE_WEIGHTS_NAMEÚWEIGHTS_INDEX_NAMEÚWEIGHTS_NAMEÚXLA_FSDPV2_MIN_VERSIONÚPushInProgressÚPushToHubMixinÚcan_return_lossÚcheck_torch_load_is_safeÚfind_labelsÚis_accelerate_availableÚis_apollo_torch_availableÚis_bitsandbytes_availableÚis_datasets_availableÚis_galore_torch_availableÚis_grokadamw_availableÚis_in_notebookÚis_liger_kernel_availableÚis_lomo_availableÚis_peft_availableÚis_sagemaker_dp_enabledÚis_sagemaker_mp_enabledÚis_schedulefree_availableÚis_torch_hpu_availableÚis_torch_mlu_availableÚis_torch_mps_availableÚis_torch_musa_availableÚis_torch_neuroncore_availableÚis_torch_npu_availableÚis_torch_optimi_availableÚis_torch_xla_availableÚis_torch_xpu_availableÚis_torchao_availableÚloggingÚ	strtobool)Údeprecate_kwarg)Úrequires)ÚQuantizationMethod)ÚNotebookProgressCallbackFz1.10)Úsmp_forward_backwardÚsmp_forward_onlyÚ
smp_gatherÚsmp_nested_concat)Ú	PeftModel)ÚAcceleratorÚskip_first_batches)ÚAcceleratorState)ÚAutocastKwargsÚDistributedDataParallelKwargsÚDistributedTypeÚload_fsdp_modelÚload_fsdp_optimizerÚsave_fsdp_modelÚsave_fsdp_optimizerú1.3.0)ÚTorchTensorParallelPlugin)ÚSeedableRandomSampler)ÚDeepSpeedSchedulerWrapperú0.28.0)ÚDataLoaderConfigurationc                 óú   — t          ¦   «         rlt          f}t          j        t          j                             d¦  «        ¦  «        t          j        d¦  «        k    rddlm} g |¢|‘R }t          | |¦  «        S dS )NÚpeftz0.7.0r   )ÚPeftMixedModelF)	r‚   rš   r   ÚparseÚ	importlibÚmetadatar¬   r­   Ú
isinstance)ÚmodelÚclasses_to_checkr­   s      úw/home/lighthouse/badge-speaker-benchmark/.benchmark-venv-community/lib/python3.11/site-packages/transformers/trainer.pyÚ_is_peft_modelrµ   ú   s„   € ÝÑÔð 3Ý%˜<ÐåŒ=�Ô+×3Ò3°FÑ;Ô;Ñ<Ô<ÅÄÈgÑ@VÔ@VÒVÐVØ+Ð+Ð+Ð+Ð+Ð+àBÐ!1ÐB°>ÐBÐBÐÝ˜%Ð!1Ñ2Ô2Ð2Øˆ5ó    c                  ó„   — t          ¦   «         r1dt          t          j        t          ¦  «        j        ¦  «        v rddiS i S )NÚadapter_onlyT)ry   ÚlistÚinspectÚ	signaturer£   Ú
parameters© r¶   r´   Ú_get_fsdp_ckpt_kwargsr¾     s@   € åÑ Ô ð  ^µt½GÔ<MÍoÑ<^Ô<^Ô<iÑ7jÔ7jÐ%jÐ%jØ Ð%Ð%àˆ	r¶   c                  ó  — t          j        t          j        ¦  «        j        t          j        d¦  «        j        k     rt          j        ¦   «         S t          j        t          j        ¦  «        t          j        d¦  «        k    rt          j        nt          j	        } | j
        j        t          j        t          j        g}|t          t          j        t          j        ¦  «        ¦  «        gz  }t          j                             |¦  «        S )Nz2.6z2.0.0)r   r®   Útorchr   ÚreleaseÚ
contextlibÚnullcontextÚnpÚ_coreÚcoreÚ
multiarrayÚ_reconstructÚndarrayÚdtypeÚtypeÚuint32ÚserializationÚsafe_globals)Únp_coreÚ	allowlists     r´   rÎ   rÎ     sº   € õ „}•UÔ&Ñ'Ô'Ô/µ'´-ÀÑ2FÔ2FÔ2NÒNÐNÝÔ%Ñ'Ô'Ð'å!œ-­¬Ñ7Ô7½7¼=ÈÑ;QÔ;QÒQÐQ�bŒhˆhÕWYÔW^€GØÔ#Ô0µ"´*½b¼hÐG€Ið •$•r”x¥¤	Ñ*Ô*Ñ+Ô+Ð,Ñ,€IåÔ×+Ò+¨IÑ6Ô6Ð6r¶   ztraining_args.binztrainer_state.jsonzoptimizer.ptz	scaler.ptzoptimizer.binzscheduler.ptÚpytorch_model_fsdp)rÀ   Ú
accelerate)Úbackendsc                   óz  — e Zd ZdZddlmZmZmZmZm	Z	  e
dddd¬¦  «        	 	 	 	 	 	 	 	 	 	 	 	 	 d¶deeej        d	f         dee         dee         deeeedf                  deeeeeef         df                  deeeeeef                  deedef                  dee         deeegef                  deee                  deeej         j!                 eej         j"        j#                 f         deee$ej         j!                 eee%f         f                  deeej&        ej&        gej&        f                  fd„¦   «         Z'e(dee         fd„¦   «         Z)e)j*        d·d„¦   «         Z)d„ Z+d„ Z,d„ Z-d „ Z.d!„ Z/d"„ Z0d#„ Z1d$„ Z2d¸d%dd&ee         fd'„Z3	 d¸ded&ee         defd(„Z4d¸dee         deej5        j6        j7                 fd)„Z8	 	 	 d¹d%ed&ed+e9d,eeegej5        j6        j7        f                  d-e:d.ee         de;fd/„Z<de;fd0„Z=dedeej5        j6        j7                 fd1„Z>d¸deeeef                  de;fd2„Z?d3ede;fd4„Z@d5e9fd6„ZAdee         fd7„ZBd8„ ZCd9„ ZDd:„ ZEd¸d;eeeej        jF        jG        f                  fd<„ZHeI	 d¸dedee         dee%e%f         fd=„¦   «         ZJd¸d5e9d>ej         j!        fd?„ZKd@e;de9fdA„ZLeId¸dBe;dCee9         de9fdD„¦   «         ZMdEedFeee%f         f         fdG„ZNdEedFeee%f         f         dHe9dIeeeOf         fdJ„ZPdKefdL„ZQd¸dM„ZRdºdN„ZSdO„ ZTd»dP„ZU	 	 	 d¼dQeeee:f                  dEedFeee%f         d	f         dReee                  dSe%fdT„ZVde9fdU„ZWde9fdV„ZX	 d½dW„ZYdX„ ZZd¸dY„Z[dZ„ Z\d[„ Z]dºd\„Z^	 d¸d]„Z_d^„ Z`d_„ Zad`„ Zbda„ Zcdb„ Zddc„ Zedd„ Zfde„ Zgdf„ Zh	 	 	 	 	 	 d¾dieedFgeeeOf         f                  djeeeeeOf         geOf                  dke9dleeee         f         dmeedneif                  doeedFgef                  deejeej         f         fdp„Zkd¸dqeeeOf         dreeO         dd	fds„Zldteej&        e%f         deej&        e%f         fdu„Zmdveeeej&        e%f         f         deeeej&        e%f         f         fdw„Zndx„ Zodveeeej&        e%f         f         fdy„Zpdz„ Zqd¿d{ee:         fd|„Zr	 d¸dej        dveeeej&        e%f         f         d}eej&                 dej&        fd~„Zs	 	 dÀdej        dveeeej&        e%f         f         de:d}eej&                 fd€„Ztde:fd�„Zude:fd‚„ZvdÁdƒee         d„e:fd…„Zwd¸dƒee         fd†„Zxd
dƒee         fd‡„Zydˆ„ Zzd	e{d*fdee         fd‰„Z|dÀd·dŠ„Z}	 	 	 dÂdeeeeeef         f                  dŒeee                  d�edeeeOf         fdŽ„Z~	 dÃd3edŒeee                  d�edefd�„Z€	 	 	 dÂd@e;d&ed‘ee:         dŒeee                  d�ede�fd’„Z‚d¸d“„Zƒ	 d¸dej        dveeeej&        e%f         f         d‘e:dŒeee                  deeej&                 eej&                 eej&                 f         f
d”„Z„dveeeej&        e%f         f         fd•„Z…d¸d–ee         fd—„Z†	 	 	 	 	 	 	 	 	 dÄd˜ee         d™ee         dšeeee         d	f         d›ee         dœee         d�eeee         d	f         džeeee         d	f         d%eeee         d	f         dŸeeee         d	f         fd „Z‡d¡„ Zˆd¢„ Z‰	 	 	 	 dÅd¤ee         d¥e:d–ee         d¦ee         def
d§„ZŠ	 	 	 dÂd@e;d&ed‘ee:         dŒeee                  d�ede�fd¨„Z‹d©„ ZŒd·dª„Z�d«„ ZŽdºd¬„Z�d­„ Z�d®ed¯ej‘        deeej&        e9f                  fd°„Z’d±e“d²e9d¯ej‘        deeeeej&        e9f                  f         fd³„Z”ded@e;d´e9fdµ„Z•d	S )ÆÚTraineruA  
    Trainer is a simple but feature-complete training and eval loop for PyTorch, optimized for ðŸ¤— Transformers.

    Args:
        model ([`PreTrainedModel`] or `torch.nn.Module`, *optional*):
            The model to train, evaluate or use for predictions. If not provided, a `model_init` must be passed.

            <Tip>

            [`Trainer`] is optimized to work with the [`PreTrainedModel`] provided by the library. You can still use
            your own models defined as `torch.nn.Module` as long as they work the same way as the ðŸ¤— Transformers
            models.

            </Tip>

        args ([`TrainingArguments`], *optional*):
            The arguments to tweak for training. Will default to a basic instance of [`TrainingArguments`] with the
            `output_dir` set to a directory named *tmp_trainer* in the current directory if not provided.
        data_collator (`DataCollator`, *optional*):
            The function to use to form a batch from a list of elements of `train_dataset` or `eval_dataset`. Will
            default to [`default_data_collator`] if no `processing_class` is provided, an instance of
            [`DataCollatorWithPadding`] otherwise if the processing_class is a feature extractor or tokenizer.
        train_dataset (Union[`torch.utils.data.Dataset`, `torch.utils.data.IterableDataset`, `datasets.Dataset`], *optional*):
            The dataset to use for training. If it is a [`~datasets.Dataset`], columns not accepted by the
            `model.forward()` method are automatically removed.

            Note that if it's a `torch.utils.data.IterableDataset` with some randomization and you are training in a
            distributed fashion, your iterable dataset should either use a internal attribute `generator` that is a
            `torch.Generator` for the randomization that must be identical on all processes (and the Trainer will
            manually set the seed of this `generator` at each epoch) or have a `set_epoch()` method that internally
            sets the seed of the RNGs used.
        eval_dataset (Union[`torch.utils.data.Dataset`, dict[str, `torch.utils.data.Dataset`, `datasets.Dataset`]), *optional*):
             The dataset to use for evaluation. If it is a [`~datasets.Dataset`], columns not accepted by the
             `model.forward()` method are automatically removed. If it is a dictionary, it will evaluate on each
             dataset prepending the dictionary key to the metric name.
        processing_class (`PreTrainedTokenizerBase` or `BaseImageProcessor` or `FeatureExtractionMixin` or `ProcessorMixin`, *optional*):
            Processing class used to process the data. If provided, will be used to automatically process the inputs
            for the model, and it will be saved along the model to make it easier to rerun an interrupted training or
            reuse the fine-tuned model.
            This supersedes the `tokenizer` argument, which is now deprecated.
        model_init (`Callable[[], PreTrainedModel]`, *optional*):
            A function that instantiates the model to be used. If provided, each call to [`~Trainer.train`] will start
            from a new instance of the model as given by this function.

            The function may have zero argument, or a single one containing the optuna/Ray Tune/SigOpt trial object, to
            be able to choose different architectures according to hyper parameters (such as layer count, sizes of
            inner layers, dropout probabilities etc).
        compute_loss_func (`Callable`, *optional*):
            A function that accepts the raw model outputs, labels, and the number of items in the entire accumulated
            batch (batch_size * gradient_accumulation_steps) and returns the loss. For example, see the default [loss function](https://github.com/huggingface/transformers/blob/052e652d6d53c2b26ffde87e039b723949a53493/src/transformers/trainer.py#L3618) used by [`Trainer`].
        compute_metrics (`Callable[[EvalPrediction], Dict]`, *optional*):
            The function that will be used to compute metrics at evaluation. Must take a [`EvalPrediction`] and return
            a dictionary string to metric values. *Note* When passing TrainingArgs with `batch_eval_metrics` set to
            `True`, your compute_metrics function must take a boolean `compute_result` argument. This will be triggered
            after the last eval batch to signal that the function needs to calculate and return the global summary
            statistics rather than accumulating the batch-level statistics
        callbacks (List of [`TrainerCallback`], *optional*):
            A list of callbacks to customize the training loop. Will add those to the list of default callbacks
            detailed in [here](callback).

            If you want to remove one of the default callbacks used, use the [`Trainer.remove_callback`] method.
        optimizers (`tuple[torch.optim.Optimizer, torch.optim.lr_scheduler.LambdaLR]`, *optional*, defaults to `(None, None)`):
            A tuple containing the optimizer and the scheduler to use. Will default to an instance of [`AdamW`] on your
            model and a scheduler given by [`get_linear_schedule_with_warmup`] controlled by `args`.
        optimizer_cls_and_kwargs (`tuple[Type[torch.optim.Optimizer], dict[str, Any]]`, *optional*):
            A tuple containing the optimizer class and keyword arguments to use.
            Overrides `optim` and `optim_args` in `args`. Incompatible with the `optimizers` argument.

            Unlike `optimizers`, this argument avoids the need to place model parameters on the correct devices before initializing the Trainer.
        preprocess_logits_for_metrics (`Callable[[torch.Tensor, torch.Tensor], torch.Tensor]`, *optional*):
            A function that preprocess the logits right before caching them at each evaluation step. Must take two
            tensors, the logits and the labels, and return the logits once processed as desired. The modifications made
            by this function will be reflected in the predictions received by `compute_metrics`.

            Note that the labels (second parameter) will be `None` if the dataset does not have them.

    Important attributes:

        - **model** -- Always points to the core model. If using a transformers model, it will be a [`PreTrainedModel`]
          subclass.
        - **model_wrapped** -- Always points to the most external model in case one or more other modules wrap the
          original model. This is the model that should be used for the forward pass. For example, under `DeepSpeed`,
          the inner model is wrapped in `DeepSpeed` and then again in `torch.nn.DistributedDataParallel`. If the inner
          model hasn't been wrapped, then `self.model_wrapped` is the same as `self.model`.
        - **is_model_parallel** -- Whether or not a model has been switched to a model parallel mode (different from
          data parallelism, this means some of the model layers are split on different GPUs).
        - **place_model_on_device** -- Whether or not to automatically place the model on the device - it will be set
          to `False` if model parallel or deepspeed is used, or if the default
          `TrainingArguments.place_model_on_device` is overridden to return `False` .
        - **is_in_train** -- Whether or not a model is currently running `train` (e.g. when `evaluate` is called while
          in `train`)

    r   )Ú_get_learning_rateÚlog_metricsÚmetrics_formatÚsave_metricsÚ
save_stateÚ	tokenizerÚprocessing_classz5.0.0T)Únew_namer   Úraise_if_both_namesN©NNr²   ÚargsÚdata_collatorÚtrain_datasetzdatasets.DatasetÚeval_datasetÚ
model_init.Úcompute_loss_funcÚcompute_metricsÚ	callbacksÚ
optimizersÚoptimizer_cls_and_kwargsÚpreprocess_logits_for_metricsc           	      ó²  — |€0d}t                                d|› d�¦  «         t          |¬¦  «        }|j        r,|	�*dt	          j        |	¦  «        j        vrt          d¦  «        ‚|j        �%|j        dk    r|€t          d|j        › d	�¦  «        ‚|j	        t          j        k    s|j        r|j        €t          d
¦  «        ‚|| _        || _        | j        j        rt#          | j        j        ¦  «        nt'          | j        j        ¦  «         d | _        d | _        d| _        || _        |                      ¦   «          t3          | j        j        ¦  «        | _        | j                             ¦   «          |                     ¦   «         }t=          j        |¦  «         |j          |€-|�|| _!        |  "                    ¦   «         }n2tG          d¦  «        ‚|�tI          j%        dtL          ¦  «         || _!        |j'        j(        tR          v rt          d|j'        j(        › d�¦  «        ‚tU          |dd¦  «        rtU          |dd¦  «        rd| _+        nd| _+        tU          |dd ¦  «        �´d„ tY          |j-         .                    ¦   «         ¦  «        D ¦   «         }t_          |¦  «        dk    rd| _+        nHt_          |¦  «        dk    r.| j        j0        tc          j0        |d         ¦  «        k    | _+        nd| _+        | j+        rt                                d¦  «         | j        j2        rÌtg          ¦   «         r¯ddl4m5} | j        j6        �| j        j6        ni }to          |tp          ¦  «        r |dPd|i|¤Ž n~ts          |d¦  «        rDto          | :                    ¦   «         tp          ¦  «        r |dPd| :                    ¦   «         i|¤Ž n*t            ;                    d¦  «         nty          d¦  «        ‚tU          |dd¦  «        otU          |dd¦  «         }tU          |dd ¦  «        d uo|j=        j>        }tU          |dd ¦  «        d uotU          |j=        d d¦  «        }|rts          |d!¦  «        rt          d"¦  «        ‚|r t          |¦  «        s|st          d#¦  «        ‚|r5|s3t          d$|j=        j@        jA        › d%|j=        j@        jA        › �¦  «        ‚|jB        d&         | _C        t_          |jD        ¦  «        dk    rG| jE        rt          d'¦  «        ‚|jB        d&         s$|jF        tŽ          jH        k    rt          d(¦  «        ‚|jI        | _I        | j+        s*| jE        s#|jJ        s|jK        r|jL        r| jC        s| jM        rd| _I        |�+to          |tœ          tž          f¦  «        rt¡          |¦  «        nt¢          }|�|n|| _R        || _S        || _T        || _U        | jI        r:tU          |d)d ¦  «        t¬          jW        k    r|  X                    ||j0        ¦  «         | j+        rd| j        _Y        || _Z        || _        | j[         \                    |¦  «        }t          |¦  «        rfts          |d¦  «        r| :                    ¦   «         }nAts          |d*¦  «        r"ts          |j]        d¦  «        r|j]        j        }nt½          d+¦  «        ‚ts          |d,¦  «        r|j_        | _`        nNt	          j        |ja        ¦  «        j        }tÅ          d-„ | .                    ¦   «         D ¦   «         ¦  «        | _`        |jc        | _c        |	| _d        || _e        |\  | _f        | _g        || _h        | jh        �| jf        �tG          d.¦  «        ‚|�| jf        €| jg        �tG          d/¦  «        ‚tÓ          ¦   «         r|| jf        �u| j                             ¦   «         D ]	}|j0        } | jf        jj        D ]0}t_          |d0         ¦  «        dk    r|d0         d         j0        } nŒ1||k    rt          d1¦  «        ‚| jC        s| jM        r| jf        €| jg        �tG          d2¦  «        ‚tÖ          tÙ          | j        jm        ¦  «        z   }|
€|n||
z   }
tÝ          |
| j        | jU        | jf        | jg        ¦  «        | _o        |  p                    | j        jq        rtä          ntæ          ¦  «         d| _t        d | _u        | j        jv        r|  w                    ¦   «          | j        jx        r tó          jz        | j        j{        d¬3¦  «         tù          | jR        ¦  «        s2tù          tU          | jR        d4d ¦  «        ¦  «        rtû          d5¦  «        ‚|j~        dk    r%|j        dk    rt                                d6¦  «         |�*�t          |¦  «        s|j~        dk    rt          d7¦  «        ‚|�:to          |tb          j�        j‚        jƒ        ¦  «        r|j„        rt          d8¦  «        ‚d | _…        d| _†        d| _‡        �t          ¦   «         �r |j‰        rt          d9¦  «        ‚�t          r�|j‹        �t          j�        jŽ        j‹        k    rnt            ;                    d:�t          j�        jŽ        j‹        › d;|j‹        › d<�t          j�        jŽ        j‹        › �¦  «         �t          j�        jŽ        j‹        |_‹        nSts          �t          j�        jŽ        d=¦  «        r3t            ;                    d:�t          j�        jŽ        j‹        › d>�¦  «         |j‹        s|j‰        rq|j�        d?k    rf|j0        tc          j0        d@¦  «        k    r&|j‹        r�t           st          dA¦  «        ‚ndB|_�        t                                dC|j�        › dD�¦  «         |j‹        s|j‰        rL| jE        sE�t          ¦   «         s6|j�        dBk    rd| _‡        tb          j‘        | _’        n|j�        dEk    rd| _†        | j        j“        dk    r!�t)          | j        j“        ¬F¦  «        | _•        nd | _•        | j        j“        dk    rAtU          | j        j–        dGd ¦  «        dHk    r"tI          j%        dI�t.          ¦  «         d | _•        �t1          ¦   «         | _™        �t5          |  ›                    ¦   «         |  œ                    ¦   «         dJ„ | jo        j�        | j™        gz   D ¦   «         ¬K¦  «        | _�        d| _ž        d | _Ÿ        | j        }t          | j        ¦  «        r@ts          | j        d¦  «        r| j         :                    ¦   «         }n| j        j]        j        }�tA          |j'        ¦  «        }| j        j¡        €|n| j        j¡        | _¡        �tE          |j'        ¦  «        | _¢        | jo         £                    | j        | j�        | j™        ¦  «        | _™        |j¤        | _¥        d| _¦        | j         §                    ¦   «          |jB         ¨                    dLd¦  «        | _©        | j©        ry�tT          st          dM¦  «        ‚�tW          j¬        ¦   «         } �t[          j®        �t[          j¯        �ta          j±        �te          | ¦  «        ¦  «        | dfdN¬O¦  «        ¦  «         | jC        o| j©         | _³        d S )QNÚtmp_trainerz1No `TrainingArguments` passed, using `output_dir=z`.)Ú
output_dirÚcompute_resultzõWhen using `batch_eval_metrics`, your `compute_metrics` function must take a `compute_result` boolean argument which will be triggered after the last batch of the eval set to signal that the summary statistics should be returned by the function.Únoz%You have set `args.eval_strategy` to zx but you didn't pass an `eval_dataset` to `Trainer`. Either set `args.eval_strategy` to `no` or pass an `eval_dataset`. zƒ`args.metric_for_best_model` must be provided when using 'best' save_strategy or if `args.load_best_model_at_end` is set to `True`.Fz<`Trainer` requires either a `model` or `model_init` argumentzÇ`Trainer` requires either a `model` or `model_init` argument, but not both. `model_init` will overwrite your model when calling the `train` method. This will become a fatal error in the next release.zThe model you have picked (a  ) cannot be used as is for training: it only computes hidden states and does not accept any labels. You should choose a model with a head suitable for your task like any of the `AutoModelForXxx` listed at https://huggingface.co/docs/transformers/model_doc/autoÚis_parallelizableÚmodel_parallelTÚhf_device_mapc                 ó   — g | ]}|d v¯|‘Œ	S ))ÚcpuÚdiskr½   )Ú.0Údevices     r´   ú
<listcomp>z$Trainer.__init__.<locals>.<listcomp>õ  s$   € ÐoÐoÐo &ÐQWÐ_nÐQnÐQn�vÐQnÐQnÐQnr¶   r   r   zªYou have loaded a model on multiple GPUs. `is_model_parallel` attribute will be force-set to `True` to avoid any unexpected behavior such as device placement mismatching.)Ú_apply_liger_kernel_to_instancer²   Úget_base_modelzRThe model is not an instance of PreTrainedModel. No liger kernels will be applied.z‡You have set `use_liger_kernel` to `True` but liger-kernel >= 0.3.0 is not available. Please install it with `pip install liger-kernel`Úis_quantizedÚ_hf_peft_config_loadedÚhf_quantizerÚis_qat_trainableÚ	_orig_modz�You cannot fine-tune quantized model with `torch.compile()` make sure to pass a non-compiled model when fine-tuning a quantized model with PEFTzæYou cannot perform fine-tuning on purely quantized models. Please attach trainable adapters on top of the quantized model to correctly perform fine-tuning. Please see: https://huggingface.co/docs/transformers/peft for more detailsz8The model you are trying to fine-tune is quantized with z³ but that quantization method do not support training. Please open an issue on GitHub: https://github.com/huggingface/transformers to request the support for training support for ÚxlazZUsing --fsdp xxx together with --deepspeed is not possible, deactivate one of those flags.z.Using fsdp only works in distributed training.Úquantization_methodÚ
base_modelz8Cannot extract base model safely from this PEFT wrapper.Úaccepts_loss_kwargsc              3   óJ   K  — | ]}|j         t          j        j        k    V — Œd S ©N)Úkindrº   Ú	ParameterÚVAR_KEYWORD)rö   Úks     r´   ú	<genexpr>z#Trainer.__init__.<locals>.<genexpr>  s@   è è € ð 1ð 1Ø<=�”�'Ô+Ô7Ò7ð1ð 1ð 1ð 1ð 1ð 1r¶   zSPassing both `optimizers` and `optimizer_cls_and_kwargs` arguments is incompatible.z¨Passing a `model_init` is incompatible with providing the `optimizers` argument. You should subclass `Trainer` and override the `create_optimizer_and_scheduler` method.Úparamsa[  The model and the optimizer parameters are not on the same device, which probably means you created an optimizer around your model **before** putting on the device and passing it to the `Trainer`. Make sure the lines `import torch_xla.core.xla_model as xm` and `model.to(xm.xla_device())` is performed before the optimizer creation in your script.z—Passing `optimizers` is not allowed if PyTorch FSDP is enabled. You should subclass `Trainer` and override the `create_optimizer_and_scheduler` method.©Úexist_okÚcollate_batchzRThe `data_collator` should be a simple callable (function, class with `__call__`).zHmax_steps is given, it will override any value given in num_train_epochszžThe train_dataset does not implement __len__, max_steps has to be specified. The number of steps needs to be known in advance for the learning rate scheduler.zTthe `--group_by_length` option is only available for `Dataset`, not `IterableDatasetzOSageMaker Model Parallelism does not support BF16 yet. Please use FP16 instead z(FP16 provided in SM_HP_MP_PARAMETERS is z+, but FP16 provided in trainer argument is z, setting to Úfp16zJ, but SageMaker Model Parallelism < 1.10 does not support FP16 in trainer.Úautorô   z2Tried to use `fp16` but it is not supported on cpuÚcpu_ampzUsing z half precision backendÚapex)ÚepsilonÚproblem_typeÚmulti_label_classificationzsLabel smoothing is not compatible with multi-label classification. Disabling label smoothing for this training run.c                 ó<   — g | ]}t          |t          ¦  «        ¯|‘ŒS r½   ©r±   r6   ©rö   Úcbs     r´   rø   z$Trainer.__init__.<locals>.<listcomp>  ó:   € ð  ð  ð  ØÕQ[Ð\^Õ`oÑQpÔQpð Øð ð  ð  r¶   )Úis_local_process_zeroÚis_world_process_zeroÚstateful_callbacksÚxla_fsdp_v2z*FSDPv2 requires `torch_xla` 2.2 or higher.)ÚfsdpÚtensor)Ú
axis_namesr½   )´ÚloggerÚinfori   Úbatch_eval_metricsrº   r»   r¼   Ú
ValueErrorÚeval_strategyÚsave_strategyrX   ÚBESTÚload_best_model_at_endÚmetric_for_best_modelrà   rå   Úfull_determinismr^   Úseedre   Úhp_nameÚ	deepspeedÚis_in_trainr²   Ú"create_accelerator_and_postprocessrY   Úskip_memory_metricsÚ_memory_trackerÚstartÚget_process_log_levelr�   Úset_verbosityÚ_setup_devicesrä   Úcall_model_initÚRuntimeErrorÚwarningsÚwarnÚFutureWarningÚ	__class__Ú__name__r.   ÚgetattrÚis_model_parallelÚsetrò   ÚvaluesÚlenr÷   rÀ   Úuse_liger_kernelr€   Úliger_kernel.transformersrù   Úliger_kernel_configr±   r*   Úhasattrrú   ÚwarningÚImportErrorrý   Úis_trainablerµ   Úquantization_configÚquant_methodÚfsdp_configÚis_fsdp_xla_enabledr  Úis_deepspeed_enabledÚparallel_moderh   ÚDISTRIBUTEDÚplace_model_on_deviceÚfp16_full_evalÚbf16_full_evalÚdo_trainÚis_fsdp_enabledr3   r    r   r   rá   râ   rã   rÜ   r”   ÚBITS_AND_BYTESÚ_move_model_to_deviceÚ_n_gpuÚmodel_wrappedÚacceleratorr,   r  ÚAttributeErrorr  Úmodel_accepts_loss_kwargsÚforwardÚanyÚneftune_noise_alpharæ   rê   Ú	optimizerÚlr_schedulerré   r�   Úparam_groupsÚDEFAULT_CALLBACKSr   Ú	report_tor4   Úcallback_handlerÚadd_callbackÚdisable_tqdmr7   ÚDEFAULT_PROGRESS_CALLBACKÚ_loggers_initializedÚhub_model_idÚpush_to_hubÚinit_hf_repoÚshould_saveÚosÚmakedirsrí   ÚcallableÚ	TypeErrorÚ	max_stepsÚnum_train_epochsra   ÚutilsÚdatar   Úgroup_by_lengthÚ_signature_columnsÚuse_apexÚuse_cpu_ampr„   Úbf16ÚIS_SAGEMAKER_MP_POST_1_10r  ÚsmpÚstateÚcfgÚhalf_precision_backendr2   Úbfloat16Ú	amp_dtypeÚlabel_smoothing_factorr?   Úlabel_smootherÚconfigÚUserWarningr:   Úcontrolr;   r  r  rç   Úcurrent_flosÚhp_search_backendrx   Úlabel_namesrv   Úon_init_endÚtrain_batch_sizeÚ_train_batch_sizeÚ_created_lr_schedulerÚstop_and_update_metricsÚgetÚis_fsdp_xla_v2_enabledÚIS_XLA_FSDPV2_POST_2_2ÚxrÚglobal_runtime_device_countÚxsÚset_global_meshÚMeshrÄ   ÚarrayÚrangeÚis_fsdp_xla_v1_enabled)!Úselfr²   rà   rá   râ   rã   rÜ   rä   rå   ræ   rç   rè   ré   rê   rí   Ú	log_levelÚdevicesrù   Úkernel_configÚ_is_quantized_and_base_modelÚ&_quantization_method_supports_trainingÚ%_is_model_quantized_and_qat_trainableÚdefault_collatorÚunwrapped_modelÚforward_paramsÚparamÚmodel_deviceÚparam_groupÚoptimizer_deviceÚdefault_callbacksÚmodel_to_inspectÚdefault_label_namesÚnum_devicess!                                    r´   Ú__init__zTrainer.__init__™  sb  € ð& ˆ<Ø&ˆJÝ�KŠKÐZÈJÐZÐZÐZÑ[Ô[Ð[Ý$°
Ð;Ñ;Ô;ˆDØÔ"ð 	 Ð'BØ¥wÔ'8¸Ñ'IÔ'IÔ'TÐTÐTÝ ðNñô ð ð
 ÔÐ)¨dÔ.@ÀDÒ.HÐ.HÈ\ÐMaÝð E¸Ô8Jð  Eð  Eð  Eñô ð ð Ô¥Ô!2Ò2Ð2°dÔ6QÐ2ØÔ)Ð1Ý ð Zñô ð ð ˆŒ	Ø!2ˆÔà37´9Ô3MÐkÕ ¤	¤Ñ/Ô/Ð/ÕS[Ð\`Ô\eÔ\jÑSkÔSkøàˆŒØˆŒØ ˆÔØˆŒ
Ø×/Ò/Ñ1Ô1Ð1õ  4°D´IÔ4QÑRÔRˆÔØÔ×"Ò"Ñ$Ô$Ð$ð ×.Ò.Ñ0Ô0ˆ	ÝÔ˜iÑ(Ô(Ð(ð 	ÔÐàˆ=ØÐ%Ø",�”Ø×,Ò,Ñ.Ô.��å"Ð#aÑbÔbÐbàÐ%Ý”ð õ "ñ	ô ð ð )ˆDŒOàŒ?Ô#Õ':Ð:Ð:ÝðJ¨e¬oÔ.Fð Jð Jð Jñô ð õ �5Ð-¨uÑ5Ô5ð 	+½'À%ÐIYÐ[`Ñ:aÔ:að 	+Ø%)ˆDÔ"Ð"à%*ˆDÔ"å�5˜/¨4Ñ0Ô0Ð<ØoÐo­C°Ô0C×0JÒ0JÑ0LÔ0LÑ,MÔ,MÐoÑoÔoˆGÝ�7‰|Œ|˜aÒÐØ)-�Ô&Ð&Ý�W‘” Ò"Ð"Ø)-¬Ô)9½U¼\È'ÐRSÌ*Ñ=UÔ=UÒ)U�Ô&Ð&à).�Ô&ð Ô%ð Ý—’ðhñô ð ð
 Œ9Ô%ð 	Ý(Ñ*Ô*ð ØUÐUÐUÐUÐUÐUð BFÄÔA^ÐAj ¤	Ô =Ð =Ðpr�å˜e¥_Ñ5Ô5ð 	à3Ð3ÐQÐQ¸%ÐQÀ=ÐQÐQÐQÐQÝ˜UÐ$4Ñ5Ô5ð ½*ÀU×EYÒEYÑE[ÔE[Õ]lÑ:mÔ:mð à3Ð3ÐbÐb¸%×:NÒ:NÑ:PÔ:PÐbÐTaÐbÐbÐbÐbå—N’NØlñô ð ð õ "ðHñô ð õ
 (/¨u°nÀeÑ'LÔ'Lð (
ÕU\ØÐ+¨UñV
ô V
ð R
Ð$õ �E˜>¨4Ñ0Ô0¸Ð<Ð`ÀÔASÔA`ð 	/õ 18¸¸~ÈtÑ0TÔ0TÐ\`Ð0`ð 1
ÕelØÔÐ 2°Eñf
ô f
Ð-ð
 (ð 	­G°E¸;Ñ,GÔ,Gð 	Ýð bñô ð ð
 (ð 	µ¸uÑ0EÔ0Eð 	ÐNsð 	Ýð$ñô ð ð
 *ð 	Ð2Xð 	ÝðzÈ5ÔK]ÔKqÔK~ð zð zàDIÔDVÔDjÔDwðzð zñô ð ð $(Ô#3°EÔ#:ˆÔ ÝˆtŒy‰>Œ>˜AÒÐØÔ(ð Ý Øpñô ð ð Ô# EÔ*ð S¨tÔ/AÅ\ÔE]Ò/]Ð/]Ý Ð!QÑRÔRÐRð &*Ô%?ˆÔ"àÔ"ð	/àÔ(ð	/ð Ô$ð	/ð )-Ô(;ð	/ð FJÄ]ð	/ð Ô'ð		/ð
 Ô#ð	/ð */ˆDÔ&ð  Ð+ÝÐ+Õ.EÕG_Ð-`ÑaÔað ,õ $Ð$4Ñ5Ô5Ð5õ 'ð	 	ð /<Ð.G˜]˜]ÐM]ˆÔØ*ˆÔØ(ˆÔØ 0ˆÔð Ô&ð	;å˜Ð4°dÑ;Ô;Õ?QÔ?`Ò`Ð`à×&Ò& u¨d¬kÑ:Ô:Ð:ð Ô!ð 	!Ø ˆDŒIÔð #ˆÔØˆŒ
ð Ô*×7Ò7¸Ñ>Ô>ˆå˜/Ñ*Ô*ð 	aÝ�Ð(8Ñ9Ô9ð aØ"1×"@Ò"@Ñ"BÔ"B��Ý˜¨,Ñ7Ô7ð a½GÀOÔD^Ð`gÑ<hÔ<hð aØ"1Ô"<Ô"B��å$Ð%_Ñ`Ô`Ð`õ �?Ð$9Ñ:Ô:ð 	Ø-<Ô-PˆDÔ*Ð*å$Ô.¨Ô/FÑGÔGÔRˆNÝ-0ð 1ð 1ØAO×AVÒAVÑAXÔAXð1ñ 1ô 1ñ .ô .ˆDÔ*ð $(Ô#;ˆÔ à.ˆÔØ-JˆÔ*Ø,6Ñ)ˆŒ˜Ô)Ø(@ˆÔ%ØÔ(Ð4¸¼Ð9SÝÐtÑuÔuÐuØÐ! t¤~Ð'AÀTÔEVÐEbÝðjñô ð õ "Ñ#Ô#ð 	¨¬Ð(BØœ×.Ò.Ñ0Ô0ð ð �Ø$œ|�ØØ#œ~Ô:ð ð �Ý�{ 8Ô,Ñ-Ô-°Ò1Ð1Ø'2°8Ô'<¸QÔ'?Ô'FÐ$Ø�Eð 2ð Ð/Ò/Ð/Ý ðnñô ð ð Ô$ð 	¨Ô(<ð 	ØŒNÐ&¨$Ô*;Ð*Gåðjñô ð õ .Õ0SÐTXÔT]ÔTgÑ0hÔ0hÑhÐØ)2Ð):Ð%Ð%Ð@QÐT]Ñ@]ˆ	Ý /Ø�t”z 4Ô#8¸$¼.È$ÔJ[ñ!
ô !
ˆÔð 	×Ò¨T¬YÔ-CÐb�/˜/ÕIbÑcÔcÐcð %*ˆÔ!ð !ˆÔØŒ9Ô ð 	 Ø×ÒÑÔÐØŒ9Ô ð 	=ÝŒK˜œ	Ô,°tÐ<Ñ<Ô<Ð<å˜Ô*Ñ+Ô+ð 	rµ½ÀÔASÐUdÐfjÑ9kÔ9kÑ0lÔ0lð 	rÝÐpÑqÔqÐqàŒ>˜AÒÐ $Ô"7¸!Ò";Ð";Ý�KŠKÐbÑcÔcÐcàÐ$®Z¸Ñ-FÔ-FÐ$È4Ì>Ð]^ÒK^ÐK^Ýðdñô ð ð Ð%Ý˜=­%¬+Ô*:Ô*JÑKÔKð &àÔ$ð &õ ÐsÑtÔtÐtà"&ˆÔð ˆŒØ ˆÔö #Ñ$Ô$ñ 	àŒyð tÝ Ð!rÑsÔsÐsæ(ð à”9¦¤	¤Ô 2Ò2Ð2Ý—N’Nð;Æ3Ä9Ä=ÔCUð ;ð ;ØDHÄIð;ð ;æ&)¤i¤mÔ&8ð;ð ;ñô ð ö
 !$¤	¤Ô 2�D”Iøõ ž3œ9œ=¨&Ñ1Ô1ð Ý—N’NðcÆ3Ä9Ä=ÔCUð cð cð cñô ð ð ŒIð 	W˜œð 	W¨Ô(CÀvÒ(MÐ(MØŒ{�eœl¨5Ñ1Ô1Ò1Ð1Ø”9ð <Þ=ð _Ý(Ð)]Ñ^Ô^Ð^ð_ð 3<�DÔ/Ý�KŠKÐU Ô!<ÐUÐUÐUÑVÔVÐVàŒIð 	%˜œð 	%¨TÔ-Fð 	%ÖJaÑJcÔJcð 	%àÔ*¨iÒ7Ð7Ø#'�Ô Ý!&¤�”�ØÔ,°Ò6Ð6Ø $�”ð Œ9Ô+¨qÒ0Ð0Þ"/¸¼	Ô8XÐ"YÑ"YÔ"YˆDÔÐà"&ˆDÔð Œ9Ô+¨aÒ/Ð/Ý�t”zÔ(¨.¸$Ñ?Ô?ÐC_Ò_Ð_Ý”ðGæñô ð ð
 '+�Ô#æ%Ñ'Ô'ˆŒæ!Ø"&×"<Ò"<Ñ">Ô">Ø"&×"<Ò"<Ñ">Ô">ð ð  Ø!Ô2Ô<ÀÄ¸~ÑMð ñ  ô  ð
ñ 
ô 
ˆŒ
ð ˆÔØ!%ˆÔàœ:ÐÝ˜$œ*Ñ%Ô%ð 	?Ý�t”zÐ#3Ñ4Ô4ð ?Ø#'¤:×#<Ò#<Ñ#>Ô#>Ð Ð ð $(¤:Ô#8Ô#>Ð Þ)Ð*:Ô*DÑEÔEÐØ26´)Ô2GÐ2OÐ.Ð.ÐUYÔU^ÔUjˆÔÞ.Ð/?Ô/IÑJÔJˆÔØÔ,×8Ò8¸¼ÀDÄJÐPTÔP\Ñ]Ô]ˆŒð "&Ô!6ˆÔØ%*ˆÔ"ð 	Ô×4Ò4Ñ6Ô6Ð6à&*Ô&6×&:Ò&:¸=È%Ñ&PÔ&PˆÔ#ØÔ&ð 	wÞ)ð OÝ Ð!MÑNÔNÐNö Ô8Ñ:Ô:ˆKÞÔžrœw¦r¤x¶°kÑ0BÔ0BÑ'CÔ'CÀkÐSTÐEUÐbtÐuÑuÔuÑvÔvÐvØ&*Ô&>Ð&bÀtÔGbÐCbˆÔ#Ð#Ð#r¶   Úreturnc                 óD   — t                                d¦  «         | j        S )NzUTrainer.tokenizer is now deprecated. You should use Trainer.processing_class instead.©r"  rG  rÜ   ©rš  s    r´   rÛ   zTrainer.tokenizer2  s   € å�ŠÐnÑoÔoÐoØÔ$Ð$r¶   c                 óH   — t                                d¦  «         || _        d S )NzjTrainer.tokenizer is now deprecated. You should use `Trainer.processing_class = processing_class` instead.r¯  )rš  rÜ   s     r´   rÛ   zTrainer.tokenizer7  s+   € å�ŠØxñ	
ô 	
ð 	
ð !1ˆÔÐÐr¶   c                 ó  — | j                              |¦  «        }t          |¦  «        r|j        j                             ¦   «         }n|                     ¦   «         }~| j        |_        |                     t          ¦  «        }|| _	        |S )z£
        Activates the neftune as presented in this code: https://github.com/neelsjain/NEFTune and paper:
        https://huggingface.co/papers/2310.05914
        )
rZ  r,   rµ   r  r²   Úget_input_embeddingsr_  Úregister_forward_hookrb   Úneftune_hook_handle)rš  r²   r¢  Ú
embeddingsÚhook_handles        r´   Ú_activate_neftunezTrainer._activate_neftune>  s„   € ð
 Ô*×7Ò7¸Ñ>Ô>ˆå˜/Ñ*Ô*ð 	@Ø(Ô3Ô9×NÒNÑPÔPˆJˆJà(×=Ò=Ñ?Ô?ˆJàà)-Ô)Aˆ
Ô&Ø ×6Ò6Õ7PÑQÔQˆØ#.ˆÔ Øˆr¶   c                 ó4  — t          | d¦  «        st          d¦  «        ‚| j                             |¦  «        }t	          |¦  «        r|j        j                             ¦   «         }n|                     ¦   «         }| j         	                    ¦   «          |`
~dS )z^
        Deactivates the neftune method. Make sure to call `_activate_neftune` first.
        rµ  zNNeftune is not activated make sure to call `trainer._activate_neftune()` firstN)rF  r%  rZ  r,   rµ   r  r²   r³  rµ  Úremover_  )rš  r²   r¢  r¶  s       r´   Ú_deactivate_neftunezTrainer._deactivate_neftuneQ  s™   € õ �tÐ2Ñ3Ô3ð 	oÝÐmÑnÔnÐnàÔ*×7Ò7¸Ñ>Ô>ˆå˜/Ñ*Ô*ð 	@Ø(Ô3Ô9×NÒNÑPÔPˆJˆJà(×=Ò=Ñ?Ô?ˆJàÔ ×'Ò'Ñ)Ô)Ð)ØÐ*¨O¨O¨Or¶   c                 ó:   — | j                              |¦  «         dS )ag  
        Add a callback to the current list of [`~transformers.TrainerCallback`].

        Args:
           callback (`type` or [`~transformers.TrainerCallback]`):
               A [`~transformers.TrainerCallback`] class or an instance of a [`~transformers.TrainerCallback`]. In the
               first case, will instantiate a member of that class.
        N)re  rf  ©rš  Úcallbacks     r´   rf  zTrainer.add_callbackb  s!   € ð 	Ô×*Ò*¨8Ñ4Ô4Ð4Ð4Ð4r¶   c                 ó6   — | j                              |¦  «        S )aK  
        Remove a callback from the current list of [`~transformers.TrainerCallback`] and returns it.

        If the callback is not found, returns `None` (and no error is raised).

        Args:
           callback (`type` or [`~transformers.TrainerCallback]`):
               A [`~transformers.TrainerCallback`] class or an instance of a [`~transformers.TrainerCallback`]. In the
               first case, will pop the first member of that class found in the list of callbacks.

        Returns:
            [`~transformers.TrainerCallback`]: The callback removed, if found.
        )re  Úpop_callbackr½  s     r´   rÀ  zTrainer.pop_callbackm  s   € ð Ô$×1Ò1°(Ñ;Ô;Ð;r¶   c                 ó:   — | j                              |¦  «         dS )aŽ  
        Remove a callback from the current list of [`~transformers.TrainerCallback`].

        Args:
           callback (`type` or [`~transformers.TrainerCallback]`):
               A [`~transformers.TrainerCallback`] class or an instance of a [`~transformers.TrainerCallback`]. In the
               first case, will remove the first member of that class found in the list of callbacks.
        N)re  Úremove_callbackr½  s     r´   rÂ  zTrainer.remove_callback}  s!   € ð 	Ô×-Ò-¨hÑ7Ô7Ð7Ð7Ð7r¶   c                 ó  — t          |dd ¦  «        �t                               d¦  «         d S |                     |¦  «        }| j        j        t          j        k    r&t          |d¦  «        r| 	                    ¦   «          d S d S d S )Nrò   zZThe model is already on multiple devices. Skipping the move to device specified in `args`.Útie_weights)
r>  r"  rG  Útorà   rO  rh   ÚTPUrF  rÄ  )rš  r²   r÷   s      r´   rW  zTrainer._move_model_to_deviceˆ  s�   € Ý�5˜/¨4Ñ0Ô0Ð<Ý�NŠNØlñô ð ð ˆFØ—’˜Ñ Ô ˆàŒ9Ô"¥lÔ&6Ò6Ð6½7À5È-Ñ;XÔ;XÐ6Ø×ÒÑÔÐÐÐð 7Ð6Ð6Ð6r¶   c                 ó&  — t          | j        t          ¦  «        r| j        j        }n| j        }t	          | j        d¦  «        o| j        j        du}i }|j        | j        j        j        k    }|r�| j        j        j        €||j        | j        j        j        k    z  }n`t          | j        j        j        t          ¦  «        r!| j        j        j        g| j        j        _        ||j        | j        j        j        vz  }|rw|j        |d<   |j        | j        j        _        |rU|j        g}| j        j        j        �!|t          | j        j        j        ¦  «        z  }d„ |D ¦   «         | j        j        _        |j        | j        j        j        k    }|r||j        | j        j        j        k    z  }|r8|j        |d<   |j        | j        j        _        |r|j        | j        j        _        |j        | j        j        j        k    }|r||j        | j        j        j        k    z  }|r8|j        |d<   |j        | j        j        _        |r|j        | j        j        _        t          |¦  «        dk    r t                               d|› d	�¦  «         dS dS )
aÖ  
        Aligns the special tokens of the tokenizer with the model configs.

        A new tokens may be defined in the tokenizer for fine-tuning purposes, e.g. an "end of turn" token may be
        added on chat models. In that case, we want the model configs to be aligned with the tokenizer, so that all
        downstream uses work as expected. This alignment should happen before training, to ensure the prediction step
        uses the new tokens as well.
        Úgeneration_configNÚeos_token_idc                 ó   — g | ]}|®|‘ŒS r  r½   )rö   Útokens     r´   rø   z1Trainer._align_special_tokens.<locals>.<listcomp>¼  s   € Ð<tÐ<tÐ<tÀuÐbgÐbs¸UÐbsÐbsÐbsr¶   Úbos_token_idÚpad_token_idr   zÞThe tokenizer has new PAD/BOS/EOS tokens that differ from the model config and generation config. The model config and generation config were aligned accordingly, being updated with the tokenizer's values. Updated tokens: ú.)r±   rÜ   r1   rÛ   rF  r²   rÈ  rÉ  r„  Úintr¹   rÌ  rÍ  rB  r"  rG  )rš  rÛ   Úmodel_has_generation_configÚupdated_tokensÚtokenizer_has_new_eosÚall_eos_tokensÚtokenizer_has_new_bosÚtokenizer_has_new_pads           r´   Ú_align_special_tokenszTrainer._align_special_tokens“  sÁ  € õ �dÔ+­^Ñ<Ô<ð 	.Ø15Ô1FÔ1PˆIˆIàÔ-ˆIå�D”JÐ 3Ñ4Ô4Ða¸¼Ô9UÐ]aÐ9að 	$ð ˆð !*Ô 6¸$¼*Ô:KÔ:XÒ XÐØ&ð 		qàŒzÔ+Ô8Ð@Ø%¨Ô)?À4Ä:ÔC_ÔClÒ)lÑlÐ%Ð%õ ˜dœjÔ:ÔGÍÑMÔMð lØAEÄÔA]ÔAjÐ@k�D”JÔ0Ô=à%¨Ô)?ÀtÄzÔGcÔGpÐ)pÑpÐ%à ð 		uØ-6Ô-CˆN˜>Ñ*Ø-6Ô-CˆDŒJÔÔ*ð +ð uØ"+Ô"8Ð!9�Ø”:Ô/Ô<ÐHØ"¥d¨4¬:Ô+GÔ+TÑ&UÔ&UÑU�NØ<tÐ<tÐP^Ð<tÑ<tÔ<t�”
Ô,Ô9ð !*Ô 6¸$¼*Ô:KÔ:XÒ XÐØ&ð 	iØ! YÔ%;¸t¼zÔ?[Ô?hÒ%hÑhÐ!à ð 	SØ-6Ô-CˆN˜>Ñ*Ø-6Ô-CˆDŒJÔÔ*Ø*ð SØ<EÔ<R�”
Ô,Ô9ð !*Ô 6¸$¼*Ô:KÔ:XÒ XÐØ&ð 	iØ! YÔ%;¸t¼zÔ?[Ô?hÒ%hÑhÐ!à ð 	SØ-6Ô-CˆN˜>Ñ*Ø-6Ô-CˆDŒJÔÔ*Ø*ð SØ<EÔ<R�”
Ô,Ô9õ ˆ~ÑÔ Ò"Ð"Ý�NŠNð=à+9ð=ð =ð =ñô ð ð ð ð #Ð"r¶   c                 ó¾  — | j         €Õ| j        }t          | j        ¦  «        r@t          | j        d¦  «        r| j                             ¦   «         }n| j        j        j        }t          j        |j        ¦  «        }t          |j
                             ¦   «         ¦  «        | _         | xj         t          t          ddg| j        z   ¦  «        ¦  «        z  c_         d S d S )Nrú   ÚlabelÚ	label_ids)rw  r²   rµ   rF  rú   r  rº   r»   r]  r¹   r¼   Úkeysr@  r‰  )rš  r©  r»   s      r´   Ú _set_signature_columns_if_neededz(Trainer._set_signature_columns_if_neededÜ  sÕ   € ØÔ"Ð*à#œzÐÝ˜dœjÑ)Ô)ð CÝ˜4œ:Ð'7Ñ8Ô8ð CØ'+¤z×'@Ò'@Ñ'BÔ'BÐ$Ð$ð (,¤zÔ'<Ô'BÐ$ÝÔ)Ð*:Ô*BÑCÔCˆIÝ&*¨9Ô+?×+DÒ+DÑ+FÔ+FÑ&GÔ&GˆDÔ#àÐ#Ô#¥t­C°¸+Ð0FÈÔIYÑ0YÑ,ZÔ,ZÑ'[Ô'[Ñ[Ð#Ô#Ð#Ð#ð +Ð*r¶   ÚdatasetÚdescriptionc                 óz  ‡— | j         j        s‰S |                      ¦   «          | j        }t	          t          ‰j        ¦  «        t          |¦  «        z
  ¦  «        }t          |¦  «        dk    rx|€dnd|› d�}t           	                    d|› d| j
        j        j        › dd                     |¦  «        › d	d                     |¦  «        › d
| j
        j        j        › d�¦  «         ˆfd„|D ¦   «         }t          |¦  «        dk    r<t          dd                     |¦  «        › dd                     |¦  «        › d�¦  «        ‚t          j        t"          j        ¦  «        t          j        d¦  «        k     r0‰                     ‰j        d         |‰j        d         ¬¦  «         ‰S ‰                     |¦  «        S )Nr   Ú zin the z setzThe following columns z) don't have a corresponding argument in `z!.forward` and have been ignored: ú, z. If z are not expected by `z/.forward`,  you can safely ignore this message.c                 ó&   •— g | ]}|‰j         v ¯|‘ŒS r½   )Úcolumn_names)rö   r	  rÜ  s     €r´   rø   z2Trainer._remove_unused_columns.<locals>.<listcomp>û  s&   ø€ ÐMÐMÐM˜°1¸Ô8LÐ3LÐ3L�1Ð3LÐ3LÐ3Lr¶   zGNo columns in the dataset match the model's forward method signature: (z-). The following columns have been ignored: [zp]. Please check the dataset and model. You may need to set `remove_unused_columns=False` in `TrainingArguments`.ú1.4.0rË   Úformat_kwargs)rË   Úcolumnsrä  )rà   Úremove_unused_columnsrÛ  rw  r¹   r@  râ  rB  r"  r#  r²   r<  r=  Újoinr%  r   r®   Údatasetsr   Ú
set_formatÚformatÚremove_columns)rš  rÜ  rÝ  Úsignature_columnsÚignored_columnsÚdset_descriptionrå  s    `     r´   Ú_remove_unused_columnszTrainer._remove_unused_columnsë  s  ø€ ØŒyÔ.ð 	ØˆNØ×-Ò-Ñ/Ô/Ð/Ø Ô3Ðå�s 7Ô#7Ñ8Ô8½3Ð?PÑ;QÔ;QÑQÑRÔRˆÝˆÑÔ !Ò#Ð#Ø%0Ð%8˜r˜rÐ>YÈÐ>YÐ>YÐ>YÐÝ�KŠKð7Ð)9ð 7ð 7Ø”JÔ(Ô1ð7ð 7ØTX×T]ÒT]Ð^mÑTnÔTnð7ð 7à—y’y Ñ1Ô1ð7ð 7àIMÌÔI]ÔIfð7ð 7ð 7ñô ð ð NÐMÐMÐMÐ/ÐMÑMÔMˆÝˆw‰<Œ<˜1ÒÐÝð@ÐZ^×ZcÒZcÐduÑZvÔZvð @ð @Ø=A¿YºYÀÑ=WÔ=Wð@ð @ð @ñô ð õ Œ=�Ô-Ñ.Ô.µ´¸wÑ1GÔ1GÒGÐGØ×ÒØ”^ FÔ+°WÈGÌNÐ[jÔLkð ñ ô ð ð ˆNà×)Ò)¨/Ñ:Ô:Ð:r¶   c                 ó¨   — | j         j        s|S |                      ¦   «          | j        }t	          ||t
          || j        j        j        ¬¦  «        }|S )z=Wrap the data collator in a callable removing unused columns.)rá   rì  r"  rÝ  Ú
model_name)	rà   ræ  rÛ  rw  rW   r"  r²   r<  r=  )rš  rá   rÝ  rì  Úremove_columns_collators        r´   Ú"_get_collator_with_removed_columnsz*Trainer._get_collator_with_removed_columns  sf   € ð ŒyÔ.ð 	!Ø Ð Ø×-Ò-Ñ/Ô/Ð/Ø Ô3Ðå"7Ø'Ø/ÝØ#Ø”zÔ+Ô4ð#
ñ #
ô #
Ðð 'Ð&r¶   c                 óž  — |€| j         }|�t          |¦  «        sd S | j        j        r—t	          ¦   «         rBt          |t          j        ¦  «        r(| j        j        |j	        v r|| j        j                 nd }nd }| j
        �| j
        j        d         nd }t          | j        j        | j        j        z  |||¬¦  «        S t          |¦  «        S )Nr   ©rÜ  ÚlengthsÚmodel_input_name)râ   ra   rà   rv  r|   r±   rè  r   Úlength_column_namerâ  rÜ   Úmodel_input_namesrA   r‹  Úgradient_accumulation_stepsr   )rš  râ   rö  r÷  s       r´   Ú_get_train_samplerzTrainer._get_train_sampler  sö   € ØÐ Ø Ô.ˆMØÐ ­
°=Ñ(AÔ(AÐ Ø�4ð Œ9Ô$ð 	0Ý$Ñ&Ô&ð ­:°mÅXÔEUÑ+VÔ+Vð ð ”yÔ3°}Ô7QÐQÐQð " $¤)Ô">Ô?Ð?àð �ð �à>BÔ>SÐ>_�Ô%Ô7¸Ô:Ð:Ðeið õ (Ø”	Ô*¨T¬YÔ-RÑRØ%ØØ!1ð	ñ ô ð õ ! Ñ/Ô/Ð/r¶   FÚ
batch_sizeÚ
sampler_fnÚis_trainingÚdataloader_keyc                 óÊ  — | j         }t          ¦   «         r2t          |t          j        ¦  «        r|                      ||¬¦  «        }n|                      | j         |¬¦  «        }||| j        j        | j        j	        | j        j
        dœ}t          |t          j        j        j        ¦  «        s^|� ||¦  «        |d<   | j        j        |d<   | j        j        |d<   |r.t#          t$          | j        j        | j        j        ¬¦  «        |d<   | j                             t-          |fi |¤Ž¦  «        }	|�0| j        j
        r$t/          | d	¦  «        r|	| j        |<   n	||	i| _        |	S )
zACreate a [`~torch.utils.data.DataLoader`] from the given dataset.)rÝ  )rü  Ú
collate_fnÚnum_workersÚ
pin_memoryÚpersistent_workersNÚsamplerÚ	drop_lastÚprefetch_factor)r  ÚrankÚworker_init_fnÚ_eval_dataloaders)rá   r|   r±   rè  r   rï  ró  rà   Údataloader_num_workersÚdataloader_pin_memoryÚdataloader_persistent_workersrÀ   rt  ru  r   Údataloader_drop_lastÚdataloader_prefetch_factorr   rd   Úprocess_indexrZ  Úpreparer   rF  r
  )
rš  rÜ  rÝ  rü  rý  rþ  rÿ  rá   Údataloader_paramsÚ
dataloaders
             r´   Ú_get_dataloaderzTrainer._get_dataloader:  s“  € ð Ô*ˆÝ Ñ"Ô"ð 	q¥z°'½8Ô;KÑ'LÔ'Lð 	qØ×1Ò1°'À{Ð1ÑSÔSˆGˆGà ×CÒCÀDÔDVÐdoÐCÑpÔpˆMð %Ø'Øœ9Ô;Øœ)Ô9Ø"&¤)Ô"Ið
ð 
Ðõ ˜'¥5¤;Ô#3Ô#CÑDÔDð 	ØÐ%Ø/9¨z¸'Ñ/BÔ/BÐ! )Ñ,Ø-1¬YÔ-KÐ˜kÑ*Ø37´9Ô3WÐÐ/Ñ0Øð Ý6=Ý¨T¬YÔ-MÐTXÔT]ÔTkð7ñ 7ô 7Ð!Ð"2Ñ3ð Ô%×-Ò-­j¸Ð.VÐ.VÐDUÐ.VÐ.VÑWÔWˆ
ð Ð%¨$¬)Ô*QÐ%Ý�tÐ0Ñ1Ô1ð FØ9C�Ô& ~Ñ6Ð6à*8¸*Ð)E�Ô&àÐr¶   c                 ó€   — | j         €t          d¦  «        ‚|                      | j         d| j        | j        d¬¦  «        S )a@  
        Returns the training [`~torch.utils.data.DataLoader`].

        Will use no sampler if `train_dataset` does not implement `__len__`, a random sampler (adapted to distributed
        training if necessary) otherwise.

        Subclass and override this method if you want to inject some custom behavior.
        Nz+Trainer: training requires a train_dataset.ÚTrainingT)rÜ  rÝ  rü  rý  rþ  )râ   r%  r  rŒ  rû  r°  s    r´   Úget_train_dataloaderzTrainer.get_train_dataloaderh  sQ   € ð ÔÐ%ÝÐJÑKÔKÐKà×#Ò#ØÔ&Ø"ØÔ-ØÔ.Øð $ñ 
ô 
ð 	
r¶   c                 óê  — |�t          |¦  «        sd S | j        j        ržt          ¦   «         r4t	          |t          j        ¦   «         t          j        ¦   «         ¬¦  «        S t          ¦   «         r?t	          |t          j
        ¦   «         t          j        ¦   «         | j        j        ¬¦  «        S t          |¦  «        S | j        j        rŠt          ¦   «         rBt!          |t"          j        ¦  «        r(| j        j        |j        v r|| j        j                 nd }nd }| j        �| j        j        d         nd }t/          | j        j        |||¬¦  «        S | j        j        dk    rt          |¦  «        S d S )N)Únum_replicasr  )r  r  rü  r   rõ  r   )ra   rà   Úuse_legacy_prediction_loopr�   rB   r’  Ú
world_sizeÚglobal_ordinalr„   r|  Údp_sizeÚdp_rankÚper_device_eval_batch_sizer   rv  r|   r±   rè  r   rø  râ  rÜ   rù  rA   Úeval_batch_size)rš  rã   rö  r÷  s       r´   Ú_get_eval_samplerzTrainer._get_eval_sampler|  s�  € ØÐ¥z°,Ñ'?Ô'?ÐØ�4ð Œ9Ô/ð 	7Ý%Ñ'Ô'ð 7Ý3Ø ­r¬}©¬ÅRÔEVÑEXÔEXðñ ô ð õ )Ñ*Ô*ð 7Ý3Ø Ý!$¤¡¤Ýœ™œØ#œyÔCð	ñ ô ð õ )¨Ñ6Ô6Ð6àŒ9Ô$ð 	Ý$Ñ&Ô&ð ­:°lÅHÔDTÑ+UÔ+Uð ð ”yÔ3°|Ô7PÐPÐPð ! ¤Ô!=Ô>Ð>àð �ð �à>BÔ>SÐ>_�Ô%Ô7¸Ô:Ð:Ðeið õ (Ø”	Ô)Ø$ØØ!1ð	ñ ô ð ð Œ9Ô 1Ò$Ð$Ý$ \Ñ2Ô2Ð2à�4r¶   c                 ót  — |€| j         €t          d¦  «        ‚t          |t          ¦  «        r|nd}t	          | d¦  «        r"|| j        v r| j        j        r| j        |         S t          |t          ¦  «        r| j         |         n
|�|n| j         }|                      |d| j        j	        | j
        |¬¦  «        S )a   
        Returns the evaluation [`~torch.utils.data.DataLoader`].

        Subclass and override this method if you want to inject some custom behavior.

        Args:
            eval_dataset (`str` or `torch.utils.data.Dataset`, *optional*):
                If a `str`, will use `self.eval_dataset[eval_dataset]` as the evaluation dataset. If a `Dataset`, will override `self.eval_dataset` and must implement `__len__`. If it is a [`~datasets.Dataset`], columns not accepted by the `model.forward()` method are automatically removed.
        Nz-Trainer: evaluation requires an eval_dataset.Úevalr
  Ú
Evaluation)rÜ  rÝ  rü  rý  rÿ  )rã   r%  r±   ÚstrrF  r
  rà   r  r  r   r!  )rš  rã   rÿ  s      r´   Úget_eval_dataloaderzTrainer.get_eval_dataloader©  sê   € ð Ð DÔ$5Ð$=ÝÐLÑMÔMÐMõ *4°LÅ#Ñ)FÔ)FÐR˜˜ÈFˆå�DÐ-Ñ.Ô.ð	:à $Ô"8Ð8Ð8Ø”	Ô7ð 9ð Ô)¨.Ô9Ð9õ ˜,­Ñ,Ô,ð#ˆDÔ˜lÔ+Ð+ð Ð'ð �àÔ"ð 	ð ×#Ò#Ø Ø$Ø”yÔ0ØÔ-Ø)ð $ñ 
ô 
ð 	
r¶   Útest_datasetc                 óR   — |                       |d| j        j        | j        ¬¦  «        S )aµ  
        Returns the test [`~torch.utils.data.DataLoader`].

        Subclass and override this method if you want to inject some custom behavior.

        Args:
            test_dataset (`torch.utils.data.Dataset`, *optional*):
                The test dataset to use. If it is a [`~datasets.Dataset`], columns not accepted by the
                `model.forward()` method are automatically removed. It must implement `__len__`.
        Útest)rÜ  rÝ  rü  rý  )r  rà   r   r!  )rš  r'  s     r´   Úget_test_dataloaderzTrainer.get_test_dataloaderÐ  s5   € ð ×#Ò#Ø ØØ”yÔ0ØÔ-ð	 $ñ 
ô 
ð 	
r¶   Únum_training_stepsc                 ó¾   — |                       ¦   «          t          r#t          j        j        j        r| j        j        }n| j        }|                      ||¬¦  «         dS )aZ  
        Setup the optimizer and the learning rate scheduler.

        We provide a reasonable default that works well. If you want to use something else, you can pass a tuple in the
        Trainer's init through `optimizers`, or subclass and override this method (or `create_optimizer` and/or
        `create_scheduler`) in a subclass.
        )r+  r`  N)Úcreate_optimizerr{  r|  r}  r~  r  r`  Úcreate_scheduler©rš  r+  r`  s      r´   Úcreate_optimizer_and_schedulerz&Trainer.create_optimizer_and_schedulerâ  s_   € ð 	×ÒÑÔÐÝ$ð 	'­¬¬Ô);ð 	'àœÔ0ˆIˆIàœˆIØ×ÒÐ1CÈyÐÑYÔYÐYÐYÐYr¶   c                 óF   — g d¢}t          |t          j        g|¦  «        }|S )a-  
        Get all parameter names that weight decay will be applied to.

        This function filters out parameters in two ways:
        1. By layer type (instances of layers specified in ALL_LAYERNORM_LAYERS)
        2. By parameter name patterns (containing 'bias', or variation of 'norm')
        )ÚbiasÚ	layernormÚrmsnormz(?:^|\.)norm(?:$|\.)z_norm(?:$|\.))rH   r   Ú	LayerNorm)rš  r²   Úforbidden_name_patternsÚdecay_parameterss       r´   Úget_decay_parameter_namesz!Trainer.get_decay_parameter_namesò  s.   € ð #qÐ"pÐ"pÐÝ.¨uµr´|°nÐF]Ñ^Ô^ÐØÐr¶   c                 ó
  ‡	— t          ¦   «         r| j        n| j        }| j        �€,|                      |¦  «        Š	ˆ	fd„|                     ¦   «         D ¦   «         | j        j        dœˆ	fd„|                     ¦   «         D ¦   «         ddœg}| j        �| j        \  }}n|  	                    | j        |¦  «        \  }}d|v r| 
                    d¦  «        }d|v r| 
                    d¦  «        }d|v r| 
                    d¦  «        } ||fi |¤Ž| _        d	t          |¦  «        v �r,|                     d
d¦  «        dk    �rddl}|j        j                             ¦   «         }d}|                     ¦   «         D ]·}t%          |t&          j        ¦  «        r›|t+          d„ |                     ¦   «         D ¦   «                              ¦   «         ¦  «        z  }t0                               d|› d|dz  › d�¦  «         |                     |dd
di¦  «         t0                               d|› d�¦  «         Œ¸t0                               d|dz  › d�¦  «         t          ¦   «         rt9          j        | j        ¦  «        | _        | j        S )a   
        Setup the optimizer.

        We provide a reasonable default that works well. If you want to use something else, you can pass a tuple in the
        Trainer's init through `optimizers`, or subclass and override this method in a subclass.
        Nc                 ó0   •— g | ]\  }}|‰v ¯	|j         ¯|‘ŒS r½   ©Úrequires_grad©rö   ÚnÚpr7  s      €r´   rø   z,Trainer.create_optimizer.<locals>.<listcomp>  s8   ø€ ð ð ð Ù"˜a ÀqÐL\ÐG\ÐG\ÐabÔapÐG\˜ÐG\ÐG\ÐG\r¶   )r  Úweight_decayc                 ó0   •— g | ]\  }}|‰v¯	|j         ¯|‘ŒS r½   r;  r=  s      €r´   rø   z,Trainer.create_optimizer.<locals>.<listcomp>  s8   ø€ ð ð ð Ù"˜a ÀqÐP`ÐG`ÐG`ÐefÔetÐG`˜ÐG`ÐG`ÐG`r¶   ç        r  r²   Úoptimizer_dictÚbitsandbytesÚ
optim_bitsé   r   c                 ó\   — i | ])}|                      ¦   «         |                     ¦   «         “Œ*S r½   )Údata_ptrÚnumel©rö   r?  s     r´   ú
<dictcomp>z,Trainer.create_optimizer.<locals>.<dictcomp>6  s*   € Ð']Ð']Ð']ÀA¨¯
ª
©¬°a·g²g±i´iÐ']Ð']Ð']r¶   zskipped ú: i   zM paramsÚweighté    zbitsandbytes: will optimize z in fp32z	skipped: )r„   rY  r²   r`  r8  Únamed_parametersrà   r@  ré   Úget_optimizer_cls_and_kwargsÚpopr%  r�  rD  ÚoptimÚGlobalOptimManagerÚget_instanceÚmodulesr±   r   Ú	EmbeddingÚsumr¼   rA  r"  r#  Úregister_module_overrideÚdebugr|  ÚDistributedOptimizer)
rš  Ú	opt_modelÚoptimizer_grouped_parametersÚoptimizer_clsÚoptimizer_kwargsrD  ÚmanagerÚskippedÚmoduler7  s
            @r´   r-  zTrainer.create_optimizerþ  s   ø€ õ +BÑ*CÔ*CÐS�DÔ&Ð&ÈÌˆ	àŒ>Ñ!Ø#×=Ò=¸iÑHÔHÐðð ð ð Ø&/×&@Ò&@Ñ&BÔ&Bðñ ô ð %)¤IÔ$:ð	ð ðð ð ð Ø&/×&@Ò&@Ñ&BÔ&Bðñ ô ð %(ð	ð ð,Ð(ð Ô,Ð8Ø26Ô2OÑ/�Ð/Ð/à26×2SÒ2SÐTXÔT]Ð_hÑ2iÔ2iÑ/�Ð/ð Ð+Ð+Ð+Ø/?×/CÒ/CÀHÑ/MÔ/MÐ,ð Ð*Ð*Ð*Ø/?×/CÒ/CÀGÑ/LÔ/LÐ,ð  Ð#3Ð3Ð3Ø/?×/CÒ/CÐDTÑ/UÔ/UÐ,à*˜]Ð+GÐ\Ð\ÐK[Ð\Ð\ˆDŒNà¥ ]Ñ!3Ô!3Ð3Ñ3Ð8H×8LÒ8LÈ\Ð[_Ñ8`Ô8`ÐdeÒ8eÑ8eØ#Ð#Ð#Ð#à&Ô,Ô?×LÒLÑNÔN�à�Ø'×/Ò/Ñ1Ô1ð Vð V�FÝ! &­"¬,Ñ7Ô7ð VØ¥3Ð']Ð']È×IZÒIZÑI\ÔI\Ð']Ñ']Ô']×'dÒ'dÑ'fÔ'fÑ#gÔ#gÑg˜ÝŸšÐ$R¨vÐ$RÐ$R¸À5¹Ð$RÐ$RÐ$RÑSÔSÐSØ×8Ò8¸ÀÈLÐZ\ÐK]Ñ^Ô^Ð^ÝŸšÐ%TÀFÐ%TÐ%TÐ%TÑUÔUÐUøÝ—’ÐA¨°%©ÐAÐAÐAÑBÔBÐBå"Ñ$Ô$ð 	FÝ Ô5°d´nÑEÔEˆDŒNàŒ~Ðr¶   c                 ób   — t          d„ | j                             ¦   «         D ¦   «         ¦  «        S )z9
        Get the number of trainable parameters.
        c              3   óL   K  — | ]}|j         ¯	|                     ¦   «         V — Œ d S r  )r<  rI  rJ  s     r´   r
  z7Trainer.get_num_trainable_parameters.<locals>.<genexpr>E  s1   è è € ÐQÐQ ÀÄÐQ�1—7’7‘9”9ÐQÐQÐQÐQÐQÐQr¶   )rW  r²   r¼   r°  s    r´   Úget_num_trainable_parametersz$Trainer.get_num_trainable_parametersA  s/   € õ ÐQÐQ d¤j×&;Ò&;Ñ&=Ô&=ÐQÑQÔQÑQÔQÐQr¶   c                 óZ   — | j         €t          d¦  «        ‚d„ | j         j        D ¦   «         S )zR
        Returns the learning rate of each parameter from self.optimizer.
        NúPTrainer optimizer is None, please make sure you have setup the optimizer before.c                 ó   — g | ]
}|d          ‘ŒS )Úlrr½   ©rö   Úgroups     r´   rø   z.Trainer.get_learning_rates.<locals>.<listcomp>M  s   € ÐEÐEÐE ��d”ÐEÐEÐEr¶   ©r`  r%  rb  r°  s    r´   Úget_learning_rateszTrainer.get_learning_ratesG  s4   € ð Œ>Ð!ÝÐoÑpÔpÐpØEÐE¨¬Ô)DÐEÑEÔEÐEr¶   r¤  c                 ó˜   — | j         €t          d¦  «        ‚|�| j         j        D ]}||d         v r|c S Œd„ | j         j        D ¦   «         S )a  
        Returns optimizer group for a parameter if given, else returns all optimizer groups for params.

        Args:
            param (`str` or `torch.nn.parameter.Parameter`, *optional*):
                The parameter for which optimizer group needs to be returned.
        Nrf  r  c                 ó   — g | ]
}|d          ‘ŒS )r  r½   ri  s     r´   rø   z/Trainer.get_optimizer_group.<locals>.<listcomp>]  s   € ÐIÐIÐI E��h”ÐIÐIÐIr¶   rk  )rš  r¤  rj  s      r´   Úget_optimizer_groupzTrainer.get_optimizer_groupO  sk   € ð Œ>Ð!ÝÐoÑpÔpÐpØÐØœÔ4ð !ð !�Ø˜E (œOÐ+Ð+Ø �L�L�Lð ,àIÐI¨T¬^Ô-HÐIÑIÔIÐIr¶   c                 óø  ‡ ‡‡'‡(— i Š'‰ j         rN‰ j                              dd¦  «                             d¦  «        D ]}|                     d¦  «        \  }}|‰'|<   Œ d‰ j        iŠ(‰ j        ‰ j        f‰ j        dœ}	 d‚dt          d	t          t          t          f         d
t          t          t          f         dt          dt          t          t          f         f
ˆ ˆˆ'ˆ(fd„}‰ j        t          j        k    r!t          }‰(                     dddœ¦  «         �nc‰ j        t          j        t          j        fv rKddlm} |}‰(                     |¦  «         ‰ j        t          j        k    r‰(                     ddi¦  «         �nù‰ j        t          j        k    r=	 ddlm} |}‰(                     |¦  «         �nÄ# t.          $ r t1          d¦  «        ‚w xY w‰ j        t          j        k    r=	 ddlm}	 |	}‰(                     |¦  «         �nr# t.          $ r t1          d¦  «        ‚w xY w‰ j        t          j        k    r=	 ddlm}
 |
}‰(                     |¦  «         �n # t.          $ r t1          d¦  «        ‚w xY w‰ j        t          j        t          j         t          j!        t          j"        t          j#        t          j$        t          j%        t          j&        t          j'        t          j(        t          j)        t          j*        t          j+        t          j,        t          j-        fv �r˜	 ddl.m}m/}m0} d}d}d}|}d‰ j        v rd}d‰ j        v rd}d‰ j        v r|}�n—d‰ j        v r|}d ‰ j        ‰ j        fi}�nzd!‰ j        v r|}‰'}�nkd"‰ j        v �ratc          ¦   «         rVte          j3        th          j5         2                    d#¦  «        ¦  «        te          j3        d$¦  «        k     rt1          d%¦  «        ‚dd&l.m6} |}to          ‰' 8                    d'‰ j        ¦  «        ¦  «        to          ‰' 8                    d(‰ j        ¦  «        ¦  «        to          ‰' 8                    d)d*¦  «        ¦  «        fto          ‰' 8                    d+d,¦  «        ¦  «        to          ‰' 8                    d-‰ j        ¦  «        ¦  «        d.œ}d/‰'v rts          ‰'d/         ¦  «        |d/<   d0‰'v rts          ‰'d0         ¦  «        |d0<   d1|i}d!‰ j        vr||d2<   ‰(                     |¦  «         ‰(                     |¦  «         n# t.          $ r t1          d3¦  «        ‚w xY wtc          ¦   «         rate          j3        th          j5         2                    d#¦  «        ¦  «        te          j3        d4¦  «        k     rtt           ;                    d5¦  «         �	n¼‰ j        t          j<        k    rí	 dd6l=m>} |}‰(                     |¦  «         ‰(                     t          ‰' 8                    d7d8¦  «        ¦  «        t�          t‚          ‰' 8                    d9d:¦  «        ¦  «        t�          t‚          ‰' 8                    d;d:¦  «        ¦  «        t�          t‚          ‰' 8                    d<d=¦  «        ¦  «        d>œ¦  «         �n×# t.          $ r t1          d?¦  «        ‚w xY w‰ j        t          jB        k    rt‚          j        jB        }�n’‰ j        t          jC        k    rt‚          j        jD        }�nj‰ j        t          jE        k    rt‚          j        j0        }�nB‰ j        t          jF        t          jG        t          jH        t          jI        t          jJ        t          jK        fv �r3t™          ¦   «         st/          d@¦  «        ‚ddAlMmN}mO}mP} t          jF        |t          jG        |t          jH        |t          jI        |t          jJ        |t          jK        |i}ts          ‰' Q                    dBdC¦  «        ¦  «        ts          ‰' Q                    dDdE¦  «        ¦  «        to          ‰' Q                    dFdG¦  «        ¦  «        ‰' Q                    dHdI¦  «        dJœ} |‰ j        ||¦  «        \  }Š(‰ j        t          jH        k    r‰(                     dddœ¦  «         �nÃ‰ j        t          jR        t          jS        fv �rt©          ¦   «         st/          dK¦  «        ‚ddLlUmV} t          jR        |t          jS        |i}ts          ‰' Q                    dBdC¦  «        ¦  «        ‰' Q                    dMdN¦  «        ‰' Q                    dOdP¦  «        ts          ‰' Q                    dDdE¦  «        ¦  «        to          ‰' Q                    dFdQ¦  «        ¦  «        ‰' Q                    dHdI¦  «        dRœ}|                     |¦  «          |‰ j        ||¦  «        \  }Š(�n’‰ j        t          jW        t          jX        fv r{t³          ¦   «         st/          dS¦  «        ‚tµ          dT¦  «        st/          dU¦  «        ‚‰€t1          dV¦  «        ‚ddWl[m\}m]} dX‰ j        v r|}n|}‰(                     dY‰i¦  «         �nø‰ j        t          j^        k    rçt¿          ¦   «         st1          dZ¦  «        ‚dd[l`ma} |}‰(                     to          ‰' 8                    d\d]¦  «        ¦  «        to          ‰' 8                    d^d_¦  «        ¦  «        to          ‰' 8                    d`da¦  «        ¦  «        to          ‰' 8                    dbda¦  «        ¦  «        to          ‰' 8                    dcdQ¦  «        ¦  «        ddœ¦  «         �nü‰ j        t          jb        t          jc        fv �rhtÉ          ¦   «         rGte          j3        th          j5         2                    de¦  «        ¦  «        te          j3        df¦  «        k     rt/          dg¦  «        ‚te          j3        th          j5         2                    dh¦  «        ¦  «        te          j3        di¦  «        k    rt/          dj¦  «        ‚te          j3        th          j5         2                    de¦  «        ¦  «        te          j3        dk¦  «        k    r	ddllemf}mg} nddllhmf}mg} ‰ j        t          jb        k    r|}n'‰ j        t          jc        k    r|}nt1          dm¦  «        ‚‰(                     |¦  «         �nt‰ j        t          ji        t          jj        t          jk        fv �rNtÙ          ¦   «         st/          dn¦  «        ‚tµ          dT¦  «        st/          do¦  «        ‚ddplmmn}mo}  i }d}!‰ j        t          ji        k    r+tÙ          dq¦  «        st/          dr¦  «        ‚ddslmmp}" |"}|}d}!nA‰ j        t          jj        k    r|}|}n'‰ j        t          jk        k    r| }nt1          dt¦  «        ‚‰ jq        |du<   |!r
‰ jr        |dv<   |                     to          ‰' 8                    dwd_¦  «        ¦  «        to          ‰' 8                    dxdy¦  «        ¦  «        dzœ¦  «         ‰(                     |¦  «         nû‰ j        t          js        k    rÏté          ¦   «         st/          d{¦  «        ‚dd|lumv}# ‰' Q                    d}d¦  «        }$|$�to          |$¦  «        }$‰' Q                    d~d¦  «        }%|%�t          |%¦  «        }%‰ jq        |du<   t          ‰' Q                    dd¦  «        ¦  «        |$|%d€œ}&|#}‰(                     |¦  «         ‰(                     |&¦  «         nt1          d�‰ j        › �¦  «        ‚|‰(fS )ƒzû
        Returns the optimizer class and optimizer parameters based on the training arguments.

        Args:
            args (`transformers.training_args.TrainingArguments`):
                The training arguments for the training session.

        ú rß  ú,ú=rh  )ÚbetasÚepsTÚoptimizer_nameÚoptimizer_mappingÚoptim_kwargsÚis_layerwise_supportedr­  c                 óº  •‡‡— |                       ¦   «                              d¦  «        }|r*‰j        t          j        k    r|rt          d| › d�¦  «        ‚||          }‰j        €t          d| › d�¦  «        ‚t          ‰j        t          t          f¦  «        st          d‰j        › �¦  «        ‚‰€t          d| › d	�¦  «        ‚t          ‰j        t          ¦  «        o‰j                             d
d¦  «        dk    }g Š‰                     ¦   «         D ]{\  }}t          ‰j        |d¬¦  «        \  }	}
t          |t          j        ¦  «        s%|	r"|
s t"                               |› d| › d�¦  «         Œ^|	s|sŒc‰                     |dz   ¦  «         Œ|t)          ‰¦  «        dk    rt          d| › d‰j        › d�¦  «        ‚ˆfd„‰                     ¦   «         D ¦   «         }ˆfd„‰                     ¦   «         D ¦   «         }|                     ‰¦  «         d|id|i|¥g}|r¢‰j        dk    rt          d| › d�¦  «        ‚i Š|D ]} |d|gigfi ‰¤Ž‰|<   Œ|D ]} |d|gi|¥gfi ‰¤Ž‰|<   Œˆfd„}‰                     ¦   «         D ]}|j        r|                     |¦  «         Œt6          }‰                     d‰i¦  «         ‰                     d|i¦  «         |‰fS )a  
            Helper function to set up low-rank optimizers like GaLore and Apollo.

            Args:
                optimizer_name (str): Name of the optimizer.
                optimizer_mapping (dict): Mapping of optimizer names to their classes.
                optim_kwargs (dict): Keyword arguments for the optimizer.
                is_layerwise_supported (bool): Whether layerwise optimization is supported.

            Returns:
                tuple[Any, Any]: Optimizer class and updated optimizer kwargs.
            Ú	layerwisezLayer-wise z" does not support DDP at this timeNz1You need to define `optim_target_modules` to use z optimizerszX`optim_target_modules` must be a list of strings, a regex string, or 'all-linear'. Got: z'You need to pass a model to initialize z optimizer.Ú_ú-z
all-linearT)Úreturn_is_regexz matched but ignored. z only supports linear layers.z.weightr   zNo target modules found for z (ú).c                 ó"   •— g | ]\  }}|‰v ¯	|‘ŒS r½   r½   ©rö   r>  r?  Útarget_params_namess      €r´   rø   zZTrainer.get_optimizer_cls_and_kwargs.<locals>.setup_low_rank_optimizer.<locals>.<listcomp>·  s(   ø€ Ð`Ð`Ð`¡4 1 aÀqÐL_ÐG_ÐG_˜QÐG_ÐG_ÐG_r¶   c                 ó"   •— g | ]\  }}|‰v¯	|‘ŒS r½   r½   r�  s      €r´   rø   zZTrainer.get_optimizer_cls_and_kwargs.<locals>.setup_low_rank_optimizer.<locals>.<listcomp>¸  s(   ø€ Ð hÐ hÐ h¡t q¨!È1ÐTgÐKgÐKg ÐKgÐKgÐKgr¶   r  r   z
Layerwise z( does not support gradient accumulation!c                 ó‚   •— | j         �6‰|                               ¦   «          ‰|                               ¦   «          d S d S r  )ÚgradÚstepÚ	zero_grad)r¤  rC  s    €r´   Úoptimizer_hookz^Trainer.get_optimizer_cls_and_kwargs.<locals>.setup_low_rank_optimizer.<locals>.optimizer_hookÊ  sG   ø€ Ø”zÐ-Ø& uÔ-×2Ò2Ñ4Ô4Ð4Ø& uÔ-×7Ò7Ñ9Ô9Ð9Ð9Ð9ð .Ð-r¶   rC  )ÚlowerÚendswithrO  rh   rP  ÚNotImplementedErrorÚoptim_target_modulesr%  r±   r¹   r%  rq  ÚreplaceÚnamed_modulesr[   r   ÚLinearr"  rG  ÚappendrB  rO  Úupdaterú  r¼   r<  Ú"register_post_accumulate_grad_hookr@   )rv  rw  rx  ry  Úis_layerwiser]  Ú
all_linearÚmodule_namera  Útarget_module_existsÚis_regexÚtarget_paramsÚnon_target_paramsrb  r¤  rˆ  rC  r‚  rà   r²   Ú
optim_argsr^  s                   @@€€€€r´   Úsetup_low_rank_optimizerzFTrainer.get_optimizer_cls_and_kwargs.<locals>.setup_low_rank_optimizerz  s  øøø€ ð$ *×/Ò/Ñ1Ô1×:Ò:¸;ÑGÔGˆLØð l Ô 2µlÔ6NÒ NÐ NÐSiÐ NÝ)Ð*j¸Ð*jÐ*jÐ*jÑkÔkÐkà-¨nÔ=ˆMàÔ(Ð0Ý Ð!pÐUcÐ!pÐ!pÐ!pÑqÔqÐqå˜dÔ7½$Å¸ÑEÔEð Ýð KÐosô  pIð  Kð  Kñô ð ð ˆ}Ý Ð!fÈ>Ð!fÐ!fÐ!fÑgÔgÐgõ ˜4Ô4µcÑ:Ô:ð PØÔ-×5Ò5°c¸3Ñ?Ô?À<ÒOð ð
 #%ÐØ',×':Ò':Ñ'<Ô'<ð Dð DÑ#�˜VÝ1KØÔ-¨{ÈDð2ñ 2ô 2Ñ.Ð$ hõ " &­"¬)Ñ4Ô4ð Ø+ð °Hð ÝŸšØ*ÐoÐoÀ.ÐoÐoÐoñô ð ð à+ð °Jð Øà#×*Ò*¨;¸Ñ+BÑCÔCÐCÐCåÐ&Ñ'Ô'¨1Ò,Ð,Ý Ð!oÀÐ!oÐ!oÐRVÔRkÐ!oÐ!oÐ!oÑpÔpÐpà`Ð`Ð`Ð`¨5×+AÒ+AÑ+CÔ+CÐ`Ñ`Ô`ˆMØ hÐ hÐ hÐ h¨u×/EÒ/EÑ/GÔ/GÐ hÑ hÔ hÐØ×Ò 
Ñ+Ô+Ð+ð Ð,Ð-Ø˜=Ð9¨LÐ9ðˆLð
 ð LØÔ3°qÒ8Ð8Ý$Ð%j°.Ð%jÐ%jÐ%jÑkÔkÐkà!#�Ø.ð eð e�EØ,9¨M¸HÀuÀgÐ;NÐ:OÐ,dÐ,dÐScÐ,dÐ,d�N 5Ñ)Ð)Ø*ð uð u�EØ,9¨M¸HÀuÀgÐ;^ÐQ]Ð;^Ð:_Ð,tÐ,tÐcsÐ,tÐ,t�N 5Ñ)Ð)ð:ð :ð :ð :ð :ð
 #×-Ò-Ñ/Ô/ð Qð Q�EØÔ*ð QØ×@Ò@ÀÑPÔPÐPøå 7�Ø ×'Ò'Ð)9¸>Ð(JÑKÔKÐKà×#Ò# X¨|Ð$<Ñ=Ô=Ð=Ø Ð"2Ð2Ð2r¶   F)Úscale_parameterÚrelative_stepr   )ÚAdamWÚfusedz7Trainer failed to import syncfree AdamW from torch_xla.)ÚNpuFusedAdamWz3Trainer failed to import FusedAdamW from torch_npu.)Ú	FusedAdamzFTrainer tried to instantiate apex FusedAdam but apex is not installed!)rž  ÚLionÚRMSproprN  NÚpagedÚ8bitrF  ÚadamÚlionrt  ÚrmspropÚademamixrD  z0.44.0z{The AdEMAMix optimizer is not supported by your current version of `bitsandbytes`. Please install `bitsandbytes` >= 0.44.0.)ÚAdEMAMixÚbeta1Úbeta2Úbeta3g§èH.ÿï?Úalphag      @ru  )rt  r®  ru  Út_alphaÚt_beta3rE  Úis_pagedzOTrainer tried to instantiate bnb optimizer but `bitsandbytes` is not installed!z0.41.1z¥You are using 8-bit optimizers with a version of `bitsandbytes` < 0.41.1. It is recommended to update your version as a major bug has been fixed in 8-bit optimizers.)ÚAnyPrecisionAdamWÚuse_kahan_summationÚFalseÚmomentum_dtypeÚfloat32Úvariance_dtypeÚcompensation_buffer_dtyper€  )r³  rµ  r·  r¸  z4Please install https://github.com/pytorch/torchdistxzŒYou need to install `galore_torch` in order to use GaLore optimizers install it with `pip install git+https://github.com/jiaweizzhao/GaLore`)ÚGaLoreAdafactorÚGaLoreAdamWÚGaLoreAdamW8bitr  é€   Úupdate_proj_gapéÈ   Úscaleg      Ð?Ú	proj_typeÚstd)r  r½  r¿  rÀ  z‹You need to install `apollo_torch` in order to use APOLLO optimizers install it with `pip install git+https://github.com/zhuhanqing/APOLLO`)ÚAPOLLOAdamWÚprojÚrandomÚ
scale_typeÚchannelç      ð?)r  rÃ  rÅ  r½  r¿  rÀ  ziYou need to install `lomo_optim` in order to use LOMO optimizers install it with `pip install lomo-optim`ú0.30.0zGYou need to have `accelerate>=0.30.0` to be able to use LOMO optimizerszMYou need to pass a `model` in order to correctly initialize a LOMO optimizer.)ÚAdaLomoÚLomoÚadar²   z5Please install grokadamw with `pip install grokadamw`)Ú	GrokAdamWÚ
alpha_initg\�Âõ(\ï?Úlambg       @Úgammagš™™™™™¹?Úgrokking_signal_decay_rateÚgradient_clipping)rÍ  rÎ  rÏ  rÐ  rÑ  Útorchaoz0.4.0z­You need to have `torchao>=0.4.0` in order to use torch 4-bit optimizers.Install it with `pip install torchao` or follow the instructions here: https://github.com/pytorch/aorÀ   z2.4z¹You need to have `torch>2.4` in order to use torch 4-bit optimizers. Install it with `pip install --upgrade torch` it is available on pipy. Otherwise, you need to install torch nightly.z0.11.0)Ú	AdamW4bitÚ	AdamW8bitzInvalid optimizerzwYou need to install `schedulefree` in order to use schedulefree optimizers. Install it with `pip install schedulefree.`zOYou need to have `accelerate>=0.30.0` to be able to use schedulefree optimizers)ÚAdamWScheduleFreeÚSGDScheduleFreerã  z‚You need to install `schedulefree>=1.4.0` in order to use RAdamScheduleFree optimizer. Install it with `pip install schedulefree.`)ÚRAdamScheduleFreezInvalid schedulefree optimizerr@  Úwarmup_stepsÚweight_lr_powerÚrrB  )rÙ  rÚ  zwYou need to install `torch-optimi` in order to use stable_adamw optimizers. Install it with `pip install torch-optimi`.)ÚStableAdamWÚmax_lrÚ	kahan_sumÚdecouple_lr)rÞ  rÜ  rÝ  z2Trainer cannot instantiate unsupported optimizer: ©T)wrš  r�  ÚsplitÚlearning_rateÚ
adam_beta1Ú
adam_beta2Úadam_epsilonr%  Údictr	   ÚboolÚtuplerR  rg   Ú	ADAFACTORr/   r‘  ÚADAMW_TORCHÚADAMW_TORCH_FUSEDÚtorch.optimrž  ÚADAMW_TORCH_XLAÚtorch_xla.amp.syncfreerH  r%  ÚADAMW_TORCH_NPU_FUSEDÚtorch_npu.optimr   ÚADAMW_APEX_FUSEDÚapex.optimizersr¡  Ú	ADAMW_BNBÚ
ADAMW_8BITÚPAGED_ADAMWÚPAGED_ADAMW_8BITÚADEMAMIXÚADEMAMIX_8BITÚPAGED_ADEMAMIXÚPAGED_ADEMAMIX_8BITÚLIONÚ	LION_8BITÚ
PAGED_LIONÚPAGED_LION_8BITÚRMSPROP_BNBÚRMSPROP_8BITÚRMSPROP_32BITÚbitsandbytes.optimr¢  r£  r{   r   r®   r¯   r°   rª  Úfloatr�  rÏ  r"  rG  ÚADAMW_ANYPRECISIONÚtorchdistx.optimizersr²  r‘   r>  rÀ   ÚSGDÚADAGRADÚAdagradÚRMSPROPÚGALORE_ADAMWÚGALORE_ADAMW_8BITÚGALORE_ADAFACTORÚGALORE_ADAMW_LAYERWISEÚGALORE_ADAMW_8BIT_LAYERWISEÚGALORE_ADAFACTOR_LAYERWISEr}   Úgalore_torchr¹  rº  r»  rQ  ÚAPOLLO_ADAMWÚAPOLLO_ADAMW_LAYERWISErz   Úapollo_torchrÂ  ÚLOMOÚADALOMOr�   ry   Ú
lomo_optimrÉ  rÊ  Ú	GROKADAMWr~   Ú	grokadamwrÌ  ÚADAMW_TORCH_4BITÚADAMW_TORCH_8BITr�   Útorchao.optimrÓ  rÔ  Útorchao.prototype.low_bit_optimÚSCHEDULE_FREE_RADAMÚSCHEDULE_FREE_ADAMWÚSCHEDULE_FREE_SGDr…   ÚschedulefreerÕ  rÖ  r×  r@  rØ  ÚSTABLE_ADAMWrŒ   ÚoptimirÛ  ))rà   r²   ÚmappingÚkeyÚvalueÚadam_kwargsr›  r]  rž  r   r¡  r¢  r£  r±  rE  Úadditional_optim_kwargsrª  Ú
bnb_kwargsr²  r¹  rº  r»  rw  Úgalore_optim_kwargsrÂ  Úapollo_optim_kwargsrÉ  rÊ  rÌ  rÓ  rÔ  rÕ  rÖ  Úrequire_warmupr×  rÛ  rÜ  rÝ  Ústable_adamw_kwargsrš  r^  s)   ``                                     @@r´   rP  z$Trainer.get_optimizer_cls_and_kwargs_  s×  øøøø€ ð ˆ
ØŒ?ð 	(Øœ?×2Ò2°3¸Ñ;Ô;×AÒAÀ#ÑFÔFð (ð (�Ø$Ÿ]š]¨3Ñ/Ô/‘
��UØ"'�
˜3‘�à  $Ô"4Ð5Ðð ”o t¤Ð7ØÔ$ð
ð 
ˆð ,0ð	]	3ð ]	3Ýð]	3å#¥C­ Hœ~ð]	3õ �s¥C˜xœ.ð]	3õ %)ð	]	3õ
 •3��8Œ_ð]	3ð ]	3ð ]	3ð ]	3ð ]	3ð ]	3ð ]	3ð ]	3ð ]	3ð~ Œ:�Ô1Ò1Ð1Ý%ˆMØ×#Ò#¸ÐPUÐ$VÐ$VÑWÔWÐWÑWØŒZ�NÔ6½Ô8XÐYÐYÐYØ)Ð)Ð)Ð)Ð)Ð)à!ˆMØ×#Ò# KÑ0Ô0Ð0ØŒz�^Ô=Ò=Ð=Ø ×'Ò'¨°$¨Ñ8Ô8Ð8ùØŒZ�>Ô9Ò9Ð9ð\Ø8Ð8Ð8Ð8Ð8Ð8à %�Ø ×'Ò'¨Ñ4Ô4Ð4Ñ4øÝð \ð \ð \Ý Ð!ZÑ[Ô[Ð[ð\øøøàŒZ�>Ô?Ò?Ð?ðXØ9Ð9Ð9Ð9Ð9Ð9à -�Ø ×'Ò'¨Ñ4Ô4Ð4Ñ4øÝð Xð Xð XÝ Ð!VÑWÔWÐWðXøøøàŒZ�>Ô:Ò:Ð:ðkØ5Ð5Ð5Ð5Ð5Ð5à )�Ø ×'Ò'¨Ñ4Ô4Ð4Ñ4øÝð kð kð kÝ Ð!iÑjÔjÐjðkøøøàŒZÝÔ$ÝÔ%ÝÔ&ÝÔ+ÝÔ#ÝÔ(ÝÔ)ÝÔ.ÝÔÝÔ$ÝÔ%ÝÔ*ÝÔ&ÝÔ'ÝÔ(ð
ð 
ñ 
ð"8tØCÐCÐCÐCÐCÐCÐCÐCÐCÐCà �Ø�
Ø $�Ø*5Ð'Ø˜dœjÐ(Ð(Ø#�HØ˜TœZÐ'Ð'Ø!"�JØ˜TœZÐ'Ð'Ø$)�M‘MØ˜tœzÐ)Ð)Ø$(�MØ/6¸¼È$Ì/Ð8ZÐ.[Ð+Ñ+Ø $¤*Ð,Ð,Ø$+�Mð /9Ð+Ñ+Ø 4¤:Ð-Ñ-Ý0Ñ2Ô2ð µw´}Ý!Ô*×2Ò2°>ÑBÔBñ8ô 8åœ hÑ/Ô/ò80ð 80õ )ðGñô ð ð
 <Ð;Ð;Ð;Ð;Ð;à$,�Mõ " *§.¢.°¸$¼/Ñ"JÔ"JÑKÔKÝ! *§.¢.°¸$¼/Ñ"JÔ"JÑKÔKÝ! *§.¢.°¸&Ñ"AÔ"AÑBÔBð"õ
 "' z§~¢~°g¸sÑ'CÔ'CÑ!DÔ!DÝ$ Z§^¢^°E¸4Ô;LÑ%MÔ%MÑNÔNð/ð /Ð+ð ! JÐ.Ð.Ý=@ÀÈIÔAVÑ=WÔ=WÐ/°	Ñ:à  JÐ.Ð.Ý=@ÀÈIÔAVÑ=WÔ=WÐ/°	Ñ:à*¨JÐ7�
Ø D¤JÐ.Ð.Ø-5�J˜zÑ*à ×'Ò'Ð(?Ñ@Ô@Ð@Ø ×'Ò'¨
Ñ3Ô3Ð3Ð3øÝð tð tð tÝ Ð!rÑsÔsÐsðtøøøå(Ñ*Ô*ð ­w¬}ÝÔ"×*Ò*¨>Ñ:Ô:ñ0ô 0å”˜hÑ'Ô'ò0(ð 0(õ —’ðrñô ð ùð ŒZ�>Ô<Ò<Ð<ðYØCÐCÐCÐCÐCÐCà 1�Ø ×'Ò'¨Ñ4Ô4Ð4ð !×'Ò'å/8¸¿ºÐH]Ð_fÑ9gÔ9gÑ/hÔ/hÝ*1µ%¸¿ºÐHXÐZcÑ9dÔ9dÑ*eÔ*eÝ*1µ%¸¿ºÐHXÐZcÑ9dÔ9dÑ*eÔ*eÝ5<Ý! :§>¢>Ð2MÈzÑ#ZÔ#Zñ6ô 6ð	ð ñ	ô 	ð 	ñ 	øõ ð Yð Yð YÝ Ð!WÑXÔXÐXðYøøøàŒZ�>Ô-Ò-Ð-Ý!œKœOˆM‰MØŒZ�>Ô1Ò1Ð1Ý!œKÔ/ˆM‰MØŒZ�>Ô1Ò1Ð1Ý!œKÔ/ˆM‰MØŒZÝÔ'ÝÔ,ÝÔ+ÝÔ1ÝÔ6ÝÔ5ð
ð 
ñ 
õ -Ñ.Ô.ð Ý!ð_ñô ð ð SÐRÐRÐRÐRÐRÐRÐRÐRÐRõ Ô+¨[ÝÔ0°/ÝÔ/°ÝÔ5°{ÝÔ:¸OÝÔ9¸?ð!Ðõ ˜JŸNšN¨6°3Ñ7Ô7Ñ8Ô8Ý#& z§~¢~Ð6GÈÑ'MÔ'MÑ#NÔ#NÝ˜zŸ~š~¨g°tÑ<Ô<Ñ=Ô=Ø'Ÿ^š^¨K¸Ñ?Ô?ð	#ð #Ðð /GÐ.FØ”
Ð-Ð/Bñ/ô /Ñ+ˆMÐ+ð Œz�^Ô<Ò<Ð<Ø ×'Ò'¸EÐTYÐ(ZÐ(ZÑ[Ô[Ð[ùØŒZÝÔ'ÝÔ1ð
ð 
ñ 
õ -Ñ.Ô.ð Ý!ð^ñô ð ð 1Ð0Ð0Ð0Ð0Ð0õ Ô+¨[ÝÔ5°{ð!Ðõ ˜JŸNšN¨6°3Ñ7Ô7Ñ8Ô8Ø"Ÿš v¨xÑ8Ô8Ø(Ÿnšn¨\¸9ÑEÔEÝ#& z§~¢~Ð6GÈÑ'MÔ'MÑ#NÔ#NÝ˜zŸ~š~¨g°sÑ;Ô;Ñ<Ô<Ø'Ÿ^š^¨K¸Ñ?Ô?ð#ð #Ðð  ×&Ò& {Ñ3Ô3Ð3à.FÐ.FØ”
Ð-Ð/Bñ/ô /Ñ+ˆMÐ+Ñ+ð ŒZ�NÔ/µÔ1GÐHÐHÐHÝ$Ñ&Ô&ð Ý!ð@ñô ð õ +¨8Ñ4Ô4ð mÝ!Ð"kÑlÔlÐlàˆ}Ý Ð!pÑqÔqÐqà0Ð0Ð0Ð0Ð0Ð0Ð0Ð0à˜œ
Ð"Ð"Ø '��à $�à×#Ò# W¨eÐ$4Ñ5Ô5Ð5Ñ5ØŒZ�>Ô3Ò3Ð3Ý)Ñ+Ô+ð ZÝ Ð!XÑYÔYÐYà+Ð+Ð+Ð+Ð+Ð+à%ˆMØ×#Ò#å"'¨
¯ª°|ÀTÑ(JÔ(JÑ"KÔ"KÝ! *§.¢.°¸Ñ"=Ô"=Ñ>Ô>Ý" :§>¢>°'¸3Ñ#?Ô#?Ñ@Ô@Ý27¸
¿ºÐGcÐehÑ8iÔ8iÑ2jÔ2jÝ).¨z¯~ª~Ð>QÐSVÑ/WÔ/WÑ)XÔ)Xðð ñô ð ñ ð ŒZÝÔ+ÝÔ+ð
ð 
ñ 
õ (Ñ)Ô)ð ­W¬]½9Ô;M×;UÒ;UÐV_Ñ;`Ô;`Ñ-aÔ-aÕdkÔdqØñeô eò .ð .õ "ð{ñô ð õ Œ}�YÔ/×7Ò7¸Ñ@Ô@ÑAÔAÅWÄ]ÐSXÑEYÔEYÒYÐYÝ!ðKñô ð õ Œ}�YÔ/×7Ò7¸	ÑBÔBÑCÔCÅwÄ}ÐU]ÑG^ÔG^Ò^Ð^à>Ð>Ð>Ð>Ð>Ð>Ð>Ð>Ð>àPÐPÐPÐPÐPÐPÐPÐPØŒz�^Ô<Ò<Ð<Ø )��Ø”�~Ô>Ò>Ð>Ø )��å Ð!4Ñ5Ô5Ð5Ø×#Ò# KÑ0Ô0Ð0Ñ0ØŒZÝÔ.ÝÔ.ÝÔ,ð
ð 
ñ 
õ
 -Ñ.Ô.ð Ý!ðBñô ð õ +¨8Ñ4Ô4ð uÝ!Ð"sÑtÔtÐtØGÐGÐGÐGÐGÐGÐGÐGà&(Ð#Ø!ˆNàŒz�^Ô?Ò?Ð?Ý0°Ñ9Ô9ð Ý%ðFñô ð ð ;Ð:Ð:Ð:Ð:Ð:à 1�Ø*5Ð'Ø!&��Ø”�~ÔAÒAÐAØ 1�Ø*5Ð'Ð'Ø”�~Ô?Ò?Ð?Ø /��å Ð!AÑBÔBÐBà6:Ô6GÐ# NÑ3Øð LØ:>Ô:KÐ'¨Ñ7Ø#×*Ò*å',¨Z¯^ª^Ð<MÈsÑ-SÔ-SÑ'TÔ'TÝ˜zŸ~š~¨c°3Ñ7Ô7Ñ8Ô8ðð ñô ð ð ×#Ò#Ð$;Ñ<Ô<Ð<Ð<ØŒZ�>Ô6Ò6Ð6Ý,Ñ.Ô.ð Ý!ðBñô ð ð +Ð*Ð*Ð*Ð*Ð*à—^’^ H¨dÑ3Ô3ˆFØÐ!Ý˜v™œ�à"Ÿš {°DÑ9Ô9ˆIØÐ$Ý  ™OœO�	à*.Ô*;ˆK˜Ñ'å# J§N¢N°=À%Ñ$HÔ$HÑIÔIØ Ø&ð#ð #Ðð (ˆMØ×#Ò# KÑ0Ô0Ð0Ø×#Ò#Ð$7Ñ8Ô8Ð8Ð8åÐ^ÐRVÔR\Ð^Ð^Ñ_Ô_Ð_ØÐ.Ð.Ð.s>   ÆF4 Æ4GÇ'H ÈH È9I ÉI2Ì%HT. Ô.U×CZ! Ú!Z;r`  c                 óÈ   — | j         €Ut          | j        j        |€| j        n|| j                             |¦  «        || j        j        ¬¦  «        | _         d| _        | j         S )zø
        Setup the scheduler. The optimizer of the trainer must have been set up either before this method is called or
        passed as an argument.

        Args:
            num_training_steps (int): The number of training steps to do.
        N)r`  Únum_warmup_stepsr+  Úscheduler_specific_kwargsT)ra  r0   rà   Úlr_scheduler_typer`  Úget_warmup_stepsÚlr_scheduler_kwargsr�  r/  s      r´   r.  zTrainer.create_scheduler3  sl   € ð ÔÐ$Ý -Ø”	Ô+Ø,5Ð,=˜$œ.˜.À9Ø!%¤×!;Ò!;Ð<NÑ!OÔ!OØ#5Ø*.¬)Ô*Gð!ñ !ô !ˆDÔð *.ˆDÔ&ØÔ Ð r¶   r  c                 ó  — 	 |j         }t          |t          ¦  «        rt          |j         j         ¦  «        S t          |j         ¦  «        S # t          t
          t          f$ r t          |¦  «        | j        j        z  cY S w xY w)zË
        Helper to get number of samples in a [`~torch.utils.data.DataLoader`] by accessing its dataset. When
        dataloader.dataset does not exist or has no length, estimates as best it can
        )	rÜ  r±   r>   rB  Ú	NameErrorr[  rq  rà   Úper_device_train_batch_size)rš  r  rÜ  s      r´   Únum_exampleszTrainer.num_examplesF  sŠ   € ð
	KØ Ô(ˆGå˜'Õ#7Ñ8Ô8ð 7Ý˜:Ô-Ô5Ñ6Ô6Ð6Ý�zÔ)Ñ*Ô*Ð*øÝ�>­9Ð5ð 	Kð 	Kð 	KÝ�z‘?”? T¤YÔ%JÑJÐJÐJÐJð	Køøøs   ‚4A ·A Á3BÂ BÚtrain_dlrr  c                 ó¼   — d}	 | D ]*}|d                               ¦   «         }|�||z  c S ||z  }Œ+n*# t          $ r t                               d¦  «         Y nw xY w|S )zq
        Helper to get number of tokens in a [`~torch.utils.data.DataLoader`] by enumerating dataloader.
        r   Ú	input_idsNz%Cannot get num_tokens from dataloader)rI  ÚKeyErrorr"  rG  )r6  rr  Útrain_tokensÚbatchÚtokenss        r´   Ú
num_tokenszTrainer.num_tokensT  sœ   € ð
 ˆð	DØ!ð 'ð '�Ø˜{Ô+×1Ò1Ñ3Ô3�ØÐ(Ø! IÑ-Ð-Ð-Ð-Ø Ñ&��ð	'øõ
 ð 	Dð 	Dð 	DÝ�NŠNÐBÑCÔCÐCÐCÐCð	DøøøàÐs   „&2 «2 ²$AÁAÚtrialzoptuna.Trialc                 ó   — || _         | j        �|€dS | j        t          j        k    r|                      |¦  «        }n~| j        t          j        k    r|}|                     dd¦  «         nP| j        t          j        k    r$d„ |j         	                    ¦   «         D ¦   «         }n| j        t          j
        k    r|}| 	                    ¦   «         D ]\  }}t          | j        |¦  «        st                               d|› d�¦  «         Œ9t          | j        |d¦  «        }|� t!          |¦  «        |¦  «        }t#          | j        ||¦  «         Œ€| j        t          j        k    r"t                               d|j        › �¦  «         | j        t          j        k    r"t                               d|j        › �¦  «         | j        t          j
        k    rt                               d|› �¦  «         | j        rÃ| j        j        €t-          d	¦  «        ‚| j                             ¦   «          d
dlm} d
dlm}  || j        j        ¦  «        | j        _        | j        j                             | j        ¦  «          || j        j        ¬¦  «        | j        _        tA          ¦   «          !                    ¦   «          |  "                    ¦   «          dS )zHP search setup codeNÚwandbc                 ób   — i | ],\  }}|t          |t          ¦  «        rt          |¦  «        n|“Œ-S r½   )r±   r%  rÏ  )rö   r	  Úvs      r´   rK  z,Trainer._hp_search_setup.<locals>.<dictcomp>p  s7   € ÐcÐcÐcÁÀÀA�a¥:¨aµÑ#5Ô#5Ð<�˜Q™œ˜¸1ÐcÐcÐcr¶   zTrying to set zY in the hyperparameter search but there is no corresponding field in `TrainingArguments`.zTrial: zSigOpt Assignments: zW&B Sweep parameters: z7For sweeps with deepspeed, `args.deepspeed` must be setr   )ÚDeepSpeedPlugin©ÚHfTrainerDeepSpeedConfig)Úhf_ds_config)#Ú_trialrˆ  rT   ÚOPTUNAÚhp_spaceÚRAYrQ  ÚSIGOPTÚassignmentsÚitemsÚWANDBrF  rà   r"  rG  r>  rË   Úsetattrr#  r  rN  r.  r%  rZ  Úfree_memoryÚaccelerate.utilsrC  Ú#transformers.integrations.deepspeedrE  Úhf_deepspeed_configÚtrainer_config_processÚdeepspeed_pluginr�   Ú_reset_stater0  )rš  r>  r  r#  r$  Úold_attrrC  rE  s           r´   Ú_hp_search_setupzTrainer._hp_search_setupd  sÈ  € àˆŒàÔ!Ð)¨U¨]ØˆFØÔ!¥_Ô%;Ò;Ð;Ø—]’] 5Ñ)Ô)ˆFˆFØÔ#¥Ô':Ò:Ð:ØˆFØ�JŠJ�w Ñ%Ô%Ð%Ð%ØÔ#¥Ô'=Ò=Ð=ØcÐcÈÔIZ×I`ÒI`ÑIbÔIbÐcÑcÔcˆFˆFØÔ#¥Ô'<Ò<Ð<ØˆFà Ÿ,š,™.œ.ð 	+ð 	+‰JˆC�Ý˜4œ9 cÑ*Ô*ð Ý—’ð, Sð ,ð ,ð ,ñô ð ð Ý˜tœy¨#¨tÑ4Ô4ˆHàÐ#Ø&�˜X™œ uÑ-Ô-�å�D”I˜s EÑ*Ô*Ð*Ð*ØÔ!¥_Ô%;Ò;Ð;Ý�KŠKÐ0 %¤,Ð0Ð0Ñ1Ô1Ð1ØÔ!¥_Ô%;Ò;Ð;Ý�KŠKÐB¨uÔ/@ÐBÐBÑCÔCÐCØÔ!¥_Ô%:Ò:Ð:Ý�KŠKÐ8°Ð8Ð8Ñ9Ô9Ð9ØÔ$ð 	.ØŒyÔ"Ð*Ý Ð!ZÑ[Ô[Ð[àÔ×(Ò(Ñ*Ô*Ð*ð 9Ð8Ð8Ð8Ð8Ð8àTÐTÐTÐTÐTÐTà,DÐ,DÀTÄYÔEXÑ,YÔ,YˆDŒIÔ)ØŒIÔ)×@Ò@ÀÄÑKÔKÐKØ)8¨ÀdÄiÔFcÐ)dÑ)dÔ)dˆDŒIÔ&õ ÑÔ×+Ò+Ñ-Ô-Ð-à×/Ò/Ñ1Ô1Ð1Ð1Ð1r¶   r†  Úmetricsc                 ó:  — | j         �|€d S |                     ¦   «         }|                      |¦  «        | _        | j         t          j        k    rœdd l}t          |d¦  «        r‚|j         	                    ¦   «         sk| 
                    | j        |¦  «         |                     ¦   «         r>| j                             | j        | j        | j        ¦  «          |j        ¦   «         ‚d S d S d S | j         t          j        k    r›dd l}t)          j        ¦   «         5 }d }| j        j        r5|                      |¬¦  «         |j        j                             |¦  «        }| j        |d<   |j         
                    ||¬¦  «         d d d ¦  «         d S # 1 swxY w Y   d S d S )Nr   Ústudy)Úcheckpoint_dirÚ	objective)Ú
checkpoint)rˆ  ÚcopyÚcompute_objectiver]  rT   rH  ÚoptunarF  r[  Ú_is_multi_objectiveÚreportÚshould_prunere  Úon_train_endrà   r}  r†  ÚTrialPrunedrJ  Ú	ray.trainÚtempfileÚTemporaryDirectoryrm  Ú_tune_save_checkpointÚtrainÚ
CheckpointÚfrom_directory)rš  r>  r†  rY  ra  ÚrayÚtemp_checkpoint_dirr^  s           r´   Ú_report_to_hp_searchzTrainer._report_to_hp_searchœ  s  € ØÔ!Ð)¨U¨]ØˆFØ—,’,‘.”.ˆØ×/Ò/°Ñ8Ô8ˆŒØÔ!¥_Ô%;Ò;Ð;ØˆMˆMˆMå�u˜gÑ&Ô&ð /¨u¬{×/NÒ/NÑ/PÔ/Pð /Ø—’˜Tœ^¨TÑ2Ô2Ð2Ø×%Ò%Ñ'Ô'ð /ØÔ)×6Ò6°t´yÀ$Ä*ÈdÌlÑ[Ô[Ð[Ø,˜&Ô,Ñ.Ô.Ð.ð	/ð /ð /ð /ð/ð /ð Ô#¥Ô':Ò:Ð:ØÐÐÐåÔ,Ñ.Ô.ð AÐ2EØ!�
Ø”<Ô+ð ZØ×.Ò.Ð>QÐ.ÑRÔRÐRØ!$¤Ô!5×!DÒ!DÐEXÑ!YÔ!Y�JØ'+¤~�˜Ñ$Ø”	× Ò  °ZÐ Ñ@Ô@Ð@ðAð Að Añ Aô Að Að Að Að Að Að Að Aøøøð Að Að Að Að Að Að ;Ð:s   ÄA*FÆFÆFr\  c                 ó¼  — t           j                             |t          › d| j        j        › �¦  «        }|                      |d¬¦  «         | j        j        �r| j	                             ¦   «         | j        j
        d<   | j                             t           j                             |t          ¦  «        ¦  «         t          j        | j                             ¦   «         t           j                             |t"          ¦  «        ¦  «         t          j        | j                             ¦   «         t           j                             |t&          ¦  «        ¦  «         d S d S )Nr}  T©Ú_internal_callr:   )rn  Úpathrç  rP   r}  Úglobal_stepÚ
save_modelrà   rm  r†  r  Úsave_to_jsonÚTRAINER_STATE_NAMErÀ   Úsaver`  Ú
state_dictÚOPTIMIZER_NAMEra  ÚSCHEDULER_NAME)rš  r\  rí   s      r´   rj  zTrainer._tune_save_checkpoint´  s  € Ý”W—\’\ .Õ5JÐ2eÐ2eÈTÌZÔMcÐ2eÐ2eÑfÔfˆ
Ø�Š˜
°4ˆÑ8Ô8Ð8ØŒ9Ô ñ 	aà>B¼l×>PÒ>PÑ>RÔ>RˆDŒJÔ)Ð*:Ñ;ØŒJ×#Ò#¥B¤G§L¢L°Õ=OÑ$PÔ$PÑQÔQÐQÝŒJ�t”~×0Ò0Ñ2Ô2µB´G·L²LÀÍ^Ñ4\Ô4\Ñ]Ô]Ð]ÝŒJ�tÔ(×3Ò3Ñ5Ô5µr´w·|²|ÀJÕP^Ñ7_Ô7_Ñ`Ô`Ð`Ð`Ð`ð	að 	ar¶   c                 óÜ   — t          | j        ¦  «        }|dk    r|                      ¦   «         }n+|dk    r|                      |¦  «        }nt          d¦  «        ‚|€t          d¦  «        ‚|S )Nr   r   z'model_init should have 0 or 1 argument.z"model_init should not return None.)rc   rä   r8  )rš  r>  Úmodel_init_argcountr²   s       r´   r7  zTrainer.call_model_init¾  st   € Ý1°$´/ÑBÔBÐØ !Ò#Ð#Ø—O’OÑ%Ô%ˆEˆEØ  AÒ%Ð%Ø—O’O EÑ*Ô*ˆEˆEåÐHÑIÔIÐIàˆ=ÝÐCÑDÔDÐDàˆr¶   c                 óR  ‡— |�s"|€t                                d¦  «         |S t          t          |¦  «        ¦  «        Š|                      ‰¦  «        Š	 t          j        |¦  «        }|                     ¦   «          |j                             dd ¦  «        }|r||_	        t          d¬¦  «        }| j                             |¬¦  «        5  t          j        ¦   «         5  t          ‰t           ¦  «        r#t          j                             |‰d¬¦  «        }n.t          j                             |ˆfd„‰D ¦   «         d¬¦  «        }d d d ¦  «         n# 1 swxY w Y   d d d ¦  «         n# 1 swxY w Y   t          j                             |¦  «        }t          j        ¦   «         5   |d
i ‰¤Ž  |d
i ‰¤Ž d d d ¦  «         n# 1 swxY w Y   |}d| _        nN# t*          t,          t.          t0          t2          f$ r(}t                                d|› d	�¦  «         Y d }~nd }~ww xY w|S )NzAfailed to use PyTorch jit mode due to current dataloader is none.Ú_original_forwardF)Úcache_enabled)Úautocast_handler)Úexample_kwarg_inputsÚstrictc                 ó"   •— i | ]}|‰|         “ŒS r½   r½   )rö   r#  Úexample_batchs     €r´   rK  z0Trainer.torch_jit_model_eval.<locals>.<dictcomp>á  s    ø€ Ð1cÐ1cÐ1cÈc°#°}ÀSÔ7IÐ1cÐ1cÐ1cr¶   z'failed to use PyTorch jit mode due to: rÎ  r½   )r"  rG  ÚnextÚiterÚ_prepare_inputsr_  r#  Ú__dict__rQ  r]  rž   rZ  ÚautocastrÀ   Úno_gradr±   rå  ÚjitÚtraceÚfreezery  r8  rq  r%  r3  Ú
IndexError)	rš  r²   r  ÚtrainingÚ	jit_modelÚoriginal_forwardr‚  Úer†  s	           @r´   Útorch_jit_model_evalzTrainer.torch_jit_model_evalÌ  s  ø€ Øñ 	OØÐ!Ý—’ÐbÑcÔcÐcØ�Ý ¥ jÑ!1Ô!1Ñ2Ô2ˆMØ ×0Ò0°Ñ?Ô?ˆMðOÝ œI eÑ,Ô,�	Ø—’Ñ Ô Ð Ø#,Ô#5×#9Ò#9Ð:MÈtÑ#TÔ#TÐ à#ð 9Ø(8�IÔ%Ý#1ÀÐ#FÑ#FÔ#FÐ ØÔ%×.Ò.Ð@PÐ.ÑQÔQð ð ÕSXÔS`ÑSbÔSbð ð Ý! -µÑ6Ô6ð Ý$)¤I§O¢O°IÐTaÐjo OÑ$pÔ$p˜	˜	å$)¤I§O¢OØ%Ø1cÐ1cÐ1cÐ1cÐUbÐ1cÑ1cÔ1cØ#(ð %4ñ %ô %˜	ð	ð ð ñ ô ð ð ð ð ð ð øøøð ð ð ð ð ð ð ñ ô ð ð ð ð ð ð øøøð ð ð ð õ "œI×,Ò,¨YÑ7Ô7�	Ý”]‘_”_ð /ð /Ø�IÐ.Ð. Ð.Ð.Ð.Ø�IÐ.Ð. Ð.Ð.Ð.ð/ð /ð /ñ /ô /ð /ð /ð /ð /ð /ð /øøøð /ð /ð /ð /ð "�Ø#(�Ô Ð øÝ ¥)­Z½ÅJÐOð Oð Oð OÝ—’ÐMÈÐMÐMÐMÑNÔNÐNÐNÐNÐNÐNÐNøøøøðOøøøð ˆs†   ÁA7G ÃE*Ã A'EÅE*ÅE	ÅE*ÅE	ÅE*ÅG Å*E.Å.G Å1E.Å25G Æ'GÆ8G ÇGÇG ÇGÇG Ç#H$Ç<HÈH$c           	      ót  — ddddœ}d}d}|                      ¦   «         D ]B\  }}t          ||d ¦  «        }t          ||d ¦  «        }	|�|	�||	k    r|d|› d|› d	|	› d
�z  }d}ŒC|j        }
|j        t	          d|j        ¦  «        z  }|
|k    r|d|
› d	|› d
�z  }d}|rt                               |¦  «         d S d S )NÚlogging_stepsÚ
eval_stepsÚ
save_steps)r—  r˜  r™  FztWarning: The following arguments do not match the ones in the `trainer_state.json` within the checkpoint directory: z
	rL  z (from args) != z (from trainer_state.json)Tr   z
	per_device_train_batch_size: )rM  r>  r4  r‹  ÚmaxÚn_gpur"  Úwarning_once)rš  Útraining_argsÚtrainer_stateÚattributes_mapÚhas_warningÚwarning_strÚarg_attrÚ
state_attrÚ	arg_valueÚstate_valueÚtrain_bs_argsÚtrain_bs_states               r´   Ú#compare_trainer_and_checkpoint_argsz+Trainer.compare_trainer_and_checkpoint_argsï  s>  € à,Ø&Ø&ð
ð 
ˆð ˆð MˆØ$2×$8Ò$8Ñ$:Ô$:ð 	#ð 	#Ñ ˆH�jÝ ¨x¸Ñ>Ô>ˆIÝ! -°¸TÑBÔBˆKàÐ$¨Ð)@ÀYÐR]ÒE]ÐE]ØÐt hÐtÐt°)ÐtÐtÈ[ÐtÐtÐtÑt�Ø"�øð &ÔAˆØ&Ô7½3¸qÀ-ÔBUÑ;VÔ;VÑVˆà˜NÒ*Ð*Øð  I¸}ð  Ið  IÐ^lð  Ið  Ið  Iñ  IˆKØˆKàð 	-Ý×Ò Ñ,Ô,Ð,Ð,Ð,ð	-ð 	-r¶   c                 ó6  ‡ ‡‡‡— t          ¦   «         rKt          ‰ j        t          j        j        ¦  «        r‰ j        S t          j        |‰ j        j        ¬¦  «        S ‰ j         	                    |d¬¦  «        |ur|S ‰ j
        r7|r5ddlm} |                     |‰ j        ‰ j        j        ¬¦  «        \  }‰ _        ‰ j        j        dk    r%t#          |dd¦  «        st%          j        |¦  «        }‰ j        j        rSt+          j        ¦   «         }‰                      |||¦  «        }t/          t+          j        ¦   «         |z
  d	¦  «        ‰ _        |s|S ‰ j        �r³	 dd
lmŠ ddlmŠ ddlm}m} ‰ j         rddl!m"Š n# tF          $ r tG          d¦  «        ‚w xY wd }d }	t#          |dd ¦  «        }
‰ j        j$         %                    d|
¦  «        }‰ j        j$        d         dk    r'tM          j'        |‰ j        j$        d         ¬¦  «        }na|�_tQ          ¦   «         }|D ]8}tS          ||¦  «        }|€tU          d¦  «        ‚| +                    |¦  «         Œ9tM          j'        ||¬¦  «        }‰ j        j,        }‰ j        j$        d         r:|j-        j.        r&t^           0                    d¦  «         d|j-        _.        ˆˆˆˆ fd„}	‰ j         rd„ } ‰||||	¬¦  «        x‰ _        }n ‰|f||	dœ|¤Žx‰ _        }di fd„}|tb          _2        �n
tg          ¦   «         rBt$          j4         5                    |tm          to          j8        d¦  «        ¦  «        g¬¦  «        }nº‰ j        j9        tt          j;        k    r ty          ¦   «         r|S i }‰ j        j=        �‰ j        j=        |d<   n&t          |t|          ¦  «        r|j?         |d<   nd|d<   ‰ j        j@        �‰ j        j@        |d <   ‰ j        jA        �‰ j        jA        |d!<   t…          d"i |¤Ž‰ j        _C        |S )#N)Úbackward_passes_per_stepF©Úkeep_torch_compiler   ©Úamp)Ú	opt_levelr   Úis_loaded_in_8bité   )ÚXlaFullyShardedDataParallel)Úcheckpoint_module)Úsize_based_auto_wrap_policyÚtransformer_auto_wrap_policy)ÚSpmdFullyShardedDataParallelzJMissing XLA FSDP related module; please make sure to use torch-xla >= 2.0.Ú_no_split_modulesÚtransformer_layer_cls_to_wrapÚmin_num_params)r¹  z@Could not find the transformer layer class to wrap in the model.)Útransformer_layer_clsÚxla_fsdp_grad_ckptzX`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`.c                 óD   •— ‰j         s‰n‰} | ‰| ¦  «        g|¢R i |¤ŽS r  )r�  )Úmrà   ÚkwargsÚ
target_clsÚFSDPÚFSDPv2r³  rš  s       €€€€r´   Úauto_wrapper_callablez2Trainer._wrap_model.<locals>.auto_wrapper_callable]  s@   ø€ Ø-1Ô-HÐ!T  Èf�JØ%˜:Ð&7Ð&7¸Ñ&:Ô&:ÐL¸TÐLÐLÐLÀVÐLÐLÐLr¶   c                 ó  — ddl m} d }t          | t          j        ¦  «        r| }n5t          | t
          ¦  «        r	| d         }nt          | |¦  «        r| j        }|€t          d¦  «        ‚t          j	        ||d¦  «         d S )Nr   )ÚCausalLMOutputWithPastr   zASomething went wrong, the output of the model shouldn't be `None`)r  NN)
Úmodeling_outputsrÄ  r±   rÀ   ÚTensorrç  Úlogitsr%  r”  Úmark_sharding)ÚoutputÚmeshrÄ  Úreal_outputs       r´   Úshard_outputz)Trainer._wrap_model.<locals>.shard_outputd  sœ   € ØHÐHÐHÐHÐHÐHà"&�KÝ! &­%¬,Ñ7Ô7ð 4Ø&,˜˜Ý# F­EÑ2Ô2ð 4Ø&,¨Q¤i˜˜Ý# FÐ,BÑCÔCð 4Ø&,¤m˜à"Ð*Ý(Ð)lÑmÔmÐmÝÔ$ [°$Ð8LÑMÔMÐMÐMÐMr¶   )rÌ  Úauto_wrap_policyrÂ  )rÍ  rÂ  c                 óJ   —  | j         di |¤Ž}|rt          j        ¦   «          |S )Nr½   )r†  ÚxmÚ	mark_step)r`  ÚbarrierÚoptimizer_argsÚlosss       r´   Úpatched_optimizer_stepz3Trainer._wrap_model.<locals>.patched_optimizer_stepƒ  s1   € Ø%�y”~Ð7Ð7¨Ð7Ð7�Øð #Ý”L‘N”N�NØ�r¶   ÚSMDATAPARALLEL_LOCAL_RANK)Ú
device_idsÚfind_unused_parametersTÚbucket_cap_mbÚbroadcast_buffersr½   )Dr„   r±   rY  r|  r²   ÚDistributedModelrà   rú  rZ  r,   rx  r  r®  Ú
initializer`  Úfp16_opt_levelr›  r>  r   ÚDataParallelÚjit_mode_evalÚtimer•  ÚroundÚjit_compilation_timerM  Útorch_xla.distributed.fsdpr²  r³  Útorch_xla.distributed.fsdp.wrapr´  rµ  r�  Ú7torch_xla.experimental.spmd_fully_sharded_data_parallelr¶  rH  rL  r�  Ú	functoolsr   r@  rG   Ú	ExceptionÚaddÚxla_fsdp_configr„  Ú	use_cacher"  rœ  rÏ  Úoptimizer_steprƒ   ÚparallelÚDistributedDataParallelrÏ  rn  ÚgetenvrO  rh   rP  rŠ   Úddp_find_unused_parametersr*   Úis_gradient_checkpointingÚddp_bucket_cap_mbÚddp_broadcast_buffersrŸ   Úddp_handler)rš  r²   r‘  r  r®  Ú
start_timer´  rµ  rÍ  rÂ  Ú%default_transformer_cls_names_to_wrapÚ"fsdp_transformer_layer_cls_to_wrapÚtransformer_cls_to_wrapÚlayer_classÚtransformer_clsÚfsdp_kwargsrÌ  rÔ  r¾  rÀ  rÁ  r³  s   `                  @@@r´   Ú_wrap_modelzTrainer._wrap_model  sc  øøøø€ Ý"Ñ$Ô$ð 	oå˜$Ô,­c¬iÔ.HÑIÔIð *ØÔ)Ð)ÝÔ'¨ÈÌ	ÔHmÐnÑnÔnÐnð Ô×(Ò(¨À5Ð(ÑIÔIÐQVÐVÐVØˆLð Œ=ð 	n˜Xð 	nØ Ð Ð Ð Ð Ð à$'§N¢N°5¸$¼.ÐTXÔT]ÔTl NÑ$mÔ$mÑ!ˆE�4”>ð Œ9Œ?˜QÒÐ¥w¨uÐ6IÈ5Ñ'QÔ'QÐÝ”O EÑ*Ô*ˆEàŒ9Ô"ð 	KÝœ™œˆJØ×-Ò-¨e°ZÀÑJÔJˆEÝ(-­d¬i©k¬k¸JÑ.FÈÑ(JÔ(JˆDÔ%ð ð 	ØˆLð Ô#ñ u	SðpØZÐZÐZÐZÐZÐZØHÐHÐHÐHÐHÐHðð ð ð ð ð ð ð ð
 Ô.ð ðð ð ð ð ð øøõ ð pð pð pÝ!Ð"nÑoÔoÐoðpøøøà#ÐØ$(Ð!Ý4;¸EÐCVÐX\Ñ4]Ô4]Ð1Ø15´Ô1F×1JÒ1JØ/Ð1Vñ2ô 2Ð.ð ŒyÔ$Ð%5Ô6¸Ò:Ð:Ý#,Ô#4Ø/ÀÄ	Ô@UÐVfÔ@gð$ñ $ô $Ð Ð ð 4Ð?Ý*-©%¬%Ð'Ø#Eð Eð E�KÝ&@ÀÈÑ&TÔ&T�OØ&Ð.Ý'Ð(jÑkÔkÐkà/×3Ò3°OÑDÔDÐDÐDå#,Ô#4Ø0à*Að$ñ $ô $Ð ð
 œ)Ô3ˆKØŒyÔ$Ð%9Ô:ð 
MØ”<Ô)ð 3Ý×'Ò'Ørñô ð ð .3�E”LÔ*ðMð Mð Mð Mð Mð Mð Mð Mð
 Ô*ð ðNð Nð Nð &, VØØ!-Ø%5Ø*?ð	&ñ &ô &ð �”
˜U˜Uð &* TØð&à%5Ø*?ð&ð &ð "ð	&ð &ð �”
˜Uð ;@ÐPRð ð ð ð ð !7�BÔÑÝ$Ñ&Ô&ð 	SÝ”K×7Ò7Ø¥3¥r¤yÐ1LÑ'MÔ'MÑ#NÔ#NÐ"Oð 8ñ ô ˆEˆEð ŒYÔ$­Ô(@Ò@Ð@Ý,Ñ.Ô.ð Ø�ØˆFØŒyÔ3Ð?Ø37´9Ô3W�Ð/Ñ0Ð0Ý˜E¥?Ñ3Ô3ð 8ð 8=Ô7VÐ3V�Ð/Ñ0Ð0à37�Ð/Ñ0àŒyÔ*Ð6Ø*.¬)Ô*E��Ñ'àŒyÔ.Ð:Ø.2¬iÔ.M�Ð*Ñ+å+HÐ+RÐ+RÈ6Ð+RÐ+RˆDÔÔ(àˆs   Å!E? Å?FÚresume_from_checkpointÚignore_keys_for_evalr¾  c           	      óþ  — |du rd}| j                              ¦   «          | j        }d| _        t	          | j        t          t          f¦  «        r)t          | j	        d¦  «        r|  
                    ¦   «          | j        �|                      | j	        ¦  «        | _	        |j        s|j        r5|j        s.| j        s'| j        € |                      | j	        |j        ¦  «         d|v r/|                     d¦  «        }t+          j        dt.          ¦  «         t1          |¦  «        dk    rEt3          dd	                     t7          |                     ¦   «         ¦  «        ¦  «        › d
�¦  «        ‚|                      |¦  «         | j        j        | _        d}| j        �i| j        j         rtC          | j        j"        ¦  «        ntG          | j        j"        ¦  «         |  $                    |¦  «        | _	        d}d\  | _%        | _&        t	          |tN          ¦  «        r0|r.tQ          |j)        ¦  «        }|€tU          d|j)        › d�¦  «        ‚|�{tW          ¦   «         s#| j,        s| j-        s|  .                    |¦  «         t_          j0        tb          j2                             |tf          ¦  «        ¦  «        }|j        �|j        | _        |r3| j4        r |                      | j	        |j        ¦  «         | j	        | _5        tm          | j7        | j        |j8        ¦  «        }|j9        rN	 tu          j;        ¦   «           |||||¬¦  «        tu          j<        ¦   «          S # tu          j<        ¦   «          w xY w |||||¬¦  «        S )aó  
        Main training entry point.

        Args:
            resume_from_checkpoint (`str` or `bool`, *optional*):
                If a `str`, local path to a saved checkpoint as saved by a previous instance of [`Trainer`]. If a
                `bool` and equals `True`, load the last checkpoint in *args.output_dir* as saved by a previous instance
                of [`Trainer`]. If present, training will resume from the model/optimizer/scheduler states loaded here.
            trial (`optuna.Trial` or `dict[str, Any]`, *optional*):
                The trial run or the hyperparameter dictionary for hyperparameter search.
            ignore_keys_for_eval (`list[str]`, *optional*)
                A list of keys in the output of your model (if it is a dictionary) that should be ignored when
                gathering predictions for evaluation during the training.
            kwargs (`dict[str, Any]`, *optional*):
                Additional keyword arguments used to hide deprecated arguments
        FNTr„  Ú
model_pathzi`model_path` is deprecated and will be removed in a future version. Use `resume_from_checkpoint` instead.r   z*train() got unexpected keyword arguments: rà  rÎ  rß   z/No valid checkpoint found in output directory (ú))rà   rû  r>  rü  )=r2  r3  rà   r/  r±   rÜ   r3   r1   rF  r²   rÖ  r_  r¸  rR  rS  rT  r?  rä   rW  r÷   rQ  r9  r:  r;  rB  rq  rç  r¹   rÚ  rX  r‹  rŒ  r+  r^   r,  re   r7  r`  ra  ræ  r`   rí   r%  r„   rN  rU  Ú_load_from_checkpointr;   Úload_from_jsonrn  rt  rx  rQ  rY  r_   Ú_inner_training_loopÚauto_find_batch_sizerk  Úhf_hub_utilsÚdisable_progress_barsÚenable_progress_bars)	rš  rû  r>  rü  r¾  rà   Úmodel_reloadedr}  Úinner_training_loops	            r´   rk  zTrainer.train¥  sÄ  € ð. " UÐ*Ð*Ø%)Ð"ð 	Ô×"Ò"Ñ$Ô$Ð$àŒyˆàˆÔõ �dÔ+Õ.EÅ~Ð-VÑWÔWð 	)Õ\cØŒJ˜ñ]
ô ]
ð 	)ð ×&Ò&Ñ(Ô(Ð(ð Ô#Ð/Ø×/Ò/°´
Ñ;Ô;ˆDŒJð
 Ô ð	@Ø$(Ô$7ð	@à”Mð	@ð Ô*ð	@ð ”Ð'à×&Ò& t¤z°4´;Ñ?Ô?Ð?à˜6Ð!Ð!Ø%+§Z¢Z°Ñ%=Ô%=Ð"ÝŒMðåñô ð õ
 ˆv‰;Œ;˜Š?ˆ?ÝÐjÈÏÊÕSWÐX^×XcÒXcÑXeÔXeÑSfÔSfÑIgÔIgÐjÐjÐjÑkÔkÐkà×Ò˜eÑ$Ô$Ð$Ø!%¤Ô!;ˆÔð ˆØŒ?Ð&à7;´yÔ7QÐoÕ# D¤I¤NÑ3Ô3Ð3ÕW_Ð`dÔ`iÔ`nÑWoÔWoøØ×-Ò-¨eÑ4Ô4ˆDŒJØ!ˆNà0:Ñ-ˆDŒN˜DÔ-õ Ð,­dÑ3Ô3ð 	gÐ8Nð 	gÝ%8¸¼Ñ%IÔ%IÐ"Ø%Ð-Ý Ð!eÐSWÔSbÐ!eÐ!eÐ!eÑfÔfÐfà!Ð-Ý*Ñ,Ô,ð C°TÔ5Nð CÐW[ÔWkð CØ×*Ò*Ð+AÑBÔBÐBå Ô/µ´·²Ð=SÕUgÑ0hÔ0hÑiÔiˆEØÔ%Ð1Ø).Ô)?�Ô&ð ð 	,ØÔ)ð DØ×*Ò*¨4¬:°t´{ÑCÔCÐCØ!%¤ˆDÔå8ØÔ% tÔ'=¸tÔ?Xñ
ô 
Ðð Ôð 	ð
4åÔ2Ñ4Ô4Ð4Ø*Ð*ØØ+AØØ)=ð	ñ ô õ Ô1Ñ3Ô3Ð3Ð3ø•Ô1Ñ3Ô3Ð3Ð3øøøà&Ð&ØØ'=ØØ%9ð	ñ ô ð s   Ì#!M ÍM-c                 óÖ   — t          | j        dd¦  «        x}�|S | j        rGt          | j        dd¦  «        x}r/|j                             di ¦  «                             dd¦  «        S dS )zGGet the tensor parallel size from either the model or DeepSpeed config.Ú_tp_sizeNrS  Útensor_parallelÚautotp_sizer   )r>  r²   rN  rà   r„  r�  )rš  Úmodel_tpÚdeepspeed_configs      r´   Úget_tp_sizezTrainer.get_tp_size	  s€   € õ   ¤
¨J¸Ñ=Ô=Ð=ˆHÐJØˆOð Ô$ð 	\½gÀdÄiÐQfÐhlÑ>mÔ>mÐ*mÐ*:ð 	\Ø#Ô*×.Ò.Ð/@À"ÑEÔE×IÒIÈ-ÐYZÑ[Ô[Ð[ð ˆqr¶   c                 ó^   — |j         |                      ¦   «         z  }| j        |j        z  |z  S )z˜Calculates total batch size (micro_batch * grad_accum * dp_world_size).

        Note: Only considers DP and TP (dp_world_size = world_size // tp_size).)r  r  rŒ  rú  )rš  rà   Údp_world_sizes      r´   Úget_total_train_batch_sizez"Trainer.get_total_train_batch_size*	  s2   € ð œ¨4×+;Ò+;Ñ+=Ô+=Ñ=ˆØÔ%¨Ô(HÑHÈ=ÑXÐXr¶   c                 ó0+  — | j                              ¦   «          || _        | j        j        r¯| j        j        | j        k    r‰ddlm}  || j	        ¦  «        \  | _	        | j
        | _	        | j        rY| j        j        }| j        t          d| j        j        ¦  «        z  | j        _        |                      d¦  «         || j        _        | j        | j        _        t                                d| j        › �¦  «         |                      ¦   «         }| j        rt)          |¦  «        }|                      |¦  «        }	|                      |||	¦  «        \  }
}}}}}}d }| j        j        r3|                      ||rd n|¦  «        }|�|r||j        z  }n
||j        z  }t6          j        | j        j        v r3| j        j        dk    rt;          d¦  «        ‚t=          | j
        ¦  «         t?          ¦   «         p| j         p| j!        }| j!        o#tE          | j         j        j#        dd¦  «        dk    }|rd	}| j$        rd | _%        d	| _$        | j        rtM          | |¬
¦  «        \  | _'        | _%        |s|  (                    |¬
¦  «         tS          d„ | j*        j+        | j,        gz   D ¦   «         ¬¦  «        | _        |d u| j        _-        | j        | j        _        | j         .                    ||¦  «         |j/        r | j
         0                    |j1        ¬¦  «         |  2                    | j	        ¦  «        }|| j
        u }|r"| j!        rtg          | j
        d¬¦  «        | _
        |r`|rH|  4                    ¦   «          | j         j5        dk    r$| j          6                    | j
        ¦  «        | _
        |  (                    |¬
¦  «         |ré| j
         7                    ¦   «          tq          | j%        d¦  «        r�| j9        r | j          6                    | j
        ¦  «        }n¸| j:        r%| j          6                    | j'        ¦  «        | _'        nŒ| j          6                    | j
        | j'        ¦  «        \  }| _'        n^| j          6                    | j
        | j'        | j%        ¦  «        \  }| _'        | _%        n$| j          6                    | j'        ¦  «        | _'        | j!        r|x| _
        | _	        || j
        ur|| _	        | j        r| j	        | _;        |�b| j        r+ty          | j	        |t{          | j
        ¦  «         ¬¦  «         n0t?          ¦   «         s| j!        r|  >                    || j	        ¦  «         |  ?                    |¦  «         |  @                    |¦  «         t            A                    d¦  «         t            A                    d|d›�¦  «         t            A                    d|
d›�¦  «         t            A                    d| j        j        d›�¦  «         | j        j        | j        k    r#t            A                    d| j        d›�¦  «         t            A                    d|	d›�¦  «         t            A                    d|j        › �¦  «         t            A                    d|d›�¦  «         t            A                    dt…          |d¬¦  «        d›�¦  «         d| j        _C        t‰          jD        ¦   «         }d}d}|��xtŠ          jF         G                    tŠ          jF         H                    |t’          ¦  «        ¦  «        �r5tS          jJ        tŠ          jF         H                    |t’          ¦  «        ¦  «        | _        |  K                    | j        | j        ¦  «         |  L                    ¦   «          t›          | j        jN        |z  ¦  «        }|jO        s| j        jN        |z  }||j        z  }nd}t            A                    d¦  «         t            A                    d|› �¦  «         t            A                    d| j        jN        › �¦  «         |jO        s!t            A                    d |› d!|› d"�¦  «         d#D ]&}t¡          | j*        |tE          | |¦  «        ¦  «         Œ'|| j*        _Q        | j         R                    | ||
|¦  «         t§          jT        d$|jU        ¬%¦  «        }d$| _V        | j        jN        | _W        | X                    ¦   «          d }d }| j*         Y                    || j        | j,        ¦  «        | _,        |jZ        r|  [                    ||d¬&¦  «         t¹          ||
¦  «        D �	]Ê}|}tq          |d'¦  «        r| ]                    |¦  «         |j^        dk    rd | __        |�tÁ          |¦  «        n|ja        |j        z  }| j*         b                    || j        | j,        ¦  «        | _,        d(} d	}!||k    rB|�@|dk    r|jO        stÇ          ||¦  «        }|dz
  } d}!n|dk    r|  d                    |¦  «         tË          |¦  «        }"||j        z  }#|#dk    r|j        }#d(}$||j        z  t›          |#|j        k     ¦  «        z   }%t¹          |%¦  «        D �]Ÿ}&|$dz  }$|$|%dz
  k    r|j        n|#}'|  f                    |"|'|jU        ¦  «        \  }(})tÁ          |(¦  «        | _g        tÑ          |(¦  «        D �]\  }*}+| dz  } | dz   |j        z  dk    p| dz   |k    },| j         ji         j                    |,¦  «         | j        jk        d)v�r�tE          | j
        d*d+¦  «        }-|-|+vrt            l                    d,¦  «         �nK| j        jk        d-k    r¥d.|+v r|+d.          m                    ¦   «         }.n | jn        �Jtq          | jn        d/¦  «        r5| jn        jo        �)|+|-         | jn        jo        k     m                    ¦   «         }.nOt            l                    d0¦  «         |+|-          p                    ¦   «         }.n|+|-          p                    ¦   «         }.t§          jT        |.| j        jU        t¦          jq        ¬1¦  «        }.| j        xjr        | j          s                    |.¦  «         m                    ¦   «          t                    ¦   «         z  c_r        |!r|  d                    |¦  «         d	}!| |j        z  dk    r+| j*         u                    || j        | j,        ¦  «        | _,        |*tÁ          |(¦  «        dz
  k    r:| j         jv        tî          jx        k    r tó          jz        | j         j{        |¬2¦  «        ntø          j}        }/ |/¦   «         5  |  ~                    ||+|)¦  «        }0d d d ¦  «         n# 1 swxY w Y   |j        rU�t          ¦   «         sFt§          j�        |0¦  «        st§          j‚        |0¦  «        r||d| j        jN        z   | jW        z
  z  z   }n4|jU        |0jU        k    rt;          d3|jU        › d4|0jU        › �¦  «        ‚||0z   }| xjƒ        �t	          |  …                    |+¦  «        ¦  «        z  c_ƒ        |,�r| j         ji         j                    d¦  «         |j†        ��`|j†        dk    �rTt?          ¦   «         r'|j‡        r | j'         ˆ                    |j†        ¦  «        }1n»| j9        rEdd5l‰mŠ}2 �t          jŒ         �                    |2 Ž                    | j'        ¦  «        |j†        ¦  «        }1notø          j}        }3| j:        rdd6l�m�}4 |4}3 |3¦   «         5  | j          �                    | ‘                    ¦   «         |j†        ¦  «        }1d d d ¦  «         n# 1 swxY w Y   �t%          ¦   «         rS| j         jv        tî          jx        k    r9| “                    ¦   «         }tq          |d7¦  «        r| t                    ¦   «         }n|1}| j*         ”                    || j        | j,        ¦  «        | _,        tø          j}        }/| j:        rdd6l�m�}4 |4}/ |/¦   «         5  | j'         •                    ¦   «          d d d ¦  «         n# 1 swxY w Y   | j*         –                    || j        | j,        ¦  «        | _,        |  —                    ¦   «         }| j         j˜        sC�t3          | j%        t¦          jš        j%        j›        ¦  «        s| j%         •                    ¦   «          | X                    ¦   «          | j        xjN        dz  c_N        || dz   |z  z   | j        _C        | j*         œ                    || j        | j,        ¦  «        | _,        |  �                    ||||||||¬8¦  «         n+| j*         ž                    || j        | j,        ¦  «        | _,        | j,        jŸ        s| j,        j         r%�t          ¦   «         r�tC          j¢        ¦   «           n�Œ| j,        jŸ        s| j,        j         r%�t          ¦   «         r�tC          j¢        ¦   «           n�Œ¡| dk     r7t            l                    d9| j        jN        › d:|› d;�¦  «         d| j,        _         | j*         £                    || j        | j,        ¦  «        | _,        |  �                    ||||||||¬8¦  «         t6          j¤        | j        j        v rQ�t          ¦   «         r(�tC          j¥        �tM          j§        ¦   «         ¦  «         nt            l                    d<¦  «         | j,        j         r n�	ŒÌ|j^        r!tq          | d=¦  «        r�tQ          | d=¦  «         t            A                    d>¦  «         |j©        r | j        jª        �|  «                    ¦   «          | xjV        | t                    ¦   «         z  c_V        t          | j        jN        d?¦  «        }5| jV        |5z  }6�tY          d@||| j        ja        |¬A¦  «        }7|  ­                    ¦   «          | j        j®        |7dB<   |6|7dC<   d	| _¯        | j°         ±                    |7¦  «         |  ²                    |7¦  «         |  ³                    |¦  «        }8|  ´                    d	|8¬D¦  «        }9| j        jµ        r€| j        jª        �t| j        j¶        dk    rd|9D ]a}:tŠ          jF         ·                    |:| j        jª        ¦  «        s5t            A                    dE|:› dF�¦  «         �tq          j¹        |:d¬G¦  «         Œb| j*         º                    || j        | j,        ¦  «        | _,        |  »                    ¦   «          | j¼        �|  ½                    | j
        ¦  «         �t}          | j        jN        |6|7¦  «        S )HNr   )Úrelease_memoryr   Tz)Currently training with a batch size of: zƒCurrently --debug underflow_overflow is not supported under DP. Please use DDP (torchrun or torch.distributed.launch (deprecated)).Úfsdp_versioné   F)r+  c                 ó<   — g | ]}t          |t          ¦  «        ¯|‘ŒS r½   r  r  s     r´   rø   z0Trainer._inner_training_loop.<locals>.<listcomp>ƒ	  r  r¶   )r  )Úgradient_checkpointing_kwargs)Ú	recursiveÚfp8r†  ©Úload_module_strictz***** Running training *****ú  Num examples = rr  z  Num Epochs = z(  Instantaneous batch size per device = zA  Training with DataParallel so batch size has been adjusted to: zE  Total train batch size (w. parallel, distributed & accumulation) = z   Gradient Accumulation steps = z  Total optimization steps = z#  Number of trainable parameters = )Útrainable_onlyzE  Continuing training from checkpoint, will skip to saved global_stepz!  Continuing training from epoch z'  Continuing training from global step z  Will skip the first z epochs then the first z batches in the first epoch.)r²   r`  ra  rB  ©r÷   )Úskip_schedulerÚ	set_epochéÿÿÿÿ)rï   FÚmain_input_namer8  zÏTried to track the number of tokens seen, however the current model is not configured properly to know what item is the input. To fix this, add a `main_input_name` attribute to the model class you are using.Únon_paddingÚattention_maskrÍ  z\Could not determine method to count non-padding tokens, falling back to counting all tokens.)r÷   rÊ   )r²   z0Calculated loss must be on the original device: z but device in use is r­  )Úimplicit_replicationÚitem)rá  zXThere seems not to be a single sample in your epoch_iterator, stopping training at step zI! This is expected if you're using an IterableDataset and set num_steps (z.) higher than the number of available samples.zƒYou enabled PyTorch/XLA debug metrics but you don't have a TPU configured. Check your training configuration if this is unexpected.Ú_pastzU

Training completed. Do not forget to share your model on huggingface.co/models =)

gü©ñÒMbP?rk  )Únum_samplesÚ	num_stepsr=  Ú
total_flosÚ
train_loss©Ú	use_mtimerí   úDeleting older checkpoint [ú] due to args.save_total_limit©Úignore_errors)¿rZ  rP  rŒ  rà   r  r}  r‹  rQ  r  rY  r²   rN  r4  rš  r›  Úpropagate_args_to_deepspeedr"  rY  r  r�  r(   r  Úset_initial_training_valuesÚinclude_tokens_per_secondr=  rs  rú  r   ÚUNDERFLOW_OVERFLOWr%  r   r„   rM  rU  r>  Úfsdp_pluginr�  ra  r%   r`  r0  r;   re  rç   r†  Úis_hyper_param_searchÚcompute_stepsÚgradient_checkpointingÚgradient_checkpointing_enabler  rú  r,   Ú_fsdp_qlora_plugin_updatesÚmixed_precisionr  rk  rF  rx  Úis_tp_enabledr.  r&   rµ   r   Ú_load_optimizer_and_schedulerÚ_load_scalerr#  rF   Úepochrß  rn  rt  Úisfilerç  rx  r  r¨  Ú_load_callback_staterÏ  ru  Úignore_data_skiprO  Útrain_dataloaderÚinit_training_referencesrÀ   r   r÷   Ú_total_loss_scalarÚ_globalstep_last_loggedr‡  Úon_train_beginÚeval_on_startÚ	_evaluater˜  r!  Ú
past_indexr(  rB  rr  Úon_epoch_beginrœ   Ú_load_rng_staterˆ  Úget_batch_samplesÚ#current_gradient_accumulation_stepsÚ	enumerateÚgradient_stateÚ_set_sync_gradientsÚinclude_num_input_tokens_seenrG  rW  rÜ   rÍ  rI  Úint64Únum_input_tokens_seenÚgatherr'  Úon_step_beginÚdistributed_typer    Ú	DEEPSPEEDrå  r   Úno_syncrÂ   rÃ   Útraining_stepÚlogging_nan_inf_filterr�   ÚisnanÚisinfr‡  r  Úfloating_point_opsÚmax_grad_normr  Úclip_master_gradsr  r®  r   rt  Úclip_grad_norm_Úmaster_paramsÚ&torch.distributed._tensor.experimentalr&  r¼   ry   Úget_global_grad_normÚon_pre_optimizer_stepr†  Úon_optimizer_steprÖ   Úoptimizer_step_was_skippedr±   rR  ÚReduceLROnPlateauÚon_step_endÚ_maybe_log_save_evaluateÚon_substep_endÚshould_epoch_stopÚshould_training_stoprÏ  rÐ  Úon_epoch_endÚTPU_METRICS_DEBUGÚmaster_printÚmetÚmetrics_reportÚdelattrr)  Úbest_model_checkpointÚ_load_best_modelrf   Ú
store_flosr+  r/  r2  rŽ  ÚlogÚ_get_output_dirÚ_sorted_checkpointsrm  Úsave_total_limitÚsamefileÚshutilÚrmtreere  Ú_finish_current_pushr_  r»  rZ   );rš  rü  rà   rû  r>  rü  r  Úoriginal_bsrE  Útotal_train_batch_sizers  Únum_update_steps_per_epochr5  Únum_train_samplesÚepoch_basedÚlen_dataloaderrr  Únum_train_tokensÚdelay_optimizer_creationÚis_fsdp2r²   Úuse_accelerator_prepareró  Úepochs_trainedÚsteps_trained_in_current_epochÚattrÚtr_lossÚ	grad_normrá  rA  Úepoch_dataloaderÚsteps_in_epochr†  Úrng_to_syncÚepoch_iteratorÚ	remainderÚupdate_stepÚtotal_updatesr|  Únum_batchesÚbatch_samplesÚnum_items_in_batchÚiÚinputsÚdo_sync_stepr#  Úinput_tokensÚcontextÚtr_loss_stepÚ
_grad_normr®  Úgrad_norm_contextr&  Úeffective_global_stepr,  rY  Úrun_dirÚcheckpoints_sortedr^  s;                                                              r´   r  zTrainer._inner_training_loop1	  s®  € ð 	Ô×$Ò$Ñ&Ô&Ð&Ø!+ˆÔØŒ9Ô)ð 	AØŒzÔ*¨dÔ.DÒDÐDØ;Ð;Ð;Ð;Ð;Ð;à(6¨°tÔ7IÑ(JÔ(JÑ%�Ô#Ø%)¤Z�Ô"ð Ô,ð Hà"&¤)Ô"G�KØ<@Ô<RÕVYÐZ[Ð]aÔ]fÔ]lÑVmÔVmÑ<m�D”IÔ9Ø×4Ò4°TÑ:Ô:Ð:Ø<G�D”IÔ9Ø*.Ô*@ˆDŒJÔ'Ý�ŠÐYÀÔAWÐYÐYÑZÔZÐZà×4Ò4Ñ6Ô6ÐØÔ&ð 	EÝ2Ð3CÑDÔDÐð "&×!@Ò!@ÀÑ!FÔ!FÐð ×,Ò,¨TÐ3CÐE[Ñ\Ô\ñ	
ØØ&ØØØØØð  ÐØŒ9Ô.ð 	EØ#ŸšÐ/?ÈÐAcÀÀÐZcÑdÔdÐàÐ)¨kÐ)Ø  DÔ$9Ñ9Ð Ð ð ! DÔ$DÑDÐ åÔ)¨T¬Y¬_Ð<Ð<ØŒyŒ Ò"Ð"õ !ðLñô ð õ
 ' t¤zÑ2Ô2Ð2å#:Ñ#<Ô#<Ð#pÀÔ@XÐ#pÐ\`Ô\pÐ ð Ô'Ðq­W°TÔ5EÔ5KÔ5WÐYgÐijÑ-kÔ-kÐopÒ-pˆØð 	-Ø',Ð$ð Ô%ð 	/Ø $ˆDÔØ).ˆDÔ&àÔ$ð 	cÝ0>¸tÐXaÐ0bÑ0bÔ0bÑ-ˆDŒN˜DÔ-à'ð 	NØ×/Ò/À9Ð/ÑMÔMÐMå!ð ð  Ø!Ô2Ô<ÀÄ¸~ÑMð ñ  ô  ð
ñ 
ô 
ˆŒ
ð
 ,1¸Ð+<ˆŒ
Ô(Ø&*Ô&<ˆŒ
Ô#ð 	Œ
× Ò   yÑ1Ô1Ð1ð Ô&ð 	wØŒJ×4Ò4ÐSWÔSuÐ4ÑvÔvÐvà× Ò  Ô!3Ñ4Ô4ˆð
 #(¨4¬:Ð"5Ðà"ð 	B tÔ';ð 	Bõ & d¤j¸DÐAÑAÔAˆDŒJà#ð 	NØ&ð Fà×/Ò/Ñ1Ô1Ð1ØÔ#Ô3°uÒ<Ð<Ø!%Ô!1×!9Ò!9¸$¼*Ñ!EÔ!E�D”JØ×/Ò/À9Ð/ÑMÔMÐMð #ð 	FØŒJ×ÒÑÔÐÝ�tÔ(¨&Ñ1Ô1ð Ø”=ð eØ Ô,×4Ò4°T´ZÑ@Ô@�E�Eð Ô)ð eØ)-Ô)9×)AÒ)AÀ$Ä.Ñ)QÔ)Q˜œ˜à04Ô0@×0HÒ0HÈÌÐUYÔUcÑ0dÔ0dÑ-˜˜tœ~˜~ð <@Ô;K×;SÒ;SØ”J ¤°Ô0Añ<ô <Ñ8��t”~ tÔ'8Ð'8ð "Ô-×5Ò5°d´nÑEÔEˆDŒNàÔð 	4Ø.3Ð3ˆDŒJ˜Ô+ð ˜œ
Ð"Ð"Ø!&ˆDÔð Ô$ð 	0Ø!Ô/ˆDŒNð "Ð-ØÔ(ð WÝ)ØÔ&Ð(>ÕWeÐfjÔfpÑWqÔWqÐSqðñ ô ð ð õ )Ñ*Ô*ð W¨dÔ.Bð WØ×*Ò*Ð+AÀ4ÔCUÑVÔVÐVð 	×*Ò*Ð+AÑBÔBÐBØ×ÒÐ0Ñ1Ô1Ð1õ 	�ŠÐ2Ñ3Ô3Ð3Ý�ŠÐ8¨Ð8Ð8Ð8Ñ9Ô9Ð9Ý�ŠÐ:Ð&6Ð:Ð:Ð:Ñ;Ô;Ð;Ý�ŠÐh¸t¼yÔ?dÐhÐhÐhÑiÔiÐiØŒ9Ô0°DÔ4JÒJÐJÝ�KŠKÐvÐ\`Ô\rÐvÐvÐvÑwÔwÐwÝ�ŠÐvÐ\rÐvÐvÐvÑwÔwÐwÝ�ŠÐY°tÔ7WÐYÐYÑZÔZÐZÝ�ŠÐA°IÐAÐAÐAÑBÔBÐBÝ�ŠÐoÕ:OÐPUÐfjÐ:kÑ:kÔ:kÐoÐoÐoÑpÔpÐpàˆŒ
ÔÝ”Y‘[”[ˆ
ØˆØ)*Ð&ð "Ñ-µ"´'·.².ÝŒG�LŠLÐ/Õ1CÑDÔDñ3
ô 3
Ñ-õ &Ô4µR´W·\²\ÐBXÕZlÑ5mÔ5mÑnÔnˆDŒJØ×4Ò4°T´YÀÄ
ÑKÔKÐKØ×%Ò%Ñ'Ô'Ð'Ý  ¤Ô!7Ð;UÑ!UÑVÔVˆNØÔ(ð 3Ø15´Ô1GÐKeÑ1fÐ.Ø.°$Ô2RÑRÐ.Ð.à12Ð.å�KŠKÐ_Ñ`Ô`Ð`Ý�KŠKÐL¸NÐLÐLÑMÔMÐMÝ�KŠKÐZÀ$Ä*ÔBXÐZÐZÑ[Ô[Ð[ØÔ(ð Ý—’ðU¨^ð Uð UØ6ðUð Uð Uñô ð ð ;ð 	Fð 	FˆDÝ�DÔ)¨4µ¸¸tÑ1DÔ1DÑEÔEÐEÐEØ1AˆÔÔ.àŒ
×+Ò+¨D°)Ð=MÈuÑUÔUÐUõ ”,˜s¨4¬;Ð7Ñ7Ô7ˆà"%ˆÔØ'+¤zÔ'=ˆÔ$Ø�ŠÑÔÐØ%)ˆ	ØˆØÔ,×;Ò;¸DÀ$Ä*ÈdÌlÑ[Ô[ˆŒàÔð 	MØ�NŠN˜5Ð"6ÀtˆNÑLÔLÐLå˜>Ð+;Ñ<Ô<ð b	ñ b	ˆEØ/ÐÝÐ'¨Ñ5Ô5ð 2Ø ×*Ò*¨5Ñ1Ô1Ð1ð Œ !Ò#Ð#Ø!�”
ð "Ð-õ Ð$Ñ%Ô%Ð%à”^ dÔ&FÑFð ð
  Ô0×?Ò?ÀÀdÄjÐRVÔR^Ñ_Ô_ˆDŒLàˆDØˆKð ˜Ò&Ð&Ð+AÐ+MØ1°AÒ5Ð5¸dÔ>SÐ5Ý'9Ð:JÐLjÑ'kÔ'kÐ$Ø9¸AÑ=�DØ"&�K�KØ3°qÒ8Ð8Ø×(Ò(Ð)?Ñ@Ô@Ð@å!Ð"2Ñ3Ô3ˆNà&¨Ô)IÑIˆIØ˜AŠ~ˆ~Ø Ô<�	ØˆKØ*¨dÔ.NÑNÕQTØ˜DÔ<Ò<ñRô Rñ ˆMõ ˜=Ñ)Ô)ð eñ e�Ø˜qÑ �ØBMÐR_ÐbcÑRcÒBdÐBd˜dÔ>Ð>Ðjs�Ø48×4JÒ4JÈ>Ð[fÐhlÔhsÑ4tÔ4tÑ1�Ð1õ <?¸}Ñ;MÔ;M�Ô8Ý!*¨=Ñ!9Ô!9ð Yñ Y‘I�A�vØ˜A‘I�DØ$(¨1¡H°Ô0PÑ#PÐTUÒ#UÐ#uÐZ^ÐabÑZbÐguÒYu�LàÔ$Ô3×GÒGÈÑUÔUÐUà”yÔ>ÀmÐSÑSÝ*1°$´*Ð>OÐQ\Ñ*]Ô*]˜Ø*°&Ð8Ð8Ý"ŸNšNð!bñô ð ñ ð  $œyÔFÈ-ÒWÐWØ#3°vÐ#=Ð#=Ø39Ð:JÔ3K×3OÒ3OÑ3QÔ3Q L Là$(Ô$9Ð$EÝ(/°Ô0EÀ~Ñ(VÔ(Vð %Fà(,Ô(=Ô(JÐ(Vð )/¨Ô(?À4ÔCXÔCeÒ(eß&)¢c¡e¤eð %1 Lõ %+§N¢Nð )Gñ%&ô %&ð %&ð 4:¸/Ô3J×3PÒ3PÑ3RÔ3R L Là/5°oÔ/F×/LÒ/LÑ/NÔ/N å+0¬<¸ÈTÌYÔM]ÕejÔepÐ+qÑ+qÔ+q˜LØ œJÐ<Ô<ÀÔ@P×@WÒ@WÐXdÑ@eÔ@e×@iÒ@iÑ@kÔ@k×@pÒ@pÑ@rÔ@rÑrÐ<Ô<à"ð ,Ø×,Ò,Ð-CÑDÔDÐDØ&+˜à˜dÔ>Ñ>À!ÒCÐCØ'+Ô'<×'JÒ'JÈ4ÐQUÔQ[Ð]aÔ]iÑ'jÔ'j˜œð
 ¥ MÑ 2Ô 2°QÑ 6Ò6Ð6Ø Ô,Ô=ÅÔAZÒZÐZõ "Ô)¨$Ô*:Ô*BÈ%ÐPÑPÔPÐPõ (Ô3ð	 ð !˜™œð ]ð ]Ø'+×'9Ò'9¸%ÀÐI[Ñ'\Ô'\˜ð]ð ]ð ]ñ ]ô ]ð ]ð ]ð ]ð ]ð ]ð ]øøøð ]ð ]ð ]ð ]ð Ô3ð9æ 6Ñ 8Ô 8ð9õ #œ[¨Ñ6Ô6ð9õ ;@¼+ÀlÑ:SÔ:Sð9ð #*¨G°q¸4¼:Ô;QÑ7QÐTXÔTpÑ7pÑ,qÑ"q˜˜à"œ>¨\Ô-@Ò@Ð@Ý",ð !OÐSZÔSað  !Oð  !Oð  zFô  zMð  !Oð  !Oñ#ô #ð ð #*¨LÑ"8˜àÐ%Ô%®¨t×/FÒ/FÀvÑ/NÔ/NÑ)OÔ)OÑOÐ%Ô%à#ñ KlàÔ(Ô7×KÒKÈDÑQÔQÐQð  Ô-Ñ9¸dÔ>PÐSTÒ>TÑ>TÝ6Ñ8Ô8ð &¸T¼Yð &Ø-1¬^×-MÒ-MÈdÔN`Ñ-aÔ-a 
 
Ø!%¤ð &Ø 4Ð 4Ð 4Ð 4Ð 4Ð 4ö .0¬X×-EÒ-EØ$'×$5Ò$5°d´nÑ$EÔ$EØ$(Ô$6ñ."ô ." 
 
õ
 5?Ô4JÐ 1Ø#'Ô#5ð !MØ$kÐ$kÐ$kÐ$kÐ$kÐ$kà8LÐ$5Ø%6Ð%6Ñ%8Ô%8ð !&ð !&Ø15Ô1A×1QÒ1QØ(-×(8Ò(8Ñ(:Ô(:Ø(,Ô(:ñ2&ô 2& Jð!&ð !&ð !&ñ !&ô !&ð !&ð !&ð !&ð !&ð !&ð !&øøøð !&ð !&ð !&ð !&ö !8Ñ 9Ô 9ð	7à$(Ô$4Ô$EÍÔIbÒ$bÐ$bà,1×,FÒ,FÑ,HÔ,H 	å#*¨9°fÑ#=Ô#=ð !AØ09·²Ñ0@Ô0@ Iøà,6 	à'+Ô'<×'RÒ'RÐSWÐY]ÔYcÐeiÔeqÑ'rÔ'r˜œå",Ô"8˜ØÔ-ð ;ØcÐcÐcÐcÐcÐcà&:˜Gà$˜W™YœYð 2ð 2Ø œN×/Ò/Ñ1Ô1Ð1ð2ð 2ð 2ñ 2ô 2ð 2ð 2ð 2ð 2ð 2ð 2øøøð 2ð 2ð 2ð 2ð (,Ô'<×'NÒ'NÈtÐUYÔU_ÐaeÔamÑ'nÔ'n˜œð )-×(?Ò(?Ñ(AÔ(A˜à#Ô/ÔJð 9æ#-¨dÔ.?ÅÄÔAYÔAkÑ#lÔ#lð 9Ø $Ô 1× 6Ò 6Ñ 8Ô 8Ð 8àŸšÑ)Ô)Ð)Øœ
Ð.Ô.°!Ñ3Ð.Ô.Ø+0°D¸1±HÀÑ3NÑ+N˜œ
Ô(Ø'+Ô'<×'HÒ'HÈÈtÌzÐ[_Ô[gÑ'hÔ'h˜œØ×5Ò5Ø#Ø%Ø!Ø!Ø!Ø0Ø&Ø*7ð 6ñ 	ô 	ð 	ð 	ð (,Ô'<×'KÒ'KÈDÐRVÔR\Ð^bÔ^jÑ'kÔ'k˜œð
 ”|Ô5ð ¸¼Ô9Zð Þ1Ñ3Ô3ð +ÞœL™NœN˜NØ˜ñð
 ”<Ô1ð °T´\Ô5Vð Þ-Ñ/Ô/ð 'Þœ™œ˜Ø�Eñð �aŠxˆxÝ—’ð]Øœ
Ô.ð]ð ]à#,ð]ð ]ð ]ñô ð ð
 59�”Ô1àÔ0×=Ò=¸dÀDÄJÐPTÔP\Ñ]Ô]ˆDŒLØ×)Ò)Ø˜ E¨5°%Ð9MÈzÐivð *ñ ô ð õ Ô,°´	´Ð?Ð?Þ)Ñ+Ô+ð æ”O¦CÔ$6Ñ$8Ô$8Ñ9Ô9Ð9Ð9å—N’Nð_ñô ð ð Œ|Ô0ð Ø�ñð Œ?ð 	#�w t¨WÑ5Ô5ð 	#æ�D˜'Ñ"Ô"Ð"å�ŠÐoÑpÔpÐpØÔ&ð 	$¨4¬:Ô+KÐ+WØ×!Ò!Ñ#Ô#Ð#ð 	ÐÔ 7§<¢<¡>¤>Ñ1ÐÔÝ # D¤JÔ$:¸EÑ BÔ BÐØÔ,Ð/DÑDˆ
æØØØ)Ø”jÔ*Ø'ð
ñ 
ô 
ˆð 	�ŠÑÔÐØ $¤
Ô 5ˆ�ÑØ *ˆ�Ñà ˆÔàÔ×4Ò4°WÑ=Ô=Ð=à�Š�ÑÔÐà×&Ò& uÑ-Ô-ˆØ!×5Ò5ÀÐRYÐ5ÑZÔZÐð Œ9Ô ð 	B T¤ZÔ%EÐ%QÐVZÔV_ÔVpÐtuÒVuÐVuØ0ð Bð B�
Ý”w×'Ò'¨
°D´JÔ4TÑUÔUð BÝ—K’KÐ h¸jÐ hÐ hÐ hÑiÔiÐiÞ”M *¸DÐAÑAÔAÐAøàÔ,×9Ò9¸$ÀÄ
ÈDÌLÑYÔYˆŒð 	×!Ò!Ñ#Ô#Ð#ð Ô#Ð/Ø×$Ò$ T¤ZÑ0Ô0Ð0æ˜4œ:Ô1°:¸wÑGÔGÐGs<   öv%	ö%v)ö,v)ý3~	þ~þ~ÁAAA1	ÁA1AA5ÁA8AA5c                 ó  — | j         �ò|�ð| j         t          j        k    r|j        }n‡| j         t          j        k    r0dd l}|j                             ¦   «                              ¦   «         }nB| j         t          j	        k    r|j
        }n%| j         t          j        k    rdd l}|j        j
        }| j        �|                      |¦  «        nd|› �}t          j                             | j        j        |¦  «        }n| j        j        }|S )Nr   zrun-)rˆ  rT   rH  ÚnumberrJ  rg  rk  Úget_contextÚget_trial_idrK  ÚidrN  r@  Úrunr-  rn  rt  rç  rà   rí   )rš  r>  Úrun_idrn  r@  Úrun_namer£  s          r´   rz  zTrainer._get_output_dir*  sô   € ØÔ!Ð-°%Ð2CØÔ%­Ô)?Ò?Ð?Øœ��ØÔ'­?Ô+>Ò>Ð>Ø Ð Ð Ð àœ×.Ò.Ñ0Ô0×=Ò=Ñ?Ô?��ØÔ'­?Ô+AÒAÐAØœ��ØÔ'­?Ô+@Ò@Ð@Ø���àœœ�Ø.2¬lÐ.F�t—|’| EÑ*Ô*Ð*ÈOÐSYÈOÈOˆHÝ”g—l’l 4¤9Ô#7¸ÑBÔBˆGˆGà”iÔ*ˆGØˆr¶   c           	      óö  ‡— |€| j         }t          j                             ‰t          ¦  «        }t          j                             ‰t
          ¦  «        }t          j                             ‰t          ¦  «        }t          j                             ‰t          ¦  «        }t          j                             ‰t          ¦  «        }t          j                             ‰t          ¦  «        }t          j                             ‰t          ¦  «        }	t          j                             ‰¦  «        oqt          ˆfd„t          j        ‰¦  «        D ¦   «         ¦  «        pDt          j                             t          j                             ‰t          › d�¦  «        ¦  «        }
t          j                             ‰¦  «        r ˆfd„t          j        ‰¦  «        D ¦   «         ng }|
r| j        st#          d‰› d�¦  «        ‚t          d„ ||||	||fD ¦   «         ¦  «        s|
s|st#          d‰› �¦  «        ‚t$                               d‰› d	�¦  «         t          j                             |¦  «        rNt)          j        |¦  «        }|j        }|�1|t.          k    r&t$                               d
|› dt.          › d�¦  «         t          j                             |¦  «        s"t          j                             |¦  «        s|
�rÄt3          ¦   «         rÝt          j                             t          j                             ‰d¦  «        ¦  «        rt5          j        ‰t          dd¬¦  «         d S t9          | j        d¦  «        r(| j        j        du rt$                               d¦  «         t?          ¦   «          tA          j!        |dd¬¦  «        }d|d<   | "                    |d¬¦  «        }~d S | j        r2tG          | j$        j%        j&        | j$        |‰fi tO          ¦   «         ¤Ž d S | j        j(        rAt          j                             |¦  «        r"tR          j          *                    |d¬¦  «        }n%t?          ¦   «          tA          j!        |dd¬¦  «        }| "                    |d¦  «        }~|  +                    |¦  «         d S tY          |¦  «        �rZt9          |d¦  «        st9          |d¦  «        �rt9          |d¦  «        �rt          j         -                    ‰¦  «        rÈt9          |d¦  «        r=|j.        }t_          |¦  «        dk    rt$                               d¦  «         |d         }n|j0        }|rX|D ]>}t          j                             ‰|¦  «        }| 1                    ||||k    ¬¦  «         Œ?| 2                    |¦  «         d S | 1                    ‰|d¬¦  «         d S t$                               dt
          › d �¦  «         d S t$                               d!¦  «         d S tg          |‰t3          ¦   «         | j        j(        ¬"¦  «        }t3          ¦   «         s|  +                    |¦  «         d S d S )#Nc              3   ó¤   •K  — | ]J}t           j                             t           j                             ‰|¦  «        ¦  «        ¯?t          |v V — ŒKd S r  )rn  rt  Úisdirrç  ÚFSDP_MODEL_NAME©rö   Úfolder_namerû  s     €r´   r
  z0Trainer._load_from_checkpoint.<locals>.<genexpr>K  sa   øè è € ð ð àÝ”7—=’=¥¤§¢Ð.DÀkÑ!RÔ!RÑSÔSðÝ ;Ð.ðð ð ð ð ð r¶   z.binc           	      óš  •— g | ]Ç}t           j                             t           j                             ‰|¦  «        ¦  «        ¯?t           j                             t           j                             ‰|t
          ¦  «        ¦  «        sCt           j                             t           j                             ‰|t          ¦  «        ¦  «        ¯Å|‘ŒÈS r½   )rn  rt  r¯  rç  rB  rl   rk   r±  s     €r´   rø   z1Trainer._load_from_checkpoint.<locals>.<listcomp>U  s¦   ø€ ð ð ð àÝ”7—=’=¥¤§¢Ð.DÀkÑ!RÔ!RÑSÔSðõ
 ”G—N’N¥2¤7§<¢<Ð0FÈÕUiÑ#jÔ#jÑkÔkðõ ”w—~’~¥b¤g§l¢lÐ3IÈ;ÕXqÑ&rÔ&rÑsÔsðØðð ð r¶   zCheckpoint found at z* is only supported when using PyTorch FSDPc              3   óT   K  — | ]#}t           j                             |¦  «        V — Œ$d S r  )rn  rt  rB  )rö   Úfs     r´   r
  z0Trainer._load_from_checkpoint.<locals>.<genexpr>f  sD   è è € ð 
ð 
àõ ”—’˜qÑ!Ô!ð
ð 
ð 
ð 
ð 
ð 
r¶   z!Can't find a valid checkpoint at zLoading model from rÎ  z9You are resuming training from a checkpoint trained with z- of Transformers but your current version is zJ. This is not recommended and could yield to errors or unwanted behaviors.úuser_content.ptF©rt  Útagr   Úload_optimizerr  TzOEnabling FP16 and loading from smp < 1.10 checkpoint together is not supported.rô   ©Úmap_locationÚweights_onlyÚ_smp_is_partial©r„  r  Úactive_adapterÚactive_adaptersÚload_adapterr   zFMultiple active adapters detected will only consider the first adapterr   )rI  újThe intermediate checkpoints of PEFT may not be saved correctly, consider using a custom callback to save úi in corresponding saving folders. Check some examples here: https://github.com/huggingface/peft/issues/96úGCould not load adapter model, make sure to have `peft>=0.3.0` installed)r„  Úprefer_safe)4r²   rn  rt  rç  rm   rl   rk   rr   rq   rp   ro   r¯  r^  ÚlistdirrB  r°  rU  r%  r"  r#  r   Úfrom_json_fileÚtransformers_versionr   rG  r„   r|  rû  rF  rà   r  rw   rÀ   ÚloadÚload_state_dictr¡   rZ  r}  r7  r¾   Úsave_safetensorsÚsafetensorsÚ	load_fileÚ_issue_warnings_after_loadrµ   ÚexistsrÀ  rB  r¿  rÁ  Úset_adapterr+   )rš  rû  r²   Úconfig_fileÚadapter_weights_fileÚadapter_safe_weights_fileÚweights_fileÚweights_index_fileÚsafe_weights_fileÚsafe_weights_index_fileÚis_fsdp_ckptÚadapter_subdirsr„  Úcheckpoint_versionrz  Úload_resultrÀ  r¿  Úsubdir_nameÚpeft_ids    `                  r´   r   zTrainer._load_from_checkpoint>  s  ø€ Øˆ=Ø”JˆEå”g—l’lÐ#9½;ÑGÔGˆÝ!œwŸ|š|Ð,BÕDXÑYÔYÐÝ$&¤G§L¢LÐ1GÕIbÑ$cÔ$cÐ!Ý”w—|’|Ð$:½LÑIÔIˆÝœWŸ\š\Ð*@ÕBTÑUÔUÐÝœGŸLšLÐ)?ÕARÑSÔSÐÝ"$¤'§,¢,Ð/EÕG^Ñ"_Ô"_ÐÝ”w—}’}Ð%;Ñ<Ô<ð 	
åð ð ð ð å#%¤:Ð.DÑ#EÔ#Eðñ ô ñ ô ð ^õ Œw�~Š~�bœgŸlšlÐ+AÅoÐC[ÐC[ÐC[Ñ\Ô\Ñ]Ô]ð 	õ* Œw�}Š}Ð3Ñ4Ô4ð
ð ð ð ð å#%¤:Ð.DÑ#EÔ#Eðñ ô ð ð ð 	ð ð 	x Ô 4ð 	xÝÐvÐ4JÐvÐvÐvÑwÔwÐwõ ð 
ð 
ð !Ø%Ø&Ø+Ø(Ø-ðð
ñ 
ô 
ñ 
ô 
ð	[ð ð	[ð ð	[õ ÐYÐAWÐYÐYÑZÔZÐZå�ŠÐCÐ*@ÐCÐCÐCÑDÔDÐDåŒ7�>Š>˜+Ñ&Ô&ð 	Ý%Ô4°[ÑAÔAˆFØ!'Ô!<ÐØ!Ð-Ð2DÍÒ2SÐ2SÝ—’ð=ÐPbð =ð =Ý@Kð=ð =ð =ñô ð õ Œ7�>Š>˜,Ñ'Ô'ð T	=­2¬7¯>ª>Ð:KÑ+LÔ+Lð T	=ÐP\ñ T	=å&Ñ(Ô(ð *=Ý”7—>’>¥"¤'§,¢,Ð/EÐGXÑ"YÔ"YÑZÔZð #õ Ô.Ø3½ÈuÐejðñ ô ð ð ð õ ˜tœy¨&Ñ1Ô1ð °d´i´nÈÐ6LÐ6LÝŸšØmñô ð õ -Ñ.Ô.Ð.Ý!&¤¨LÀuÐ[_Ð!`Ñ!`Ô!`�Jà49�JÐ0Ñ1Ø"'×"7Ò"7¸
È4Ð"7Ñ"PÔ"P�Kà"˜
˜
ØÔ%ð =ÝØÔ$Ô*Ô6ØÔ$ØØ*ð	ð õ
 ,Ñ-Ô-ðð ð ð ð ð ”9Ô-ð aµ"´'·.².ÐARÑ2SÔ2Sð aÝ!,Ô!2×!<Ò!<Ð=NÐW\Ð!<Ñ!]Ô!]�J�Jå,Ñ.Ô.Ð.Ý!&¤¨LÀuÐ[_Ð!`Ñ!`Ô!`�Jð $×3Ò3°JÀÑFÔF�àØ×/Ò/°Ñ<Ô<Ð<Ð<Ð<õ ˜EÑ"Ô"ñ %	=õ ˜Ð/Ñ0Ô0ð jµG¸EÐCTÑ4UÔ4Uñ jÕ[bØ�~ñ\ô \ñ jõ ”7—>’>Ð"8Ñ9Ô9ð å˜uÐ&7Ñ8Ô8ð >Ø*/Ô*?˜Ý˜Ñ/Ô/°!Ò3Ð3Ý"ŸNšNÐ+sÑtÔtÐtØ)8¸Ô);˜˜à).Ô)=˜à&ð fØ+:ð sð s˜KÝ&(¤g§l¢lÐ3IÈ;Ñ&WÔ&W˜GØ!×.Ò.¨w¸ÐS^ÐbpÒSpÐ.ÑrÔrÐrÐrØ×)Ò)¨.Ñ9Ô9Ð9Ð9Ð9à×*Ò*Ð+AÀ>Ð`dÐ*ÑeÔeÐeÐeÐeå—N’NðbÝDXðbð bð bñô ð ð ð õ —’ÐhÑiÔiÐiÐiÐiõ 2ØÐ-Õ6MÑ6OÔ6OÐ]aÔ]fÔ]wðñ ô ˆKõ +Ñ,Ô,ð =Ø×/Ò/°Ñ<Ô<Ð<Ð<Ð<ð=ð =r¶   c                 óê  — t                                d| j        j        › d| j        j        › d�¦  «         t
          j                             | j        j        t          ¦  «        }t
          j                             | j        j        t          ¦  «        }t
          j                             | j        j        t          ¦  «        }t
          j                             | j        j        t          ¦  «        }t          ¦   «         r| j        n| j        }| j        r6t!          | j        | j        j        t#          | j        ¦  «         ¬¦  «         d S | j        r<t'          | j        j        j        | j        || j        j        fi t-          ¦   «         ¤Ž}d S t
          j                             |¦  «        s^t
          j                             |¦  «        s?t
          j                             |¦  «        s t
          j                             |¦  «        �rcd}t          ¦   «         �r t
          j                             t
          j                             | j        j        d¦  «        ¦  «        r)t3          j        | j        j        t          dd¬¦  «         d S | j        j        rAt
          j                             |¦  «        r"t:          j                             |d	¬
¦  «        }n%tA          ¦   «          t=          j!        |d	d¬¦  «        }d|d<   | "                    |d¬¦  «        }d S t#          |¦  «        �r�tG          |d¦  «        stG          |d¦  «        �rQtG          |d¦  «        �r@tG          |d¦  «        r@|j$        d         }	tK          |j$        ¦  «        dk    rt            &                    d¦  «         n|j'        }	t
          j                             |¦  «        st
          j                             |¦  «        r…	 | (                    | j        j        |	¦  «         nP# tR          $ rC}
|j*        |	         j+        r+d|j*        |	         j,        j-        › d�}tS          |¦  «        |
‚‚ d }
~
ww xY wddl.m/}  |g g ¦  «        }nËt            &                    dt          › d�¦  «         d}n¥t            &                    d¦  «         d}nˆ| j        j        rAt
          j                             |¦  «        r"t:          j                             |d	¬
¦  «        }n%tA          ¦   «          t=          j!        |d	d¬¦  «        }| "                    |d¦  «        }t          ¦   «         s|r|  0                    |¦  «         d S d S d S t
          j                             t
          j                             | j        j        tb          ¦  «        ¦  «        sLt
          j                             t
          j                             | j        j        td          ¦  «        ¦  «        rOtg          || j        j        t          ¦   «         ¬¦  «        }t          ¦   «         s|  0                    |¦  «         d S d S t            &                    d|› d�¦  «         d S )NzLoading best model from z	 (score: r  r  Tr¶  Fr·  rô   r  rº  r½  r¾  r¿  rÀ  rÁ  r   r   zCDetected multiple active adapters, will only consider the first onez0When using prompt learning PEFT methods such as zî, setting load_best_model_at_end=True can lead to errors, it is recommended to set this to False and to load the model manually from the checkpoint directory using PeftModel.from_pretrained(base_model, <path>) after training has finished.)Ú_IncompatibleKeysrÂ  rÃ  rÄ  z#Could not locate the best model at zi, if you are running a distributed training on multiple nodes, you should activate `--save_on_each_node`.)4r"  r#  r}  rv  Úbest_metricrn  rt  rç  rr   rp   rl   rk   r„   rY  r²   rN  r&   rµ   rU  r¡   rZ  r7  r¾   rÏ  rB  r|  rû  rà   rË  rÌ  rÀ   rÍ  rw   rÉ  rÊ  rF  rÀ  rB  rG  r¿  rÁ  r8  Úpeft_configÚis_prompt_learningÚ	peft_typer$  Útorch.nn.modules.modulerß  rÎ  ro   rq   r+   )rš  Úbest_model_pathÚbest_safe_model_pathÚbest_adapter_model_pathÚbest_safe_adapter_model_pathr²   rÛ  Úhas_been_loadedrz  r¿  ÚexcÚmsgrß  s                r´   rw  zTrainer._load_best_modelØ  ss  € Ý�ŠÐt¨t¬zÔ/OÐtÐtÐZ^ÔZdÔZpÐtÐtÐtÑuÔuÐuÝœ'Ÿ,š, t¤zÔ'GÍÑVÔVˆÝ!œwŸ|š|¨D¬JÔ,LÕN_Ñ`Ô`ÐÝ"$¤'§,¢,¨t¬zÔ/OÕQeÑ"fÔ"fÐÝ')¤w§|¢|°D´JÔ4TÕVoÑ'pÔ'pÐ$å&=Ñ&?Ô&?ÐO�Ô"Ð"ÀTÄZˆØÔ$ð r	Ý%ØÔ"Ø”
Ô0Ý'5°d´jÑ'AÔ'AÐ#Aðñ ô ð ð ð ð
 Ô!ð l	Ý)ØÔ Ô&Ô2ØÔ ØØ”
Ô0ð	ð õ
 (Ñ)Ô)ðð ˆKˆKˆKõ ŒG�NŠN˜?Ñ+Ô+ðd	åŒw�~Š~Ð2Ñ3Ô3ðd	õ Œw�~Š~Ð5Ñ6Ô6ðd	õ Œw�~Š~Ð:Ñ;Ô;ñ	d	ð #ˆOÝ&Ñ(Ô(ñ PAÝ”7—>’>¥"¤'§,¢,¨t¬zÔ/OÐQbÑ"cÔ"cÑdÔdð Qõ Ô.Ø!œZÔ=Ý(Ø %Ø',ð	ñ ô ð ð ð ð ”yÔ1ð hµb´g·n²nÐEYÑ6ZÔ6Zð hÝ%0Ô%6×%@Ò%@ÐAUÐ^cÐ%@Ñ%dÔ%d˜
˜
å0Ñ2Ô2Ð2Ý%*¤Z°ÈeÐbfÐ%gÑ%gÔ%g˜
à49�JÐ0Ñ1Ø"'×"7Ò"7¸
È4Ð"7Ñ"PÔ"P�K�K�Kå! %Ñ(Ô(ñ 8Kõ   Ð'7Ñ8Ô8ð )0½GÀEÐK\Ñ<]Ô<]ñ )0ÕcjØ˜~ñdô dñ )0õ # 5Ð*;Ñ<Ô<ð BØ-2Ô-BÀ1Ô-E˜NÝ" 5Ô#8Ñ9Ô9¸AÒ=Ð=Ý &§¢Ð/tÑ uÔ uÐ uøà-2Ô-A˜Nåœ7Ÿ>š>Ð*AÑBÔBð 4ÅbÄgÇnÂnÐUqÑFrÔFrð 4ð*Ø %× 2Ò 2°4´:Ô3SÐUcÑ dÔ dÐ dÐ døÝ#/ð *ð *ð *Ø#(Ô#4°^Ô#DÔ#Wð !*ð)8Ø+0Ô+<¸^Ô+LÔ+VÔ+\ð)8ð )8ð )8ð %(õ +7°sÑ*;Ô*;ÀÐ$Dà$)øøøøð*øøøð RÐQÐQÐQÐQÐQà*;Ð*;¸BÀÑ*CÔ*C˜K˜Kå"ŸNšNð!jÝL`ð!jð !jð !jñô ð ð
 /4˜O˜OåŸšÐ'pÑqÔqÐqØ*/˜˜ð ”yÔ1ð hµb´g·n²nÐEYÑ6ZÔ6Zð hÝ%0Ô%6×%@Ò%@ÐAUÐ^cÐ%@Ñ%dÔ%d˜
˜
å0Ñ2Ô2Ð2Ý%*¤Z°ÈeÐbfÐ%gÑ%gÔ%g˜
ð
 #(×"7Ò"7¸
ÀEÑ"JÔ"J�KÝ.Ñ0Ô0ð A°_ð AØ×3Ò3°KÑ@Ô@Ð@Ð@Ð@ðAð Að Að AåŒW�^Š^�BœGŸLšL¨¬Ô)IÕKbÑcÔcÑdÔdð 	ÕhjÔho×hvÒhvÝŒG�LŠL˜œÔ9Õ;MÑNÔNñi
ô i
ð 	õ 2Ø�t”zÔ7Õ@WÑ@YÔ@Yðñ ô ˆKõ +Ñ,Ô,ð =Ø×/Ò/°Ñ<Ô<Ð<Ð<Ð<ð=ð =õ �NŠNðP°oð Pð Pð Pñô ð ð ð s   Ï4 P Ð
Q"Ð>QÑQ"c                 ó   — t          |j        ¦  «        dk    rx| j        j        �It	          |j        ¦  «        t	          | j        j        ¦  «        k    r| j                             ¦   «          n#t                               d|j        › d�¦  «         t          |j        ¦  «        dk    r%t                               d|j        › d�¦  «         d S d S )Nr   z8There were missing keys in the checkpoint model loaded: rÎ  z;There were unexpected keys in the checkpoint model loaded: )	rB  Úmissing_keysr²   Ú_keys_to_ignore_on_saver@  rÄ  r"  rG  Úunexpected_keys)rš  rÛ  s     r´   rÎ  z"Trainer._issue_warnings_after_loadT  sÝ   € Ýˆ{Ô'Ñ(Ô(¨AÒ-Ð-ØŒzÔ1Ð=Å#ÀkÔF^ÑB_ÔB_ÕcfØ”
Ô2ñdô dò Cð Cð ”
×&Ò&Ñ(Ô(Ð(Ð(å—’ÐuÐZeÔZrÐuÐuÐuÑvÔvÐvÝˆ{Ô*Ñ+Ô+¨qÒ0Ð0Ý�NŠNØlÈkÔNiÐlÐlÐlñô ð ð ð ð 1Ð0r¶   c           
      óð  — |                       |¬¦  «        }|                      || j        j        |¦  «         t	          | j        t          j        j        j        ¦  «        r•|s“| j	        j
        }|                     d¦  «        sd|› �}	 | j                             ||         ¦  «         nK# t          $ r>}t          d|› dt          |                     ¦   «         ¦  «        › d|› d�¦  «        |‚d }~ww xY w|S )N©Úignore_keysÚeval_ú9The `metric_for_best_model` training argument is set to 'úW', which is not found in the evaluation metrics. The available evaluation metrics are: zX. Please ensure that the `compute_metrics` function returns a dictionary that includes 'zM' or consider changing the `metric_for_best_model` via the TrainingArguments.)Úevaluaterp  r}  ru  r±   ra  rÀ   rR  rj  rà   r*  Ú
startswithr†  r9  r¹   rÚ  )rš  r>  rü  r   rY  Úmetric_to_checkrê  s          r´   rK  zTrainer._evaluatea  s5  € Ø—-’-Ð,@�-ÑAÔAˆØ×!Ò! %¨¬Ô)?ÀÑIÔIÐIõ �dÔ'­¬Ô)AÔ)SÑTÔTð 	Ð]kð 	Ø"œiÔ=ˆOØ"×-Ò-¨gÑ6Ô6ð <Ø";¨/Ð";Ð";�ð	ØÔ!×&Ò& w¨Ô'?Ñ@Ô@Ð@Ð@øÝð ð ð Ýð`ÐP_ð `ð `å=AÀ'Ç,Â,Á.Ä.Ñ=QÔ=Qð`ð `ð n}ð`ð `ð `ñô ð ðøøøøðøøøð ˆs   Â
 B+ Â+
C3Â59C.Ã.C3c	                 ó  — | j         j        �r>| j        j        | j        k    �r(t          ¦   «         rt          j        ¦   «          i }	|                      |¦  «         	                    ¦   «          
                    ¦   «         }
||z  }t          |
| j        j        | j        z
  z  d¦  «        |	d<   |�3t          |t          j        ¦  «        r| 
                    ¦   «         n||	d<   |�||	d<   n|                      ¦   «         |	d<   | xj        |
z  c_        | j        j        | _        |                      ¦   «          |                      |	|¦  «         d }| j         j        rS|                      ||¦  «        }|                      ||¬¦  «        }| j        j        t0          j        k    r|| j         _        | j         j        rH|                      ||¦  «         | j                             | j        | j        | j         ¦  «        | _         d S d S )Nr±  rÓ  r�  rá  )rY  r>  )r†  Ú
should_logr}  ru  rH  r�   rÏ  rÐ  Ú_nested_gatherÚmeanr'  rà  r±   rÀ   rÆ  rÖ   rG  rx  ry  Úshould_evaluaterK  Ú_determine_best_metricrà   r'  rX   r(  rm  Ú_save_checkpointre  Úon_save)rš  rŽ  r�  r²   r>  rA  rü  ró  rá  ÚlogsÚtr_loss_scalarrY  Úis_new_best_metrics                r´   rl  z Trainer._maybe_log_save_evaluatev  sô  € ð Œ<Ô"ñ 	' t¤zÔ'=ÀÔ@\Ò'\Ñ'\Ý%Ñ'Ô'ð Ý”‘”�à%'ˆDð "×0Ò0°Ñ9Ô9×>Ò>Ñ@Ô@×EÒEÑGÔGˆNð �wÑˆGå  °4´:Ô3IÈDÔLhÑ3hÑ!iÐklÑmÔmˆD�‰LØÐ$Ý8BÀ9ÍeÌlÑ8[Ô8[Ð$j I§N¢NÑ$4Ô$4Ð$4Ðaj��[Ñ!ØÐ(Ø(5��_Ñ%Ð%à(,×(?Ò(?Ñ(AÔ(A��_Ñ%àÐ#Ô# ~Ñ5Ð#Ô#Ø+/¬:Ô+AˆDÔ(Ø�OŠOÑÔÐà�HŠH�T˜:Ñ&Ô&Ð&àˆØŒ<Ô'ð 	>Ø—n’n UÐ,@ÑAÔAˆGØ!%×!<Ò!<ÀWÐTYÐ!<Ñ!ZÔ!ZÐàŒyÔ&­,Ô*;Ò;Ð;Ø+=�”Ô(àŒ<Ô#ð 	^Ø×!Ò! %¨Ñ/Ô/Ð/ØÔ0×8Ò8¸¼ÀDÄJÐPTÔP\Ñ]Ô]ˆDŒLˆLˆLð	^ð 	^r¶   c                 óL  — |€d S | j         j        dk    rp| j         j        }t          j                             |d|› d�¦  «        }t          j                             |¦  «        s t                               d|› d�¦  «         d S n[t          j                             |d¦  «        }t          j                             |¦  «        st                               d¦  «         d S t          ¦   «         5  t          j        |¦  «        }d d d ¦  «         n# 1 swxY w Y   t          j        |d         ¦  «         t          j                             |d	         ¦  «         t          j                             |d
         ¦  «         t#          ¦   «         rt%          j        |d         ¦  «         | j         j        t(          j        k    }t          j                             ¦   «         rt1          dt          j        ||¦  «         t3          ¦   «         rt1          dt          j        ||¦  «         t7          ¦   «         rt1          dt          j        ||¦  «         t;          ¦   «         rt1          dt          j        ||¦  «         t?          ¦   «         rt1          dt          j         ||¦  «         d S d S )Nr   Ú
rng_state_ú.pthz$Didn't find an RNG file for process zr, if you are resuming a training that wasn't launched in a distributed fashion, reproducibility is not guaranteed.úrng_state.pthz†Didn't find an RNG file, if you are resuming a training that was launched in a distributed fashion, reproducibility is not guaranteed.ÚpythonÚnumpyrô   r   ÚCUDAÚNPUÚHPUÚMLUÚMUSA)!rà   r  r  rn  rt  rç  rB  r"  r#  rÎ   rÀ   rÉ  rÄ  ÚsetstaterÄ   Ú	set_stateÚset_rng_stater�   rÏ  rO  rh   rP  ÚcudaÚis_availablerO   r‹   Únpur†   Úhpur‡   Úmlur‰   Úmusa)rš  r^  r  Úrng_fileÚcheckpoint_rng_stateÚis_distributeds         r´   rN  zTrainer._load_rng_stateŸ  s¶  € àÐØˆFàŒ9Ô !Ò#Ð#Ø œIÔ3ˆMÝ”w—|’| JÐ0P¸]Ð0PÐ0PÐ0PÑQÔQˆHÝ”7—>’> (Ñ+Ô+ð Ý—’ðc¸=ð cð cð cñô ð ð �ðõ ”w—|’| J°Ñ@Ô@ˆHÝ”7—>’> (Ñ+Ô+ð Ý—’ðBñô ð ð �å‰^Œ^ð 	8ð 	8Ý#(¤:¨hÑ#7Ô#7Ð ð	8ð 	8ð 	8ñ 	8ô 	8ð 	8ð 	8ð 	8ð 	8ð 	8ð 	8øøøð 	8ð 	8ð 	8ð 	8åŒÐ,¨XÔ6Ñ7Ô7Ð7Ý
Œ	×ÒÐ0°Ô9Ñ:Ô:Ð:ÝŒ×"Ò"Ð#7¸Ô#>Ñ?Ô?Ð?Ý!Ñ#Ô#ð 	:ÝÔÐ1°%Ô8Ñ9Ô9Ð9àœÔ0µLÔ4LÒLˆÝŒ:×"Ò"Ñ$Ô$ð 	_Ý$ V­U¬ZÐ9MÈ~Ñ^Ô^Ð^Ý!Ñ#Ô#ð 	]Ý$ U­E¬IÐ7KÈ^Ñ\Ô\Ð\Ý!Ñ#Ô#ð 	]Ý$ U­E¬IÐ7KÈ^Ñ\Ô\Ð\Ý!Ñ#Ô#ð 	]Ý$ U­E¬IÐ7KÈ^Ñ\Ô\Ð\Ý"Ñ$Ô$ð 	_Ý$ V­U¬ZÐ9MÈ~Ñ^Ô^Ð^Ð^Ð^ð	_ð 	_s   Ã.DÄDÄDc           
      ó˜  — d}| j         j        ��:| j         j        }|                     d¦  «        sd|› �}	 ||         }nH# t          $ r;}t          d|› dt	          |                     ¦   «         ¦  «        › d�¦  «        |‚d}~ww xY w| j         j        rt          j        nt          j	        }| j
        j        €4| j         j        rt          d¦  «        nt          d¦  «        | j
        _         ||| j
        j        ¦  «        rH|| j
        _        | j         j        t          j        t          j        fv r| j
        j        | j
        _        d	}|S )
z­
        Determine if the model should be saved based on the evaluation metrics.

        Returns:
            bool: True if a new best metric was found, else False
        FNró  rô  rõ  zJ. Consider changing the `metric_for_best_model` via the TrainingArguments.z-infÚinfT)rà   r*  r÷  r9  r¹   rÚ  Úgreater_is_betterrÄ   ÚgreaterÚlessr}  rà  r  r'  rX   ÚSTEPSÚEPOCHru  Úbest_global_step)rš  rY  r>  r  rø  Úmetric_valuerê  Úoperators           r´   rþ  zTrainer._determine_best_metricÊ  sh  € ð #ÐàŒ9Ô*Ñ6Ø"œiÔ=ˆOà"×-Ò-¨gÑ6Ô6ð <Ø";¨/Ð";Ð";�ðØ& Ô7��øÝð ð ð Ýð^ÐP_ð ^ð ^Ý=AÀ'Ç,Â,Á.Ä.Ñ=QÔ=Qð^ð ^ð ^ñô ð ðøøøøðøøøð &*¤YÔ%@ÐM•r”z�zÅbÄgˆHàŒzÔ%Ð-Ø:>¼)Ô:UÐ)g­¨v©¬¨Õ[`ÐafÑ[gÔ[g�”
Ô&àˆx˜ d¤jÔ&<Ñ=Ô=ð *Ø)5�”
Ô&à”9Ô*­|Ô/AÅ<ÔCUÐ.VÐVÐVØ26´*Ô2H�D”JÔ/à%)Ð"à!Ð!s   ·A  Á 
BÁ
6B Â Bc                 óø  — t           › d| j        j        › �}| j        €|€|                      ¦   «          |                      |¬¦  «        }t          j                             ||¦  «        }|  	                    |d¬¦  «         | j
        j        t          j        t          j        fv rß| j        j        rÓt!          ¦   «         rt#          j        d¦  «         nO| j
        j        t(          j        k    rt-          j        ¦   «          n!t1          ¦   «         rt3          j        ¦   «          t           › d| j        j        › �}t          j                             ||¦  «        }t          j                             |¦  «        r|| j        _        | j
        j        s?|                      |¦  «         |                      |¦  «         |                      |¦  «         | j
        j         rÙd„ | j!        j"        | j#        gz   D ¦   «         D ]|}|j$        j%        }	|                     ¦   «         }
tM          | j        j'        |	         tP          ¦  «        r&| j        j'        |	          )                    |
¦  «         Œm|
| j        j'        |	<   Œ}| j         *                    t          j                             |tV          ¦  «        ¦  «         | j
        j,        r|  -                    |¦  «         | j
        j         r|  .                    d|¬¦  «         d S d S )Nr}  )r>  Trr  r)  c                 ó<   — g | ]}t          |t          ¦  «        ¯|‘ŒS r½   r  r  s     r´   rø   z,Trainer._save_checkpoint.<locals>.<listcomp>  s:   € ð ð ð ØÕQ[Ð\^Õ`oÑQpÔQpðØðð ð r¶   r-  )/rP   r}  ru  rˆ  rx  rz  rn  rt  rç  rv  rà   r'  rX   r   r!  r"  r�   rÏ  Ú
rendezvousrO  rh   rP  ÚdistrÑ  r„   r|  rÏ  rv  Úsave_only_modelÚ_save_optimizer_and_schedulerÚ_save_scalerÚ_save_rng_staterm  re  rç   r†  r<  r=  r±   r  r¹   r�  rw  rx  rk  Ú_push_from_checkpointÚ_rotate_checkpoints)rš  r²   r>  Úcheckpoint_folderr£  rí   Úbest_checkpoint_folderÚbest_checkpoint_dirr  Úcb_nameÚcb_states              r´   rÿ  zTrainer._save_checkpointð  sÐ  € õ  5ÐOÐO°t´zÔ7MÐOÐOÐàÔ!Ð)¨e¨mØ�OŠOÑÔÐà×&Ò&¨UÐ&Ñ3Ô3ˆÝ”W—\’\ 'Ð+<Ñ=Ô=ˆ
Ø�Š˜
°4ˆÑ8Ô8Ð8àŒ9Ô"¥|Ô'9½<Ô;MÐ&NÐNÐNÐSWÔS]ÔSnÐNõ &Ñ'Ô'ð Ý”Ð6Ñ7Ô7Ð7Ð7Ø”Ô(­LÔ,DÒDÐDÝ”‘”��Ý(Ñ*Ô*ð Ý”‘”�å(=Ð%]Ð%]ÀÄ
Ô@[Ð%]Ð%]Ð"Ý"$¤'§,¢,¨wÐ8NÑ"OÔ"OÐåŒw�~Š~Ð1Ñ2Ô2ð GØ3F�”
Ô0àŒyÔ(ð 	-à×.Ò.¨zÑ:Ô:Ð:Ø×Ò˜jÑ)Ô)Ð)à× Ò  Ñ,Ô,Ð,ð Œ9Ô ð 	Rðð Ø!Ô2Ô<ÀÄ¸~ÑMðñ ô ð Fð F�ð œ,Ô/�ØŸ8š8™:œ:�Ý˜dœjÔ;¸GÔDÅdÑKÔKð FØ”JÔ1°'Ô:×AÒAÀ(ÑKÔKÐKÐKà=E�D”JÔ1°'Ñ:Ð:ØŒJ×#Ò#¥B¤G§L¢L°Õ=OÑ$PÔ$PÑQÔQÐQàŒ9Ô ð 	3Ø×&Ò& zÑ2Ô2Ð2ð Œ9Ô ð 	Ià×$Ò$¨tÀÐ$ÑHÔHÐHÐHÐHð	Ið 	Ir¶   c           	      óÂ  — t          j        ¦   «         t          j                              ¦   «         t          j                              ¦   «         dœ}t          j                             ¦   «         rg| j        j	        t          j        k    r't          j        j                              ¦   «         |d<   n&t          j        j                              ¦   «         |d<   t          ¦   «         rt          j        ¦   «         |d<   t          ¦   «         rg| j        j	        t          j        k    r't          j        j                              ¦   «         |d<   n&t          j        j                              ¦   «         |d<   t#          ¦   «         rg| j        j	        t          j        k    r't          j        j                              ¦   «         |d<   n&t          j        j                              ¦   «         |d<   t'          ¦   «         rg| j        j	        t          j        k    r't          j        j                              ¦   «         |d<   n&t          j        j                              ¦   «         |d<   t+          ¦   «         r]| j        j	        t          j        k    r"t          j                             ¦   «         |d<   n!t          j                             ¦   «         |d<   t/          j        |d¬	¦  «         | j        j        d
k    r5t	          j        |t.          j                             |d¦  «        ¦  «         d S t	          j        |t.          j                             |d| j        j        › d�¦  «        ¦  «         d S )N)r  r	  rô   r  r   r  r  r  r  Tr  r   r  r  r  )rÄ  ÚgetstaterÄ   Ú	get_staterÀ   Úget_rng_stater  r  rà   rO  rh   rP  Úget_rng_state_allr�   rÏ  r‹   r  r†   r  r‡   r  r‰   r  rn  ro  r  ry  rt  rç  r  )rš  rí   Ú
rng_statess      r´   r,  zTrainer._save_rng_state,  s±  € õ ”oÑ'Ô'Ý”Y×(Ò(Ñ*Ô*Ý”<×-Ò-Ñ/Ô/ð
ð 
ˆ
õ
 Œ:×"Ò"Ñ$Ô$ð 	GØŒyÔ&­,Ô*BÒBÐBå%*¤ZÔ%6×%HÒ%HÑ%JÔ%J�
˜6Ñ"Ð"å%*¤ZÔ%6×%DÒ%DÑ%FÔ%F�
˜6Ñ"å!Ñ#Ô#ð 	3Ý "Ô 0Ñ 2Ô 2ˆJ�uÑå!Ñ#Ô#ð 	EØŒyÔ&­,Ô*BÒBÐBÝ$)¤IÔ$4×$FÒ$FÑ$HÔ$H�
˜5Ñ!Ð!å$)¤IÔ$4×$BÒ$BÑ$DÔ$D�
˜5Ñ!å!Ñ#Ô#ð 	EØŒyÔ&­,Ô*BÒBÐBÝ$)¤IÔ$4×$FÒ$FÑ$HÔ$H�
˜5Ñ!Ð!å$)¤IÔ$4×$BÒ$BÑ$DÔ$D�
˜5Ñ!å!Ñ#Ô#ð 	EØŒyÔ&­,Ô*BÒBÐBÝ$)¤IÔ$4×$FÒ$FÑ$HÔ$H�
˜5Ñ!Ð!å$)¤IÔ$4×$BÒ$BÑ$DÔ$D�
˜5Ñ!å"Ñ$Ô$ð 	@ØŒyÔ&­,Ô*BÒBÐBÝ%*¤Z×%AÒ%AÑ%CÔ%C�
˜6Ñ"Ð"å%*¤Z×%=Ò%=Ñ%?Ô%?�
˜6Ñ"õ 	Œ�J¨Ð.Ñ.Ô.Ð.àŒ9Ô 1Ò$Ð$ÝŒJ�z¥2¤7§<¢<°
¸OÑ#LÔ#LÑMÔMÐMÐMÐMåŒJ�z¥2¤7§<¢<°
Ð<fÈÌÔI`Ð<fÐ<fÐ<fÑ#gÔ#gÑhÔhÐhÐhÐhr¶   c                 ó	  — t          ¦   «         �r‚t          j        d¦  «         | j        r‹| j                             ¦   «         | j                             ¦   «         dœ}t          j        |t          j
                             |d| j        j        › d| j        j        › dt          › �¦  «        d¬¦  «         nOt          j        | j                             ¦   «         t          j
                             |t          ¦  «        ¦  «         t!          j        d¬	¦  «        5 }t          j        | j                             ¦   «         t          j
                             |t&          ¦  «        ¦  «         t)          |¦  «         d d d ¦  «         n# 1 swxY w Y   �nt+          ¦   «         r¬| j                             d¬
¦  «        }t/          j        ¦   «          t/          j        ¦   «         dk    st.          j        j        j        rOt/          j        |t          j
                             |t          ¦  «        dt.          j        j        j        ¬¦  «         �nb| j        r’dt=          t?          j         | j!        j"        ¦  «        j#         $                    ¦   «         ¦  «        v }|r1tK          | j        ¦  «        r| j!         "                    |d¬¦  «         nä| j!         "                    |¦  «         nÉ| j&        rgtO          | j(        j        j)        | j(        | j        |fi tU          ¦   «         ¤Ž tW          | j(        j        j)        | j(        | j        | j        |¦  «         n[| j        j,        rOt[          j        | j                             ¦   «         t          j
                             |t          ¦  «        ¦  «         | j        ot]          | j        t^          ¦  «         }| j        j,        r¤| j        r|rŸt          ¦   «         s�t!          j        d¬	¦  «        5 }t[          j        | j                             ¦   «         t          j
                             |t&          ¦  «        ¦  «         d d d ¦  «         n# 1 swxY w Y   t)          |¦  «         d S d S d S d S )NÚsaving_optimizer_states)r`  Úshard_metadatar  ú-of-r}  F©Úmaster_onlyT©Úrecord)Úgather_if_shardr   )r   Úv3Úexclude_frozen_parameters)rD  )0r�   rÏ  r'  r™  r`  rz  r²   Úget_shard_metadatary  rn  rt  rç  rà   r  r  r{  r9  Úcatch_warningsra  r|  rM   r„   Úlocal_state_dictr|  rÑ  Úrdp_rankr}  r~  Úshard_optimizer_staterN  r@  rº   r»   rY  Úsave_checkpointr¼   rÚ  rµ   rU  r£   rZ  r7  r¾   r¤   rm  rÀ   r±   r¨   )rš  rí   ÚoptmÚcaught_warningsÚopt_state_dictÚ accept_exclude_frozen_parametersÚis_deepspeed_custom_schedulers          r´   r*  z%Trainer._save_optimizer_and_scheduler^  sÄ  € Ý!Ñ#Ô#ñ 1	^ÝŒMÐ3Ñ4Ô4Ð4ØÔ*ð _à!%¤×!:Ò!:Ñ!<Ô!<Ø&*¤j×&CÒ&CÑ&EÔ&Eðð �õ ”ØÝ”G—L’LØ"Ð$o¨4¬9Ô+BÐ$oÐ$oÈÌ	ÔH\Ð$oÐ$oÕ_mÐ$oÐ$oñô ð !&ðñ ô ð ð õ ”˜œ×1Ò1Ñ3Ô3µR´W·\²\À*ÍnÑ5]Ô5]Ñ^Ô^Ð^ÝÔ(°Ð5Ñ5Ô5ð 5¸Ý”˜Ô)×4Ò4Ñ6Ô6½¼¿ºÀZÕQ_Ñ8`Ô8`ÑaÔaÐaÝ# OÑ4Ô4Ð4ð5ð 5ð 5ñ 5ô 5ð 5ð 5ð 5ð 5ð 5ð 5øøøð 5ð 5ð 5ð 5ùõ %Ñ&Ô&ð 	^Ø!œ^×<Ò<ÈUÐ<ÑSÔSˆNÝŒK‰MŒMˆMÝŒ|‰~Œ~ Ò"Ð"¥c¤i¤mÔ&IÐ"Ý”Ø"Ý”G—L’L ­^Ñ<Ô<Ø Ý”y”}Ô:ð	ñ ô ð ùð Ô&ð 	^ð 0KÍcÝÔ! $Ô"4Ô"DÑEÔEÔP×UÒUÑWÔWñOô Oð 0Ð,ð 0ð ?µNÀ4Ä:Ñ4NÔ4Nð ?ØÔ"×2Ò2°:ÐY]Ð2Ñ^Ô^Ð^Ð^àÔ"×2Ò2°:Ñ>Ô>Ð>Ð>ØÔ!ð 
	^åØÔ Ô&Ô2°DÔ4DÀdÄjÐR\ðð Ý`uÑ`wÔ`wðð ð õ  ØÔ Ô&Ô2°DÔ4DÀdÄnÐVZÔV`Ðblñô ð ð ð ŒYÔ"ð 	^åŒJ�t”~×0Ò0Ñ2Ô2µB´G·L²LÀÍ^Ñ4\Ô4\Ñ]Ô]Ð]ð )-Ô(Að )
Í*ØÔÕ8ñK
ô K
ð G
Ð%ð ŒIÔ!ð	1àÔ.ð	1à2Oð	1õ +Ñ,Ô,ð	1õ
 Ô(°Ð5Ñ5Ô5ð e¸Ý”
˜4Ô,×7Ò7Ñ9Ô9½2¼7¿<º<È
ÕTbÑ;cÔ;cÑdÔdÐdðeð eð eñ eô eð eð eð eð eð eð eøøøð eð eð eð eå Ñ0Ô0Ð0Ð0Ð0ð	1ð 	1ð 	1ð 	1ð 	1ð 	1s&   ÄAFÆF	ÆF	ÐAQ$Ñ$Q(Ñ+Q(c                 ó  ‡— ‰€dS | j         r·t          | j        t          ¦  «        s›t	          j        d¬¦  «        5 }t          ¦   «          | j                             t          j	        t          j                             ‰t          ¦  «        d¬¦  «        ¦  «         ddd¦  «         n# 1 swxY w Y   t          |¦  «         dS t          ¦   «         r:t!          j        t          j                             ‰t"          ¦  «        dz   ¦  «        nÏt          j                             t          j                             ‰t"          ¦  «        ¦  «        p�t          j                             t          j                             ‰t&          ¦  «        ¦  «        pKt          j                             ‰¦  «        o,t+          ˆfd„t          j        ‰¦  «        D ¦   «         ¦  «        }| j        rGt!          j        t          j                             ‰d| j        j        › dt"          › �¦  «        ¦  «        n|}|�rÖt          j                             t          j                             ‰t          ¦  «        ¦  «        �r•t5          ¦   «         �rµ| j        rnt          ¦   «          t          j	        t          j                             ‰d	| j        j        › d
| j        j        › dt"          › �¦  «        dd¬¦  «        }|d         }nHt          ¦   «          t          j	        t          j                             ‰t"          ¦  «        dd¬¦  «        }t	          j        d¬¦  «        5 }t          ¦   «          t          j	        t          j                             ‰t          ¦  «        dd¬¦  «        }ddd¦  «         n# 1 swxY w Y   t          |¦  «         t9          j        || j        j        ¦  «         t9          j        || j        j        ¦  «         | j                             |¦  «         | j                             |¦  «         dS t          ¦   «         rct          j                             t          j                             ‰d¦  «        ¦  «        rˆfd„}nˆfd„}| j          !                    |¦  «         nÁ| j        j        dk    r| j        j        nd}| j"        r<tG          | j$        j%        j&        | j$        | j        | j'        ‰fi tQ          ¦   «         ¤Ž n`t          ¦   «          | j                             t          j	        t          j                             ‰t"          ¦  «        |d¬¦  «        ¦  «         t	          j        d¬¦  «        5 }t          ¦   «          | j                             t          j	        t          j                             ‰t          ¦  «        d¬¦  «        ¦  «         ddd¦  «         n# 1 swxY w Y   t          |¦  «         dS dS dS )z3If optimizer and scheduler states exist, load them.NTr@  ©r¼  ú_*c              3   óÖ   •K  — | ]c}t           j                             t           j                             ‰|¦  «        ¦  «        ¯?t                               d ¦  «        d         |v V — ŒddS )rÎ  r   N)rn  rt  r¯  rç  ÚOPTIMIZER_NAME_BINrà  )rö   r²  r^  s     €r´   r
  z8Trainer._load_optimizer_and_scheduler.<locals>.<genexpr>·  ss   øè è € ð ð à'Ýœ7Ÿ=š=­¬¯ª°jÀ+Ñ)NÔ)NÑOÔOðÝ*×0Ò0°Ñ5Ô5°aÔ8¸KÐGðð ð ð ð ð r¶   z	rank*-of-r}  r  r=  rô   rº  r`  r¶  c                 ó    •— |                      t          j        t          j                             ‰t          ¦  «        d¬¦  «        ¦  «         d S )NTr   )rÊ  r|  rÉ  rn  rt  rç  r{  ©ÚmodÚoptr^  s     €r´   Úopt_load_hookz<Trainer._load_optimizer_and_scheduler.<locals>.opt_load_hookè  s>   ø€ Ø×/Ò/µ´½¼¿ºÀjÕR`Ñ9aÔ9aÐkoÐ0pÑ0pÔ0pÑqÔqÐqÐqÐqr¶   c                 óL  •— t           rO|                     t          j        t          j                             ‰t          ¦  «        dd¬¦  «        ¦  «         d S |                     t          j        t          j                             ‰t          ¦  «        d¬¦  «        ¦  «         d S )NT)r   Úback_compatr   )r{  rÊ  r|  rÉ  rn  rt  rç  r{  rV  s     €r´   rY  z<Trainer._load_optimizer_and_scheduler.<locals>.opt_load_hookí  s‘   ø€ Ý8ð vØ #× 3Ò 3Ý$'¤H­R¬W¯\ª\¸*ÅnÑ-UÔ-UÐ_cÐquÐ$vÑ$vÔ$vñ!"ô !"ð !"ð !"ð !"ð !$× 3Ò 3µC´H½R¼W¿\º\È*ÕVdÑ=eÔ=eÐosÐ4tÑ4tÔ4tÑ uÔ uÐ uÐ uÐ ur¶   r   ))rN  r±   ra  r¨   r9  rF  rw   rÊ  rÀ   rÉ  rn  rt  rç  r|  rM   r„   Úglobr{  rB  rT  r¯  r^  rÆ  r™  rà   r  r�   r  rÏ  Úsend_cpu_data_to_devicer÷   r`  rY  Úregister_post_step_hookrU  r¢   rZ  r}  r7  r²   r¾   )rš  r^  rL  Úcheckpoint_file_existsÚoptimizer_stateÚlr_scheduler_staterY  r»  s    `      r´   r?  z%Trainer._load_optimizer_and_schedulerŸ  sn  ø€ àÐØˆFàÔ$ð 		å˜dÔ/Õ1JÑKÔKð 5ÝÔ,°DÐ9Ñ9Ô9ð ¸_Ý,Ñ.Ô.Ð.ØÔ%×5Ò5Ýœ
¥2¤7§<¢<°
½NÑ#KÔ#KÐZ^Ð_Ñ_Ô_ñô ð ðð ð ñ ô ð ð ð ð ð ð øøøð ð ð ð õ
 $ OÑ4Ô4Ð4ØˆFõ 'Ñ(Ô(ð�DŒI•b”g—l’l :­~Ñ>Ô>ÀÑEÑFÔFÐFõ ”—’�rœwŸ|š|¨J½ÑGÔGÑHÔHð 	Ý”7—>’>¥"¤'§,¢,¨zÕ;MÑ"NÔ"NÑOÔOð	õ ”G—M’M *Ñ-Ô-ð Ýð ð ð ð å+-¬:°jÑ+AÔ+Aðñ ô ñ ô ð 	ð$ Ô*ð(�DŒI•b”g—l’l :Ð/b¸4¼9Ô;OÐ/bÐ/bÕR`Ð/bÐ/bÑcÔcÑdÔdÐdà'ð 	ð
 "ñ L	5¥b¤g§n¢nµR´W·\²\À*ÍnÑ5]Ô5]Ñ&^Ô&^ñ L	5å%Ñ'Ô'ñ J5àÔ.ð Ý,Ñ.Ô.Ð.Ý&+¤jÝœŸšØ&Ð(s¨t¬yÔ/FÐ(sÐ(sÈDÌIÔL`Ð(sÐ(sÕcqÐ(sÐ(sñô ð &+Ø%)ð'ñ 'ô '�Oð '6°kÔ&B�O�Oå,Ñ.Ô.Ð.Ý&+¤jÝœŸš ZµÑ@Ô@ÈuÐcgð'ñ 'ô '�Oõ Ô,°DÐ9Ñ9Ô9ð ¸_Ý,Ñ.Ô.Ð.Ý).¬ÝœŸš ZµÑ@Ô@ÈuÐcgð*ñ *ô *Ð&ðð ð ñ ô ð ð ð ð ð ð øøøð ð ð ð õ
 $ OÑ4Ô4Ð4åÔ*¨?¸D¼IÔ<LÑMÔMÐMÝÔ*Ð+=¸t¼yÔ?OÑPÔPÐPà”×.Ò.¨Ñ?Ô?Ð?ØÔ!×1Ò1Ð2DÑEÔEÐEÐEÐEå*Ñ,Ô,ð %Ý”w—~’~¥b¤g§l¢l°:Ð?PÑ&QÔ&QÑRÔRð vðrð rð rð rð rð rð
vð vð vð vð vð Ô&×>Ò>¸}ÑMÔMÐMÐMð
 8<´yÔ7KÈaÒ7OÐ7O 4¤9Ô#3Ð#3ÐUZ�LØÔ+ð Ý+Ø Ô,Ô2Ô>Ø Ô,Ø œNØ œJØ&ðð õ 4Ñ5Ô5ðð ð ð õ 1Ñ2Ô2Ð2Øœ×6Ò6Ý!œJÝ "¤§¢¨Z½Ñ HÔ HÐWcÐrvðñ ô ñô ð õ
 Ô,°DÐ9Ñ9Ô9ð ¸_Ý,Ñ.Ô.Ð.ØÔ%×5Ò5Ýœ
¥2¤7§<¢<°
½NÑ#KÔ#KÐZ^Ð_Ñ_Ô_ñô ð ðð ð ñ ô ð ð ð ð ð ð øøøð ð ð ð õ
 $ OÑ4Ô4Ð4Ð4Ð4ðYL	5ð L	5ð L	5ð L	5s8   ¼A B(Â(B,Â/B,ÍA	N(Î(N,Î/N,Õ=A W)×)W-×0W-c                 ó  — 	 | j         j        }n# t          $ r Y d S w xY w|€d S t          ¦   «         r¤t	          j        d¦  «         t          j        d¬¦  «        5 }t	          j        | j         j         	                    ¦   «         t          j                             |t          ¦  «        ¦  «         t          |¦  «         d d d ¦  «         n# 1 swxY w Y   | j        j        r t          ¦   «         s”t          j        d¬¦  «        5 }t#          j        | j         j         	                    ¦   «         t          j                             |t          ¦  «        ¦  «         d d d ¦  «         n# 1 swxY w Y   t          |¦  «         d S d S d S )NÚsaving_scaler_stateTr@  )rZ  Úscalerr[  r�   rÏ  r'  r9  rF  ry  rz  rn  rt  rç  ÚSCALER_NAMErM   rà   rm  rÀ   )rš  rí   rd  rL  s       r´   r+  zTrainer._save_scaler  sø  € ð	ØÔ%Ô,ˆFˆFøÝð 	ð 	ð 	ØˆFˆFð	øøøàˆ>ØˆFÝ!Ñ#Ô#ð 	5ÝŒMÐ/Ñ0Ô0Ð0ÝÔ(°Ð5Ñ5Ô5ð 5¸Ý”˜Ô(Ô/×:Ò:Ñ<Ô<½b¼g¿lºlÈ:ÕWbÑ>cÔ>cÑdÔdÐdÝ# OÑ4Ô4Ð4ð5ð 5ð 5ñ 5ô 5ð 5ð 5ð 5ð 5ð 5ð 5øøøð 5ð 5ð 5ð 5ð
 Œ9Ô ð 	1Õ)?Ñ)AÔ)Að 	1ÝÔ(°Ð5Ñ5Ô5ð h¸Ý”
˜4Ô+Ô2×=Ò=Ñ?Ô?ÅÄÇÂÈjÕZeÑAfÔAfÑgÔgÐgðhð hð hñ hô hð hð hð hð hð hð høøøð hð hð hð hå Ñ0Ô0Ð0Ð0Ð0ð	1ð 	1ð 	1ð 	1s2   ‚ �
œÁA$CÃCÃCÄAE&Å&E*Å-E*c           	      ó€  — |€dS t           j                             t           j                             |t          ¦  «        ¦  «        }|�rtt          ¦   «         rÄt          j        d¬¦  «        5 }t          ¦   «          t          j
        t           j                             |t          ¦  «        dd¬¦  «        }ddd¦  «         n# 1 swxY w Y   t          |¦  «         t          j        || j        j        ¦  «         | j        j                             |¦  «         dS t          j        d¬¦  «        5 }t          ¦   «          | j        j                             t          j
        t           j                             |t          ¦  «        d¬¦  «        ¦  «         ddd¦  «         n# 1 swxY w Y   t          |¦  «         dS dS )z If scaler state exists, load it.NTr@  rô   rº  rQ  )rn  rt  rB  rç  re  r�   r9  rF  rw   rÀ   rÉ  rM   rÏ  r]  rà   r÷   rZ  rd  rÊ  )rš  r^  r_  rL  Úscaler_states        r´   r@  zTrainer._load_scaler&  s  € àÐØˆFå!#¤§¢µ´·²¸ZÍÑ0UÔ0UÑ!VÔ!VÐà!ñ 	5õ &Ñ'Ô'ð 5ÝÔ,°DÐ9Ñ9Ô9ð ¸_Ý,Ñ.Ô.Ð.Ý#(¤:ÝœŸš ZµÑ=Ô=ÈEÐ`dð$ñ $ô $�Lðð ð ñ ô ð ð ð ð ð ð øøøð ð ð ð õ
 $ OÑ4Ô4Ð4ÝÔ*¨<¸¼Ô9IÑJÔJÐJØÔ Ô'×7Ò7¸ÑEÔEÐEÐEÐEåÔ,°DÐ9Ñ9Ô9ð ¸_Ý,Ñ.Ô.Ð.ØÔ$Ô+×;Ò;Ýœ
¥2¤7§<¢<°
½KÑ#HÔ#HÐW[Ð\Ñ\Ô\ñô ð ðð ð ñ ô ð ð ð ð ð ð øøøð ð ð ð õ
 $ OÑ4Ô4Ð4Ð4Ð4ð%	5ð 	5s&   Á-A	CÃCÃ	CÄ1A%F"Æ"F&Æ)F&c                 ó  ‡— | j         j        sdS g }g }| j        j        | j        gz   }| j        j                             ¦   «         D �]T\  Š}t          |t          ¦  «        s|g}t          ˆfd„|D ¦   «         ¦  «        �rˆfd„|D ¦   «         }t          ||¦  «        D ]Ê\  }}|                     di ¦  «        }|                     di ¦  «        }	 t          |¦  «        di |¤Ž}
|	                     ¦   «         D ]\  }}t          |
||¦  «         Œt          |t          ¦  «        r|
| _        n|                     |
¦  «         | j                             t          |
¦  «        ¦  «         ŒËt$                               d¦  «         �Œ?|                     ‰¦  «         �ŒVt)          |¦  «        dk    r1t$                               dd	                     |¦  «        › d
�¦  «         |D ]}| j                             |¦  «         ŒdS )zLIf callback states exist and were passed in, restore their states if enabledNc              3   ó8   •K  — | ]}|j         j        ‰k    V — Œd S r  ©r<  r=  ©rö   r¾  Ústored_callbacks     €r´   r
  z/Trainer._load_callback_state.<locals>.<genexpr>L  s.   øè è € ÐeÐeÀh�8Ô%Ô.°/ÒAÐeÐeÐeÐeÐeÐer¶   c                 ó4   •— g | ]}|j         j        ‰k    ¯|‘ŒS r½   rj  rk  s     €r´   rø   z0Trainer._load_callback_state.<locals>.<listcomp>N  s1   ø€ ð ð ð Ø!)À8ÔCUÔC^ÐbqÒCqÐCq�HÐCqÐCqÐCqr¶   rà   Ú
attributeszPContinuing training from checkpoint, restoring any callbacks that were passed inr   zPCheckpoint included callbacks not included in current configuration. Ignoring. (rà  rÿ  r½   )rà   Ú'restore_callback_states_from_checkpointre  rç   r†  r}  r  rM  r±   r¹   r^  Úzipr�  rË   rO  r:   r�  rÂ  r"  r#  rB  rG  rç  rf  )rš  Ú	not_foundÚnew_callbacksÚoriginal_callbacksru  Ú
duplicatesr¾  Úcallback_datarà   rn  Únew_callbackÚ	attributer$  rl  s                @r´   rC  zTrainer._load_callback_stateA  sV  ø€ àŒyÔ@ð 	ØˆFàˆ	ØˆØ!Ô2Ô<ÀÄ¸~ÑMÐØ%)¤ZÔ%B×%HÒ%HÑ%JÔ%Jð 	2ñ 	2Ñ!ˆO˜TÝ˜d¥DÑ)Ô)ð Ø�v�ÝÐeÐeÐeÐeÐRdÐeÑeÔeÑeÔeñ 2ðð ð ð Ø-?ðñ ô �
õ 03°:¸tÑ/DÔ/Dð Nð NÑ+�H˜mØ(×,Ò,¨V°RÑ8Ô8�DØ!.×!2Ò!2°<ÀÑ!DÔ!D�JØ#1¥4¨¡>¤>Ð#9Ð#9°DÐ#9Ð#9�LØ,6×,<Ò,<Ñ,>Ô,>ð @ð @Ñ(˜	 5Ý ¨i¸Ñ?Ô?Ð?Ð?Ý! (­NÑ;Ô;ð ;à'3˜œ˜à%×,Ò,¨\Ñ:Ô:Ð:àÔ)×9Ò9½$¸|Ñ:LÔ:LÑMÔMÐMÐMÝ—’ÐnÑoÔoÐoÑoà× Ò  Ñ1Ô1Ð1Ñ1Ýˆy‰>Œ>˜AÒÐÝ�NŠNØzÐcg×clÒclÐmvÑcwÔcwÐzÐzÐzñô ð ð &ð 	9ð 	9ˆHØÔ!×.Ò.¨xÑ8Ô8Ð8Ð8ð	9ð 	9r¶   é   ÚminimizerI  r`  Ún_trialsÚ	directionÚbackendr%  r-  c                 óJ  — |€t          ¦   «         }t          |¦  «        }t          |         ¦   «         }|                     ¦   «          || _        | j        €t          d¦  «        ‚|€|j        n|| _        || _	        |€t          n|| _         |j        | ||fi |¤Ž}	d| _        |	S )az  
        Launch an hyperparameter search using `optuna` or `Ray Tune` or `SigOpt`. The optimized quantity is determined
        by `compute_objective`, which defaults to a function returning the evaluation loss when no metric is provided,
        the sum of all metrics otherwise.

        <Tip warning={true}>

        To use this method, you need to have provided a `model_init` when initializing your [`Trainer`]: we need to
        reinitialize the model at each new run. This is incompatible with the `optimizers` argument, so you need to
        subclass [`Trainer`] and override the method [`~Trainer.create_optimizer_and_scheduler`] for custom
        optimizer/scheduler.

        </Tip>

        Args:
            hp_space (`Callable[["optuna.Trial"], dict[str, float]]`, *optional*):
                A function that defines the hyperparameter search space. Will default to
                [`~trainer_utils.default_hp_space_optuna`] or [`~trainer_utils.default_hp_space_ray`] or
                [`~trainer_utils.default_hp_space_sigopt`] depending on your backend.
            compute_objective (`Callable[[dict[str, float]], float]`, *optional*):
                A function computing the objective to minimize or maximize from the metrics returned by the `evaluate`
                method. Will default to [`~trainer_utils.default_compute_objective`].
            n_trials (`int`, *optional*, defaults to 100):
                The number of trial runs to test.
            direction (`str` or `list[str]`, *optional*, defaults to `"minimize"`):
                If it's single objective optimization, direction is `str`, can be `"minimize"` or `"maximize"`, you
                should pick `"minimize"` when optimizing the validation loss, `"maximize"` when optimizing one or
                several metrics. If it's multi objectives optimization, direction is `list[str]`, can be List of
                `"minimize"` and `"maximize"`, you should pick `"minimize"` when optimizing the validation loss,
                `"maximize"` when optimizing one or several metrics.
            backend (`str` or [`~training_utils.HPSearchBackend`], *optional*):
                The backend to use for hyperparameter search. Will default to optuna or Ray Tune or SigOpt, depending
                on which one is installed. If all are installed, will default to optuna.
            hp_name (`Callable[["optuna.Trial"], str]]`, *optional*):
                A function that defines the trial/run name. Will default to None.
            kwargs (`dict[str, Any]`, *optional*):
                Additional keyword arguments for each backend:

                - `optuna`: parameters from
                  [optuna.study.create_study](https://optuna.readthedocs.io/en/stable/reference/generated/optuna.study.create_study.html)
                  and also the parameters `timeout`, `n_jobs` and `gc_after_trial` from
                  [optuna.study.Study.optimize](https://optuna.readthedocs.io/en/stable/reference/generated/optuna.study.Study.html#optuna.study.Study.optimize)
                - `ray`: parameters from [tune.run](https://docs.ray.io/en/latest/tune/api_docs/execution.html#tune-run).
                  If `resources_per_trial` is not set in the `kwargs`, it defaults to 1 CPU core and 1 GPU (if available).
                  If `progress_reporter` is not set in the `kwargs`,
                  [ray.tune.CLIReporter](https://docs.ray.io/en/latest/tune/api/doc/ray.tune.CLIReporter.html) is used.
                - `sigopt`: the parameter `proxies` from
                  [sigopt.Connection.set_proxies](https://docs.sigopt.com/support/faq#how-do-i-use-sigopt-with-a-proxy).

        Returns:
            [`trainer_utils.BestRun` or `list[trainer_utils.BestRun]`]: All the information about the best run or best
            runs for multi-objective optimization. Experiment summary can be found in `run_summary` attribute for Ray
            backend.
        NzXTo use hyperparameter search, you need to pass your model through a model_init function.)r#   rT   r"   Úensure_availablerˆ  rä   r8  Údefault_hp_spacerI  r-  r\   r`  rª  )
rš  rI  r`  rz  r{  r|  r-  r¾  Úbackend_objÚbest_runs
             r´   Úhyperparameter_searchzTrainer.hyperparameter_searchh  sÅ   € ð@ ˆ?Ý/Ñ1Ô1ˆGÝ! 'Ñ*Ô*ˆÝ8¸ÔAÑCÔCˆØ×$Ò$Ñ&Ô&Ð&Ø!(ˆÔØŒ?Ð"ÝØjñô ð ð 9AÐ8H˜Ô4Ð4ÈhˆŒØˆŒØ>OÐ>WÕ!:Ð!:Ð]nˆÔà"�;”? 4¨°9ÐGÐGÀÐGÐGˆà!%ˆÔØˆr¶   r  ró  c                 ó   — | j         j        �| j         j        |d<   | j        j        dk    r@| j         j        |d<   |�/|                     t          d|| j         j        ¬¦  «        ¦  «         i |¥d| j         j        i¥}| j         j         	                    |¦  «         | j
                             | j        | j         | j        |¦  «        | _        dS )a8  
        Log `logs` on the various objects watching training.

        Subclass and override this method to inject custom behavior.

        Args:
            logs (`dict[str, float]`):
                The values to log.
            start_time (`Optional[float]`):
                The start of training.
        NrA  rï   rV  rk  )r=  r†  )r}  rA  rà   rT  rV  r‘  rf   ru  Úlog_historyr�  re  Úon_logr†  )rš  r  ró  rÉ  s       r´   ry  zTrainer.log¼  sÄ   € ð Œ:ÔÐ'Ø œJÔ,ˆD�‰MØŒ9Ô2°dÒ:Ð:Ø,0¬JÔ,LˆDÐ(Ñ)ØÐ%Ø—’�M¨'°:È$Ì*ÔJjÐkÑkÔkÑlÔlÐlà=�DÐ=˜V T¤ZÔ%;Ð<Ð=ˆØŒ
Ô×%Ò% fÑ-Ô-Ð-ØÔ,×3Ò3°D´I¸t¼zÈ4Ì<ÐY]Ñ^Ô^ˆŒˆˆr¶   ru  c                 ó`  ‡ — t          |t          ¦  «        r6 t          |¦  «        ˆ fd„|                     ¦   «         D ¦   «         ¦  «        S t          |t          t
          f¦  «        r$ t          |¦  «        ˆ fd„|D ¦   «         ¦  «        S t          |t          j        ¦  «        r‡d‰ j        j	        i}‰ j
        ret          j        |¦  «        st          j        |¦  «        r=|                     d‰ j        j        j        j                             ¦   «         i¦  «          |j        di |¤ŽS |S )z|
        Prepares one `data` before feeding it to the model, be it a tensor or a nested list/dictionary of tensors.
        c                 óB   •— i | ]\  }}|‰                      |¦  «        “ŒS r½   ©Ú_prepare_input)rö   r	  rB  rš  s      €r´   rK  z*Trainer._prepare_input.<locals>.<dictcomp>Ø  s-   ø€ ÐRÐRÐR¹T¸QÀ˜q $×"5Ò"5°aÑ"8Ô"8ÐRÐRÐRr¶   c              3   óB   •K  — | ]}‰                      |¦  «        V — Œd S r  rˆ  )rö   rB  rš  s     €r´   r
  z)Trainer._prepare_input.<locals>.<genexpr>Ú  s1   øè è € ÐCÐC¸˜d×1Ò1°!Ñ4Ô4ÐCÐCÐCÐCÐCÐCr¶   r÷   rÊ   r½   )r±   r   rË   rM  rç  r¹   rÀ   rÆ  rà   r÷   rN  Úis_floating_pointÚ
is_complexr‘  rZ  r}  rU  rF  rÊ   rÅ  )rš  ru  r¾  s   `  r´   r‰  zTrainer._prepare_inputÓ  s/  ø€ õ �d�GÑ$Ô$ð 	%Ø•4˜‘:”:ÐRÐRÐRÐRÀTÇZÂZÁ\Ä\ÐRÑRÔRÑSÔSÐSÝ˜�u¥d˜mÑ,Ô,ð 		%Ø•4˜‘:”:ÐCÐCÐCÐC¸dÐCÑCÔCÑCÔCÐCÝ˜�eœlÑ+Ô+ð 	%Ø ¤	Ô 0Ð1ˆFØÔ(ð g­eÔ.EÀdÑ.KÔ.Kð gÍuÔO_Ð`dÑOeÔOeð gð —’˜w¨Ô(8Ô(>Ô(OÔ(\×(bÒ(bÑ(dÔ(dÐeÑfÔfÐfØ�4”7Ð$Ð$˜VÐ$Ð$Ð$Øˆr¶   r›  c                 óî   — |                       |¦  «        }t          |¦  «        dk    r+t          dd                     | j        ¦  «        › d�¦  «        ‚| j        j        dk    r| j        �
| j        |d<   |S )z�
        Prepare `inputs` before feeding them to the model, converting them to tensors if they are not already and
        handling potential state.
        r   z”The batch received was empty, your model won't be able to train on it. Double-check that your training dataset contains keys expected by the model: rr  rÎ  NÚmems)r‰  rB  r%  rç  rw  rà   rL  r(  ©rš  r›  s     r´   r‰  zTrainer._prepare_inputså  sŽ   € ð
 ×$Ò$ VÑ,Ô,ˆÝˆv‰;Œ;˜!ÒÐÝðnØILÏÊÐRVÔRiÑIjÔIjðnð nð nñô ð ð Œ9Ô 1Ò$Ð$¨¬Ð)?Ø!œZˆF�6‰Nàˆr¶   c                 óÖ  — |€dS |                      ¦   «         dk    rdS |                      ¦   «         dv r´|j        d         }|dk    rdS |                      ¦   «         dk    r	|d         }n|d	         }t          j        |d¬
¦  «        }t          j        |dk    ¦  «        pt          j        |dk     ¦  «        }t          |t          j        ¦  «        r|                     ¦   «         n|S dS )a£  
        Check if an attention mask is causal (compatible with causal attention).
        Context parallelism only supports causal attention patterns. This function
        checks if the provided attention mask is compatible.

        Args:
            attention_mask (torch.Tensor): The attention mask to check

        Returns:
            bool: True if the mask is causal or compatible with causal attention
        NTr  )é   r±  r"  r   r±  )r   r   r   )Údiagonalg�íµ ÷Æ°>g     ˆÃÀF)ÚdimÚshaperÀ   ÚtriuÚallr±   rÆ  r'  )rš  r%  Úseq_lenÚmaskÚupper_triangularÚ	is_causals         r´   Ú_is_attention_mask_causalz!Trainer._is_attention_mask_causalõ  s  € ð Ð!Ø�4ð ×ÒÑÔ 1Ò$Ð$à�4Ø×ÒÑ!Ô! VÐ+Ð+ð %Ô*¨2Ô.ˆGØ˜!Š|ˆ|Ø�tð ×!Ò!Ñ#Ô# qÒ(Ð(Ø% dÔ+��à% aÔ(�õ  %œz¨$¸Ð;Ñ;Ô;Ðõ œ	Ð"2°dÒ":Ñ;Ô;Ða½u¼yÐIYÐ\`ÒI`Ñ?aÔ?aˆIÝ'1°)½U¼\Ñ'JÔ'JÐY�9—>’>Ñ#Ô#Ð#ÐPYÐYð ˆur¶   c                 ó  — t          | j        dd¦  «        ��é| j        j        j        �r×t	          |d¦  «        r-|j        j        dk    rt          d|j        j        › d�¦  «        ‚d|vr„t           	                    d¦  «         t          j        |d	                              d
¦  «        |d	         j        ¬¦  «                             |d	                              d¦  «        d¦  «        |d<   d|vrkt           	                    d¦  «         d|v rMd}t          j                             |d         d|¬¦  «        }|dd…d
d…f                              ¦   «         |d<   g }g }d	|v r0|                     |d	         ¦  «         |                     d
¦  «         d|v r0|                     |d         ¦  «         |                     d
¦  «         d|v r0|                     |d         ¦  «         |                     d
¦  «         d|v r—t          | dd¦  «        s3|d         }|                      |¦  «        st          d¦  «        ‚d| _        | j        rL|d         }|                     ¦   «         dk    r+|                     |¦  «         |                     d
¦  «         n	 d|v r8|d         �0|                     |d         ¦  «         |                     d
¦  «         t/          | j        j        ||t3          |¦  «        ¬¦  «        |fS t4          j        |fS )as  
        Prepare inputs for context parallelism by setting up buffers and validation.

        Args:
            model: The model being trained
            inputs: Input tensors to prepare

        Returns:
            tuple: (context_manager, prepared_inputs) where context_manager is either
                   the context parallelism wrapper or a no-op context
        Úparallelism_configNr„  ÚsdpazIContext parallelism is supported only with SDPA attention, you are using rÎ  Úposition_idsz9Position IDs not found in the inputs, generating manuallyr8  r   r  r   r"  Úshift_labelsz7Shift labels not found in the inputs, shifting manuallyÚlabelséœÿÿÿ)r   r   )r$  r%  Ú_attn_mask_causal_checkedFzëContext parallelism only supports causal attention masks. The provided attention_mask is not causal. Please ensure your data uses causal masking (lower triangular) or remove the attention_mask to use the model's default causal masking.Tr  )ÚbuffersÚbuffer_seq_dimsÚno_restore_buffers)r>  rZ  r�  Ú
cp_enabledrF  r„  Ú_attn_implementationr%  r"  rœ  rÀ   ÚarangeÚsizer÷   Úexpandr   Ú
functionalÚpadÚ
contiguousr�  r›  r£  r“  r   Úmaybe_context_parallelr@  rÂ   rÃ   )rš  r²   r›  Ú_ignore_indexr¡  r¤  r¥  r%  s           r´   Ú _prepare_context_parallel_inputsz(Trainer._prepare_context_parallel_inputs   sw  € õ �DÔ$Ð&:¸DÑAÔAÑMØÔ Ô3Ô>ñ Nõ �u˜hÑ'Ô'ð Ø”<Ô4¸Ò>Ð>Ý$ð IÐdiÔdpô  eFð  Ið  Ið  Iñô ð ð  VÐ+Ð+Ý×#Ò#Ð$_Ñ`Ô`Ð`Ý).¬Ø˜;Ô'×,Ò,¨QÑ/Ô/¸¸{Ô8KÔ8Rð*ñ *ô *ç’&˜ Ô,×1Ò1°!Ñ4Ô4°bÑ9Ô9ð �~Ñ&ð  VÐ+Ð+Ý×#Ò#Ð$]Ñ^Ô^Ð^Ø˜vÐ%Ð%Ø$(�MÝœ]×.Ò.¨v°hÔ/?ÀÈ}Ð.Ñ]Ô]�FØ-3°A°A°A°q°r°r°E¬]×-EÒ-EÑ-GÔ-G�F˜>Ñ*àˆGØ ˆOà˜fÐ$Ð$Ø—’˜v kÔ2Ñ3Ô3Ð3Ø×&Ò& qÑ)Ô)Ð)Ø˜6Ð!Ð!Ø—’˜v hÔ/Ñ0Ô0Ð0Ø×&Ò& qÑ)Ô)Ð)Ø Ð'Ð'Ø—’˜v nÔ5Ñ6Ô6Ð6Ø×&Ò& qÑ)Ô)Ð)à 6Ð)Ð)å˜tÐ%@À%ÑHÔHð :ð &,Ð,<Ô%=�NØ×9Ò9¸.ÑIÔIð Ý(ðfñô ð ð 6:�DÔ2ØÔ1ð à%+Ð,<Ô%=�NØ%×)Ò)Ñ+Ô+¨qÒ0Ð0ØŸš ~Ñ6Ô6Ð6Ø'×.Ò.¨qÑ1Ô1Ð1Ð1ð à Ð'Ð'¨F°>Ô,BÐ,NØ—’˜v nÔ5Ñ6Ô6Ð6Ø×&Ò& qÑ)Ô)Ð)åØÔ Ô7ØØ /Ý#& w¡<¤<ð	ñ ô ð
 ðð õ Ô% vÐ-Ð-r¶   c                 ó²   — t          j        ¦   «         }|                      ¦   «         }t          |t           j        ¦  «        s|                     |¦  «         |S )zF
        A helper wrapper to group together context managers.
        )rÂ   Ú	ExitStackÚautocast_smart_context_managerr±   rÃ   Úenter_context)rš  Ú	ctx_stackÚautocast_ctxs      r´   Úcompute_loss_context_managerz$Trainer.compute_loss_context_manageru  sR   € õ Ô(Ñ*Ô*ˆ	à×:Ò:Ñ<Ô<ˆÝ˜,­
Ô(>Ñ?Ô?ð 	2Ø×#Ò# LÑ1Ô1Ð1àÐr¶   r�  c                 ót   — | j         rt          j        d|| j        ¬¦  «        }nt	          j        ¦   «         }|S )z©
        A helper wrapper that creates an appropriate context manager for `autocast` while feeding it the desired
        arguments, depending on the situation.
        rô   )Údevice_typer�  rÊ   )ry  rÀ   r‹  r�  rÂ   rÃ   )rš  r�  Úctx_managers      r´   r´  z&Trainer.autocast_smart_context_manager�  s?   € ð
 Ôð 	3Ýœ.°UÈ-Ð_cÔ_mÐnÑnÔnˆKˆKå$Ô0Ñ2Ô2ˆKàÐr¶   r™  c                 ó¨  — |                       ||¦  «        \  }} |¦   «         5  |                     ¦   «          t          | j        d¦  «        r2t	          | j        j        ¦  «        r| j                             ¦   «          |                      |¦  «        }t          ¦   «         rjt          ||| j        j	        ¦  «        }| 
                    ¦   «                              ¦   «                              | j        j        ¦  «        cddd¦  «         S |                      ¦   «         5  |                      |||¬¦  «        }ddd¦  «         n# 1 swxY w Y   ~| j        j        ��F| j        j        | j        j        z  dk    �r(t'          ¦   «         rt(          j                             ¦   «          nût/          ¦   «         rt(          j                             ¦   «          nÎt3          ¦   «         rt(          j                             ¦   «          n¡t7          ¦   «         rt(          j                             ¦   «          ntt;          ¦   «         rt(          j                             ¦   «          nGt?          ¦   «         rt@           !                    d¦  «         nt(          j"                             ¦   «          i }| j        j#        tH          j%        tH          j&        fv r|  '                    ¦   «         |d<   | j        j(        dk    r| )                    ¦   «         }| j*        rNddl+m,} | -                    || j        ¦  «        5 }	|	 .                    ¦   «          ddd¦  «         n# 1 swxY w Y   nL| j/        r|€| j0        €
|| j1        z  }| j2        j3        th          j5        k    rd	|d
<    | j2        j.        |fi |¤Ž |                     ¦   «         cddd¦  «         S # 1 swxY w Y   dS )aq  
        Perform a training step on a batch of inputs.

        Subclass and override to inject custom behavior.

        Args:
            model (`nn.Module`):
                The model to train.
            inputs (`dict[str, Union[torch.Tensor, Any]]`):
                The inputs and targets of the model.

                The dictionary will be unpacked before being fed to the model. Most models expect the targets under the
                argument `labels`. Check your model's documentation for all accepted arguments.

        Return:
            `torch.Tensor`: The tensor with training loss on this batch.
        rk  N©r™  r   zW`torch_empty_cache_steps` is set but HPU device/backend does not support empty_cache().rá  r   r­  FÚscale_wrt_gas)6r±  rk  rF  r`  rp  r‰  r„   r–   rà   rú  Úreduce_meanÚdetachrÅ  r÷   r¸  Úcompute_lossÚtorch_empty_cache_stepsr}  ru  rŽ   rÀ   ÚxpuÚempty_cacher‡   r  r‰   r  r‹   r  rˆ   Úmpsr†   r"  rG  r  rR  rg   r  r  rÖ   r›  rü  rx  r  r®  Ú
scale_lossÚbackwardr\  rå   rP  rZ  rY  r    rZ  )
rš  r²   r›  r™  Ú
cp_contextÚloss_mbrÓ  r¾  r®  Úscaled_losss
             r´   r\  zTrainer.training_step�  sœ  € ð2 "×BÒBÀ5È&ÑQÔQÑˆ
�Fð ˆZ‰\Œ\ð @	!ð @	!Ø�KŠK‰MŒMˆMÝ�t”~ wÑ/Ô/ð 'µH¸T¼^Ô=QÑ4RÔ4Rð 'Ø”×$Ò$Ñ&Ô&Ð&à×)Ò)¨&Ñ1Ô1ˆFÝ&Ñ(Ô(ð KÝ.¨u°f¸d¼iÔ>cÑdÔd�Ø×*Ò*Ñ,Ô,×3Ò3Ñ5Ô5×8Ò8¸¼Ô9IÑJÔJð@	!ð @	!ð @	!ð @	!ñ @	!ô @	!ð @	!ð @	!ð ×2Ò2Ñ4Ô4ð _ð _Ø×(Ò(¨°ÐK]Ð(Ñ^Ô^�ð_ð _ð _ñ _ô _ð _ð _ð _ð _ð _ð _øøøð _ð _ð _ð _ð à”	Ô1Ñ=Ø”JÔ*¨T¬YÔ-NÑNÐRSÒSÑSå)Ñ+Ô+ð -Ý”I×)Ò)Ñ+Ô+Ð+Ð+Ý+Ñ-Ô-ð -Ý”I×)Ò)Ñ+Ô+Ð+Ð+Ý,Ñ.Ô.ð -Ý”J×*Ò*Ñ,Ô,Ð,Ð,Ý+Ñ-Ô-ð 	-Ý”I×)Ò)Ñ+Ô+Ð+Ð+Ý+Ñ-Ô-ð -Ý”I×)Ò)Ñ+Ô+Ð+Ð+Ý+Ñ-Ô-ð -Ý—N’NØqñô ð ð õ ”J×*Ò*Ñ,Ô,Ð,àˆFð ŒyŒ¥>Ô#6½Ô8NÐ"OÐOÐOØ*.×*AÒ*AÑ*CÔ*C��Ñ'àŒyŒ Ò"Ð"Ø—y’y‘{”{�àŒ}ð :Ø$Ð$Ð$Ð$Ð$Ð$à—^’^ D¨$¬.Ñ9Ô9ð +¸[Ø×(Ò(Ñ*Ô*Ð*ð+ð +ð +ñ +ô +ð +ð +ð +ð +ð +ð +øøøð +ð +ð +ð +øð
 Ô6ðKØ:LÐ:TØÔ,Ð4à $Ô"JÑJ�Dð Ô#Ô4½Ô8QÒQÐQØ.3�F˜?Ñ+à)�Ô Ô)¨$Ð9Ð9°&Ð9Ð9Ð9à—;’;‘=”=ðA@	!ð @	!ð @	!ð @	!ñ @	!ô @	!ð @	!ð @	!ð @	!ð @	!ð @	!ð @	!øøøð @	!ð @	!ð @	!ð @	!ð @	!ð @	!sb   ¤COÄOÄ!EÄ:OÅE
	Å
OÅE
	ÅG OÌ.MÍOÍM	ÍOÍM	ÍA#OÏOÏOÚreturn_outputsc           	      óx  — | j         €| j        �d|v r|                     d¦  «        }nd}| j        ri }|�||d<   i |¥|¥} |di |¤Ž}| j        j        dk    r|| j        j                 | _        | j        �6|€t                               d¦  «         |                      |||¬¦  «        }�n@|� | j	         
                    |¦  «        }	t          |	¦  «        r|	j        j                             ¦   «         n|	                     ¦   «         }
|
t          j        ¦   «         v r|                       ||d¬¦  «        }nµ|                       ||¦  «        }nžt#          |t$          ¦  «        rdd	|vr`t'          d
d                     |                     ¦   «         ¦  «        › dd                     |                     ¦   «         ¦  «        › d�¦  «        ‚t#          |t$          ¦  «        r|d	         n|d         }| j        j        r;| j        s| j        r-|�+|| j        j        dk    r| j	        j        n| j        j        z  }|r||fn|S )aÿ  
        How the loss is computed by Trainer. By default, all models return the loss in the first element.

        Args:
            model (`nn.Module`):
                The model to compute the loss for.
            inputs (`dict[str, Union[torch.Tensor, Any]]`):
                The input data for the model.
            return_outputs (`bool`, *optional*, defaults to `False`):
                Whether to return the model outputs along with the loss.
            num_items_in_batch (Optional[torch.Tensor], *optional*):
                The number of items in the batch. If num_items_in_batch is not passed,

        Returns:
            The loss of the model along with its output if return_outputs was set to True

        Subclass and override for custom behavior. If you are not using `num_items_in_batch` when computing your loss,
        make sure to overwrite `self.model_accepts_loss_kwargs` to `False`. Otherwise, the loss calculating might be slightly inaccurate when performing gradient accumulation.
        Nr¡  r™  r   z|Trainer: `compute_loss_func` is defined but `labels=None`. Your custom loss function will still be called with labels=None. r½  T)r   rÓ  zJThe model did not return a loss from the inputs, only the following keys: rr  z,. For reference, the inputs it received are rÎ  r   r½   )rƒ  rå   rQ  r\  rà   rL  r(  r"  rG  rZ  r,   rµ   r  r²   Ú	_get_namer-   rA  r±   rå  r%  rç  rÚ  Úaverage_tokens_across_devicesr›  Únum_processes)rš  r²   r›  rË  r™  r¡  r¾  ÚoutputsrÓ  r¢  rñ  s              r´   rÁ  zTrainer.compute_lossë  s«  € ð4 ÔÐ+¨tÔ/EÐ/QÐW_ÐciÐWiÐWiØ—Z’Z Ñ)Ô)ˆFˆFàˆFØÔ)ð 	*ØˆFØ!Ð-Ø/A�Ð+Ñ,Ø)˜Ð) &Ð)ˆFØ�%�/�/˜&�/�/ˆð Œ9Ô 1Ò$Ð$Ø  ¤Ô!5Ô6ˆDŒJð Ô!Ð-Øˆ~Ý—’ðXñô ð ð ×)Ò)ØØØ#5ð *ñ ô ˆD‰Dð ÐØ"Ô.×;Ò;¸EÑBÔBˆOõ " /Ñ2Ô2ð1�Ô*Ô0×:Ò:Ñ<Ô<Ð<à$×.Ò.Ñ0Ô0ð ð
 Õ>ÔEÑGÔGÐGÐGØ×*Ò*¨7°FÈÐ*ÑNÔN��à×*Ò*¨7°FÑ;Ô;��å˜'¥4Ñ(Ô(ð ¨V¸7Ð-BÐ-BÝ ðxØ—x’x §¢¡¤Ñ/Ô/ðxð xØ]`×]eÒ]eÐfl×fqÒfqÑfsÔfsÑ]tÔ]tðxð xð xñô ð õ
 '1°½$Ñ&?Ô&?ÐO�7˜6”?�?ÀWÈQÄZˆDð ŒIÔ3ð	`àÔ/ð	`à37Ô3Ið	`ð #Ð.à°d´i´oÈÒ6JÐ6J�DÔ$Ô2Ð2ÐPTÔPYÔP_Ñ_ˆDà"0Ð:��gˆˆ°dÐ:r¶   c                 ó"   — | j         j        dk    S )z 
        Whether or not this process is the local (e.g., on one machine if training in a distributed fashion on several
        machines) main process.
        r   )rà   Úlocal_process_indexr°  s    r´   r  zTrainer.is_local_process_zero>  s   € ð
 ŒyÔ,°Ò1Ð1r¶   c                 ól   — t          ¦   «         rt          j        ¦   «         dk    S | j        j        dk    S )z¿
        Whether or not this process is the global main process (when training in a distributed fashion on several
        machines, this is only going to be `True` for one process).
        r   )r„   r|  r  rà   r  r°  s    r´   r  zTrainer.is_world_process_zeroE  s2   € õ #Ñ$Ô$ð 	0Ý”8‘:”: ’?Ð"à”9Ô*¨aÒ/Ð/r¶   rí   rs  c                 óÐ  — |€| j         j        }t          ¦   «         r|                      |¦  «         �nt	          ¦   «         ršt          j        |d¬¦  «         | j                             ¦   «         }| j         j	        r|  
                    ||¬¦  «         t          r?t          t
          j                             |d¦  «        ¦  «                             ¦   «          �n×t!          | j        dd¦  «        �#| j        j        r|  
                    |¦  «         �nžt!          | j        dd¦  «        x}�|d	k    r|  
                    |¦  «         �ni| j        rhd
t+          | j        j        j        j        ¦  «        v rB| j                             | j        ¦  «        }| j         j	        r|  
                    ||¬¦  «         nú| j        rÒ	 | j                             | j        ¦  «        }| j         j	        r|  
                    ||¬¦  «         n¯# t8          $ r� t:                               d¦  «         | j         j	        r|  
                    |i ¬¦  «         t?          | j         j	        |t@          tB          g¦  «         | j         "                    |¦  «         Y n%w xY w| j         j	        r|  
                    |¦  «         | j         j#        r%|s%|  #                    d| j         j$        ¬¦  «         dS dS dS )z…
        Will save the model, so you can reload it using `from_pretrained()`.

        Will only save from the main process.
        NTr  )rz  r¶  r�  r
  r   r   ÚFULL_STATE_DICTz| stage3_gather_16bit_weights_on_model_save=false. Saving the full checkpoint instead, use zero_to_fp32.py to recover weightsz
Model save)Úcommit_messageÚrevision)%rà   rí   r�   Ú	_save_tpur„   rn  ro  rY  rz  rm  Ú_saver{  r   rt  rç  Útouchr>  rZ  Úshould_save_modelr²   rU  r%  r}  r7  Ústate_dict_typeÚget_state_dictrN  r.  r%  r"  rG  rN   rr   rp   rJ  rk  Úhub_revision)rš  rí   rs  rz  Útp_sizes        r´   rv  zTrainer.save_modelQ  sò  € ð ÐØœÔ-ˆJå!Ñ#Ô#ð (	#Ø�NŠN˜:Ñ&Ô&Ð&Ñ&Ý$Ñ&Ô&ð &	#åŒK˜
¨TÐ2Ñ2Ô2Ð2ØÔ+×6Ò6Ñ8Ô8ˆJØŒyÔ$ð >Ø—
’
˜:°*�
Ñ=Ô=Ð=Ý(ð Jå•R”W—\’\ *Ð.?Ñ@Ô@ÑAÔA×GÒGÑIÔIÐIùå�TÔ%Ð';¸TÑBÔBÐNØÔÔ1ð 'Ø—
’
˜:Ñ&Ô&Ð&ùå  ¤¨Z¸Ñ;Ô;Ð;ˆgÐHÈWÐWXÊ[È[Ø�JŠJ�zÑ"Ô"Ð"Ñ"ØÔ!ð 	#Ø ¥C¨Ô(8Ô(>Ô(JÔ(ZÑ$[Ô$[Ð[Ð[Ø!Ô-×<Ò<¸T¼ZÑHÔH�
Ø”9Ô(ð BØ—J’J˜z°j�JÑAÔAÐAøØÔ&ð 	#ð?Ø!Ô-×<Ò<¸T¼^ÑLÔL�
Ø”9Ô(ð BØ—J’J˜z°j�JÑAÔAÐAøøÝð 	?ð 	?ð 	?Ý—’ð:ñô ð ð ”9Ô(ð :Ø—J’J˜z°b�JÑ9Ô9Ð9å'¨¬	Ô(=¸zÍLÕZkÐKlÑmÔmÐmØÔ"×2Ò2°:Ñ>Ô>Ð>Ð>Ð>ð	?øøøð ŒYÔ"ð 	#Ø�JŠJ�zÑ"Ô"Ð"ð Œ9Ô ð 	[¨ð 	[Ø×Ò¨LÀ4Ä9ÔCYÐÑZÔZÐZÐZÐZð	[ð 	[ð 	[ð 	[s   ÇAH ÈBJÊJc           
      óD  — |�|n| j         j        }t                               d|› �¦  «         | j        }t          j        ¦   «          t          j        d¬¦  «        rSt          j	        |d¬¦  «         t          j        | j         t          j                             |t          ¦  «        ¦  «         t          f}t          j        d¦  «         | j        �r˜|                     ¦   «         |                     ¦   «         dœ}t          j                             |d| j         j        › d	| j         j        › d
t,          › �¦  «        }t          j        ||d¬¦  «         t          j        d¦  «         | j         j        rôddlm}  |t          j                             |d¦  «        dt,          › �d¬¦  «        \  }}|j        j        }| j                             |¦  «        }	t;          |	|¦  «        r/|	                     ||t
          j        | j         j        ¬¦  «         �n×t                               d¦  «         t          j        |t          j                             |t,          ¦  «        ¦  «         �nƒt;          ||¦  «        �st;          | j                             |¦  «        |¦  «        ru| j                             |¦  «                             || j         j        t          j         |                     ¦   «         ¦  «        t
          j        | j         j        ¬¦  «         nÕt                               d¦  «         t          j         |                     ¦   «         ¦  «        }
t          j        |
t          j                             |t,          ¦  «        ¦  «         n\|                     || j         j        t
          j        | j         j        t          j         |                     ¦   «         ¦  «        ¬¦  «         | j!        �(| j         j        r| j!                             |¦  «         d S d S d S )NúSaving model checkpoint to F)ÚlocalTr  Úsaving_checkpoint)r²   r<  r  r=  r}  r>  Úsave_full_checkpointsr   )Ú%consolidate_sharded_model_checkpointsrß  zrank*-of-*-)Úckpt_prefixÚckpt_suffixrv  )rz  Úsave_functionÚsafe_serializationúETrainer.model is not a `PreTrainedModel`, only saving its state dict.)Úis_main_processrz  rè  ré  )rë  rè  ré  rz  )"rà   rí   r"  r#  r²   rÏ  rÐ  Úis_master_ordinalrn  ro  rÀ   ry  rt  rç  ÚTRAINING_ARGS_NAMEru   r'  r™  rz  rE  r  r  rr   rm  râ  rå  ra  rZ  r,   r±   Úsave_pretrainedrË  Ú_maybe_convert_to_cpurÜ   )rš  rí   r²   Úsupported_classesÚckptÚ	ckpt_pathrå  Úfull_state_dictr|  r¢  rz  s              r´   rØ  zTrainer._save_tpu‰  sÑ  € Ø#-Ð#9�Z�Z¸t¼yÔ?Sˆ
å�ŠÐ>°*Ð>Ð>Ñ?Ô?Ð?Ø”
ˆÝ
Œ‰ŒˆåÔ eÐ,Ñ,Ô,ð 	PÝŒK˜
¨TÐ2Ñ2Ô2Ð2ÝŒJ�t”y¥"¤'§,¢,¨zÕ;MÑ"NÔ"NÑOÔOÐOõ ,Ð-ÐÝ
ŒÐ)Ñ*Ô*Ð*ØÔ&ñ 5	à×)Ò)Ñ+Ô+Ø"'×":Ò":Ñ"<Ô"<ðð ˆDõ œŸšØÐe 4¤9Ô#:ÐeÐeÀÄ	Ô@TÐeÐeÕWcÐeÐeñô ˆIõ ŒG�D˜)°Ð7Ñ7Ô7Ð7åŒMÐ1Ñ2Ô2Ð2àŒyÔ$ð UØ\Ð\Ð\Ð\Ð\Ð\à%JÐ%JÝ "¤§¢¨Z¸Ñ <Ô <Ø <­lÐ <Ð <Ø$ð&ñ &ô &Ñ"� ð
 œÔ+�Ø"&Ô"2×"?Ò"?ÀÑ"FÔ"F�Ý˜oÐ/@ÑAÔAð 	UØ#×3Ò3Ø"Ø#2Ý&(¤gØ+/¬9Ô+Eð	 4ñ ô ð ñ õ —K’KÐ gÑhÔhÐhÝ”G˜O­R¬W¯\ª\¸*ÅlÑ-SÔ-SÑTÔTÐTùÝ˜EÐ#4Ñ5Ô5ñ 	Ý˜$Ô*×7Ò7¸Ñ>Ô>Ð@QÑRÔRð LØÔ ×-Ò-¨eÑ4Ô4×DÒDØØ$(¤IÔ$9Ý!Ô7¸×8HÒ8HÑ8JÔ8JÑKÔKÝ"$¤'Ø'+¤yÔ'Að Eñ ô ð ð õ —’ÐcÑdÔdÐdÝÔ5°e×6FÒ6FÑ6HÔ6HÑIÔI�
Ý”˜
¥B¤G§L¢L°½\Ñ$JÔ$JÑKÔKÐKÐKà×!Ò!ØØ $¤	Ô 5Ý œgØ#'¤9Ô#=ÝÔ3°E×4DÒ4DÑ4FÔ4FÑGÔGð "ñ ô ð ð Ô Ð,°´Ô1FÐ,ØÔ!×1Ò1°*Ñ=Ô=Ð=Ð=Ð=ð -Ð,Ð,Ð,r¶   c                 ó4  — |�|n| j         j        }t          j        |d¬¦  «         t                               d|› �¦  «         t          ¦   «         st          fnt          t          f}t          | j
        |¦  «        �s3|€| j
                             ¦   «         }t          | j                             | j
        d¬¦  «        |¦  «        rB| j                             | j
        d¬¦  «                             ||| j         j        ¬¦  «         nÎt                               d¦  «         | j         j        rHt           j                             |t          j                             |t*          ¦  «        dd	i¬
¦  «         n`t#          j        |t          j                             |t.          ¦  «        ¦  «         n'| j
                             ||| j         j        ¬¦  «         | j        �| j                             |¦  «         na| j        �Zt5          | j        d¦  «        rE| j        j        �9t                               d¦  «         | j        j                             |¦  «         t#          j        | j         t          j                             |t8          ¦  «        ¦  «         d S )NTr  rá  Fr«  )rz  ré  rê  rê  Úpt)r°   rÛ   zWSaving Trainer.data_collator.tokenizer by default as Trainer.processing_class is `None`)rà   rí   rn  ro  r"  r#  r‚   r*   rš   r±   r²   rz  rZ  r,   rî  rË  rÌ  rÀ   Ú	save_filert  rç  rp   ry  rr   rÜ   rá   rF  rÛ   rí  )rš  rí   rz  rð  s       r´   rÙ  zTrainer._saveÑ  sk  € à#-Ð#9�Z�Z¸t¼yÔ?Sˆ
Ý
Œ�J¨Ð.Ñ.Ô.Ð.Ý�ŠÐ>°*Ð>Ð>Ñ?Ô?Ð?å6GÑ6IÔ6IÐk�_Ð.Ð.ÕP_ÕajÐOkÐõ ˜$œ*Ð&7Ñ8Ô8ñ 	ØÐ!Ø!œZ×2Ò2Ñ4Ô4�
å˜$Ô*×7Ò7¸¼
ÐW\Ð7Ñ]Ô]Ð_pÑqÔqð SØÔ ×-Ò-¨d¬jÈUÐ-ÑSÔS×cÒcØ¨:È$Ì)ÔJdð dñ ô ð ð õ —’ÐcÑdÔdÐdØ”9Ô-ð SÝÔ%×/Ò/Ø"¥B¤G§L¢L°Õ=NÑ$OÔ$OÐ[cÐeiÐZjð 0ñ ô ð ð õ ”J˜z­2¬7¯<ª<¸
ÅLÑ+QÔ+QÑRÔRÐRÐRàŒJ×&Ò&Ø zÀdÄiÔF`ð 'ñ ô ð ð Ô Ð,ØÔ!×1Ò1°*Ñ=Ô=Ð=Ð=àÔÐ*Ý˜Ô*¨KÑ8Ô8ð +àÔ"Ô,Ð8å�KŠKÐqÑrÔrÐrØÔÔ(×8Ò8¸ÑDÔDÐDõ 	Œ
�4”9�bœgŸlšl¨:Õ7IÑJÔJÑKÔKÐKÐKÐKr¶   c                 ó>  — | j         j        t          j        k    ra| j        xj        t          | j        g| j         j        ¬¦  «         	                    ¦   «          
                    ¦   «         z  c_        d| _        d S | j        xj        | j        z  c_        d| _        d S )Nr  r   )rà   rO  rh   rP  r}  r+  rC   r‡  r÷   rW  r'  r°  s    r´   rx  zTrainer.store_flosü  s•   € àŒ9Ô"¥lÔ&>Ò>Ð>ØŒJÐ!Ô!Ý-¨tÔ/@Ð.AÈ$Ì)ÔJZÐ[Ñ[Ô[×_Ò_ÑaÔa×fÒfÑhÔhñÐ!Ô!ð !"ˆDÔÐÐàŒJÐ!Ô! TÔ%6Ñ6Ð!Ô!Ø !ˆDÔÐÐr¶   c                 ó  — g }d„ t          |¦  «                             |› d�¦  «        D ¦   «         }|D ]¤}|r5|                     t          j                             |¦  «        |f¦  «         Œ9t          j        d|› d�|¦  «        }|�P|                     ¦   «         �<|                     t          |                     ¦   «         d         ¦  «        |f¦  «         Œ¥t          |¦  «        }|rbt          |¦  «        dk    rO|d         d         |d         d         z
  }	|	dk     r,t          j        d	¦  «         |                      d
||¬¦  «        S d„ |D ¦   «         }| j        j        �¡t#          t          | j        j        ¦  «        ¦  «        |v ry|                     t#          t          | j        j        ¦  «        ¦  «        ¦  «        }
t'          |
t          |¦  «        dz
  ¦  «        D ]}||dz            ||         c||<   ||dz   <   Œ |S )Nc                 ój   — g | ]0}t           j                             |¦  «        ¯!t          |¦  «        ‘Œ1S r½   )rn  rt  r¯  r%  )rö   Úxs     r´   rø   z/Trainer._sorted_checkpoints.<locals>.<listcomp>  s6   € ÐpÐpÐp qÕ_aÔ_f×_lÒ_lÐmnÑ_oÔ_oÐp�C ™FœFÐpÐpÐpr¶   ú-*z.*z	-([0-9]+)r   r   r"  rÇ  zPmtime may not be reliable on this filesystem, falling back to numerical orderingF)r.  rí   Úcheckpoint_prefixc                 ó   — g | ]
}|d          ‘ŒS )r   r½   )rö   r^  s     r´   rø   z/Trainer._sorted_checkpoints.<locals>.<listcomp>   s   € ÐQÐQÐQ°
˜j¨œmÐQÐQÐQr¶   r  )r   r\  r�  rn  rt  ÚgetmtimeÚreÚmatchÚgroupsrÏ  ÚsortedrB  r9  r:  r{  r}  rv  r%  Úindexr˜  )rš  rí   rü  r.  Úordering_and_checkpoint_pathÚglob_checkpointsrt  Úregex_matchr¤  Ú
mtime_diffÚbest_model_indexrš  s               r´   r{  zTrainer._sorted_checkpoints  sK  € ð (*Ð$àpÐp­D°Ñ,<Ô,<×,AÒ,AÐEVÐBZÐBZÐBZÑ,[Ô,[ÐpÑpÔpÐà$ð 	^ð 	^ˆDØð ^Ø,×3Ò3µR´W×5EÒ5EÀdÑ5KÔ5KÈTÐ4RÑSÔSÐSÐSå œhÐ'HÐ,=Ð'HÐ'HÐ'HÈ$ÑOÔO�ØÐ*¨{×/AÒ/AÑ/CÔ/CÐ/OØ0×7Ò7½¸[×=OÒ=OÑ=QÔ=QÐRSÔ=TÑ9UÔ9UÐW[Ð8\Ñ]Ô]Ð]øå#Ð$@ÑAÔAÐð ð 	�Ð9Ñ:Ô:¸QÒ>Ð>Ø+¨BÔ/°Ô2Ð5GÈÔ5JÈ1Ô5MÑMˆJØ˜CÒÐÝ”ÐpÑqÔqÐqØ×/Ò/Ø#°
ÐN_ð 0ñ ô ð ð RÐQÐ>PÐQÑQÔQÐð ŒJÔ,Ð8Ý•D˜œÔ9Ñ:Ô:Ñ;Ô;Ð?QÐQÐQà1×7Ò7½½DÀÄÔAaÑ<bÔ<bÑ8cÔ8cÑdÔdÐÝÐ+­SÐ1CÑ-DÔ-DÀqÑ-HÑIÔIð tð t�ØCUÐVWÐZ[ÑV[ÔC\Ð^pÐqrÔ^sÐ@Ð" 1Ñ%Ð'9¸!¸a¹%Ñ'@Ð'@Ø!Ð!r¶   c                 óô  — | j         j        �| j         j        dk    rd S |                      ||¬¦  «        }t          |¦  «        | j         j        k    rd S | j         j        }| j        j        �(| j         j        dk    r|d         | j        j        k    rd}t          dt          |¦  «        |z
  ¦  «        }|d |…         }|D ]6}t                               d|› d�¦  «         t          j
        |d¬	¦  «         Œ7d S )
Nr   r-  r   r"  r  r/  r0  Tr1  )rà   r|  r{  rB  r}  rv  rš  r"  r#  r~  r  )rš  r.  rí   r¤  r|  Únumber_of_checkpoints_to_deleteÚcheckpoints_to_be_deletedr^  s           r´   r.  zTrainer._rotate_checkpoints,  s  € ØŒ9Ô%Ð-°´Ô1KÈqÒ1PÐ1PØˆFð "×5Ò5À	ÐV`Ð5ÑaÔaÐÝÐ!Ñ"Ô" d¤iÔ&@Ò@Ð@ØˆFð  œ9Ô5ÐàŒJÔ,Ð8Ø”	Ô*¨aÒ/Ð/Ø" 2Ô&¨$¬*Ô*JÒJÐJà Ðå*-¨aµÐ5GÑ1HÔ1HÐK[Ñ1[Ñ*\Ô*\Ð'Ø$6Ð7WÐ8WÐ7WÔ$XÐ!Ø3ð 	:ð 	:ˆJÝ�KŠKÐ`°jÐ`Ð`Ð`ÑaÔaÐaÝŒM˜*°DÐ9Ñ9Ô9Ð9Ð9ð	:ð 	:r¶   r#  rò  Úmetric_key_prefixc                 ó®  — |du}|r|n| j         }t          |t          ¦  «        rTi }|                     ¦   «         D ];\  }}|                      |r|n|||› d|› �¬¦  «        }|                     |¦  «         Œ<|S | j                             ¦   «          |                      |¦  «        }	| j	        rt          |	¦  «        }	t          j        ¦   «         }
| j        j        r| j        n| j        } ||	d| j        €dnd||¬¦  «        }| j        j        | j        j        z  }|› d�|j        v r|
|j        |› d�         z  }
|› d�|j        v r|
|j        |› d�         z  }
|j                             t)          ||
|j        t-          j        |j        |z  ¦  «        ¬	¦  «        ¦  «         |                      |j        ¦  «         t2          j        | j        j        v r%t9          j        t=          j        ¦   «         ¦  «         | j          !                    | j        | j"        | j#        |j        ¦  «        | _#        | j         $                    |j        ¦  «         |j        S )
aÛ  
        Run evaluation and returns metrics.

        The calling script will be responsible for providing a method to compute metrics, as they are task-dependent
        (pass it to the init `compute_metrics` argument).

        You can also subclass and override this method to inject custom behavior.

        Args:
            eval_dataset (Union[`Dataset`, dict[str, `Dataset`]), *optional*):
                Pass a dataset if you wish to override `self.eval_dataset`. If it is a [`~datasets.Dataset`], columns
                not accepted by the `model.forward()` method are automatically removed. If it is a dictionary, it will
                evaluate on each dataset, prepending the dictionary key to the metric name. Datasets must implement the
                `__len__` method.

                <Tip>

                If you pass a dictionary with names of datasets as keys and datasets as values, evaluate will run
                separate evaluations on each dataset. This can be useful to monitor how training affects other
                datasets or simply to get a more fine-grained evaluation.
                When used with `load_best_model_at_end`, make sure `metric_for_best_model` references exactly one
                of the datasets. If you, for example, pass in `{"data1": data1, "data2": data2}` for two datasets
                `data1` and `data2`, you could specify `metric_for_best_model="eval_data1_loss"` for using the
                loss on `data1` and `metric_for_best_model="eval_data2_loss"` for the loss on `data2`.

                </Tip>

            ignore_keys (`list[str]`, *optional*):
                A list of keys in the output of your model (if it is a dictionary) that should be ignored when
                gathering predictions.
            metric_key_prefix (`str`, *optional*, defaults to `"eval"`):
                An optional prefix to be used as the metrics key prefix. For example the metrics "bleu" will be named
                "eval_bleu" if the prefix is "eval" (default)

        Returns:
            A dictionary containing the evaluation loss and the potential metrics computed from the predictions. The
            dictionary also contains the epoch number which comes from the training state.
        Nr|  )rã   rò  r  r$  T)rÝ  Úprediction_loss_onlyrò  r  Ú_jit_compilation_timeÚ_model_preparation_time©r)  r*  )%rã   r±   rå  rM  rö  r‘  r2  r3  r&  r�  r(   rß  rà   r  Úprediction_loopÚevaluation_loopræ   r   r  rY  rf   r)  ÚmathÚceilry  r   rq  rY  rÏ  rr  rs  rt  re  Úon_evaluater}  r†  rŽ  )rš  rã   rò  r  ÚoverriderY  Úeval_dataset_nameÚ_eval_datasetÚdataset_metricsÚeval_dataloaderró  Ú	eval_looprÉ  Útotal_batch_sizes                 r´   rö  zTrainer.evaluateE  s�  € ðZ   tÐ+ˆØ'/ÐF�|�|°TÔ5FˆÝ�l¥DÑ)Ô)ð 		ØˆGØ4@×4FÒ4FÑ4HÔ4Hð 0ð 0Ñ0Ð! =Ø"&§-¢-Ø2:Ð!Q  Ð@QØ +Ø):Ð&PÐ&PÐ=NÐ&PÐ&Pð #0ñ #ô #�ð
 —’˜Ñ/Ô/Ð/Ð/ØˆNð 	Ô×"Ò"Ñ$Ô$Ð$à×2Ò2°<Ñ@Ô@ˆØÔ&ð 	CÝ1°/ÑBÔBˆOå”Y‘[”[ˆ
à,0¬IÔ,PÐj�DÔ(Ð(ÐVZÔVjˆ	Ø�ØØ$ð *.Ô)=Ð)E  È4Ø#Ø/ð
ñ 
ô 
ˆð  œ9Ô4°t´yÔ7KÑKÐØÐ6Ð6Ð6¸&¼.ÐHÐHØ˜&œ.Ð,=Ð)TÐ)TÐ)TÔUÑUˆJØÐ8Ð8Ð8¸F¼NÐJÐJØ˜&œ.Ð,=Ð)VÐ)VÐ)VÔWÑWˆJØŒ×ÒÝØ!ØØ"Ô.Ýœ) FÔ$6Ð9IÑ$IÑJÔJð	ñ ô ñ	
ô 	
ð 	
ð 	�Š�”Ñ Ô Ð åÔ(¨D¬I¬OÐ;Ð;åŒO�CÔ.Ñ0Ô0Ñ1Ô1Ð1àÔ,×8Ò8¸¼ÀDÄJÐPTÔP\Ð^dÔ^lÑmÔmˆŒàÔ×4Ò4°V´^ÑDÔDÐDàŒ~Ðr¶   r)  c                 ó  — | j                              ¦   «          |                      |¦  «        }t          j        ¦   «         }| j        j        r| j        n| j        } ||d||¬¦  «        }| j        j        | j        j	        z  }|› d�|j
        v r||j
        |› d�         z  }|› d�|j
        v r||j
        |› d�         z  }|j
                             t          |||j        t          j        |j        |z  ¦  «        ¬¦  «        ¦  «         | j                             | j        | j        | j        |j
        ¦  «        | _        | j                              |j
        ¦  «         t+          |j        |j        |j
        ¬¦  «        S )aŒ  
        Run prediction and returns predictions and potential metrics.

        Depending on the dataset and your use case, your test dataset may contain labels. In that case, this method
        will also return metrics, like in `evaluate()`.

        Args:
            test_dataset (`Dataset`):
                Dataset to run the predictions on. If it is an `datasets.Dataset`, columns not accepted by the
                `model.forward()` method are automatically removed. Has to implement the method `__len__`
            ignore_keys (`list[str]`, *optional*):
                A list of keys in the output of your model (if it is a dictionary) that should be ignored when
                gathering predictions.
            metric_key_prefix (`str`, *optional*, defaults to `"test"`):
                An optional prefix to be used as the metrics key prefix. For example the metrics "bleu" will be named
                "test_bleu" if the prefix is "test" (default)

        <Tip>

        If your predictions or labels have different sequence length (for instance because you're doing dynamic padding
        in a token classification task) the predictions will be padded (on the right) to allow for concatenation into
        one array. The padding index is -100.

        </Tip>

        Returns: *NamedTuple* A namedtuple with the following keys:

            - predictions (`np.ndarray`): The predictions on `test_dataset`.
            - label_ids (`np.ndarray`, *optional*): The labels (if the dataset contained some).
            - metrics (`dict[str, float]`, *optional*): The potential dictionary of metrics (if the dataset contained
              labels).
        Ú
Prediction)rÝ  rò  r  r  r  r  )ÚpredictionsrÙ  rY  )r2  r3  r*  rß  rà   r  r  r  r   r  rY  r‘  rf   r)  r  r  re  Ú
on_predictr}  r†  rŽ  rV   r   rÙ  )	rš  r'  rò  r  Útest_dataloaderró  r  rÉ  r  s	            r´   ÚpredictzTrainer.predict­  sœ  € ðH 	Ô×"Ò"Ñ$Ô$Ð$à×2Ò2°<Ñ@Ô@ˆÝ”Y‘[”[ˆ
à,0¬IÔ,PÐj�DÔ(Ð(ÐVZÔVjˆ	Ø�Ø¨À;Ðbsð
ñ 
ô 
ˆð  œ9Ô4°t´yÔ7KÑKÐØÐ6Ð6Ð6¸&¼.ÐHÐHØ˜&œ.Ð,=Ð)TÐ)TÐ)TÔUÑUˆJØÐ8Ð8Ð8¸F¼NÐJÐJØ˜&œ.Ð,=Ð)VÐ)VÐ)VÔWÑWˆJØŒ×ÒÝØ!ØØ"Ô.Ýœ) FÔ$6Ð9IÑ$IÑJÔJð	ñ ô ñ	
ô 	
ð 	
ð Ô,×7Ò7¸¼	À4Ä:ÈtÌ|Ð]cÔ]kÑlÔlˆŒØÔ×4Ò4°V´^ÑDÔDÐDå¨FÔ,>È&ÔJZÐdjÔdrÐsÑsÔsÐsr¶   r  c           	      ó  — | j         }|�|n|j        }| j        r| j        €t	          | dd¬¦  «        \  }}|                      | j        d|¬¦  «        }t          | j        j	        ¦  «        dk    rÖ|| j        u rÍt          j
        ¦   «         }	| j        s#| j        r6| j        j        dk    r&| j         j        s| j                             |¦  «        n| j                             |d¬¦  «        }t!          t          j
        ¦   «         |	z
  d	¦  «        | _        | j        r|| _        || j        ur|| _        | j        r| j        | _        | j        s[|j        r'|                     t,          j        |j        ¬
¦  «        }n-|j        r&|                     t,          j        |j        ¬
¦  «        }| j         j        }
t8                               d|› d�¦  «         t=          |¦  «        r1t8                               d|                      |¦  «        › �¦  «         nt8                               d¦  «         t8                               d|
› �¦  «         tA          |d¦  «        r(tC          |j"        ¦  «        r| "                    ¦   «          tA          | j#        d¦  «        r2tC          | j#        j"        ¦  «        r| j#         "                    ¦   «          || j$        _%        tM          |dd¦  «        }|j'        dk    rd| _(        tS          | j         j*        d¬¦  «        }tS          | j         j*        d¬¦  «        }tS          | j         j*        d¬¦  «        }tS          | j         j*        d¬¦  «        }d}i }d}tW          |¦  «        D �]P\  }}tY          |¦  «        }|�	||z  }|
€|}
|  -                    ||||¬¦  «        \  }}}tM          | j        dd¦  «        }d|j.        v r|  /                    ||         ¦  «        nd}ta          ¦   «         rtc          j2        ¦   «          |�=|  3                    | 4                    |
¦  «        ¦  «        }| 5                    |¦  «         |�Y| j         6                    |dd¬¦  «        }|  3                    |¦  «        }| j         j7        r|dk    r| 5                    |¦  «         |�| j         6                    |dd¬¦  «        }|�v| j         6                    |dd¬¦  «        }| j8        �|  8                    ||¦  «        }|  3                    |¦  «        }| j         j7        r|dk    r| 5                    |¦  «         |�<|  3                    |¦  «        }| j         j7        r|dk    r| 5                    |¦  «         | j$         9                    || j:        | j;        ¦  «        | _;        | j         j7        r†| j<        �[|�Y|�W| j        j=        j>        }i }d|j.        v r|nd|d<   d|j.        v r|nd|d<   |  <                    t          d(||dœ|¤Ž|¬¦  «        }~~~~t,          j@         A                    ¦   «          �ŒÆ|jB        �ƒ|dz   |jB        z  dk    rr| C                    ¦   «          | C                    ¦   «          | C                    ¦   «          | C                    ¦   «          ~~~~t,          j@         A                    ¦   «          �ŒR| j        jD        | _3        |j'        r tA          | d¦  «        rt‹          | d¦  «         | F                    ¦   «         }| F                    ¦   «         }| F                    ¦   «         }| F                    ¦   «         }t=          |¦  «        rt          |¦  «        }nYt�          |t�          ¦  «        rtM          |d d¦  «        dk    r|j        }n't=          |¦  «        r|                      |¦  «        }n|}|dk    r|dk    r|}| j<        �S|�Q|�O| j         j7        sCd|j.        v r|nd|d<   d|j.        v r|nd|d<   |  <                    t          d(||dœ|¤Ž¦  «        }n|€i }t“          |¦  «        }t�          |t”          ¦  «        rA|r?t—          jL        |¦  «         M                    ¦   «          N                    ¦   «         ||› d!�<   nFt�          |t–          jO        ¦  «        r,| M                    ¦   «          N                    ¦   «         ||› d!�<   tA          | d"¦  «        r| jP        ||› d#�<   tA          | d$¦  «        r| j        ||› d%�<   t•          | Q                    ¦   «         ¦  «        D ]7}| R                    |› d&�¦  «        s| S                    |¦  «        ||› d&|› �<   Œ8t©          ||||¬'¦  «        S ))ú‘
        Prediction/evaluation loop, shared by `Trainer.evaluate()` and `Trainer.predict()`.

        Works both with or without labels.
        Nr   T©r+  Ú	inferenceF©r‘  r  r  ©Úevaluation_moder±  ©rÊ   r÷   ú
***** Running ú *****r  z  Num examples: Unknownú  Batch size = r#  rÜ  r¢  ©Úpadding_indexrñ  r#  r8  r›  r   )r“  Ú	pad_indexr  rÓ  Úlosses©r   rÙ  ©rî   r(  r5  Ú_lossrá  r  Úmodel_preparation_timer  r|  ©r   rÙ  rY  r)  r½   )Urà   r  rN  r.  r%   rú  r²   rB  rZ  Ú_modelsrß  rU  r=  Útorch_compiler  Úprepare_modelrà  r6  rY  r/  rR  rÅ  rÀ   Úfloat16r÷   rS  r€  r   r"  r#  ra   r5  rF  rp  r#  r`  re  r  r>  rL  r(  r=   Úeval_do_concat_batchesrQ  rE   Úprediction_stepÚinclude_for_metricsr‰  r�   rÏ  rÐ  Úgather_functionÚrepeatrç  Úpad_across_processesr$  rê   Úon_prediction_stepr}  r†  ræ   rR  Úend_of_dataloaderrS   r  rÄ  Úeval_accumulation_stepsÚto_cpu_and_numpyÚgather_for_metricsru  Ú
get_arraysr±   r>   r]   r¹   rÄ   Úconcatenaterü  r'  rÉ   rá  rÚ  r÷  rQ  rR   )rš  r  rÝ  r  rò  r  rà   r|  r²   ró  rü  rã   Ú
all_lossesÚ	all_predsÚ
all_labelsÚ
all_inputsrY  Úeval_set_kwargsÚobserved_num_examplesr†  r›  Úobserved_batch_sizer2  rÇ  r¡  r#  Úinputs_decodeÚis_last_stepÚbatch_kwargsr)  r#  s                                  r´   r  zTrainer.evaluation_loopí  sN
  € ð Œyˆà7KÐ7WÐ3Ð3Ð]aÔ]vÐð Ô$ð 	N¨¬Ð)?Ý! $¸1ÈÐMÑMÔM‰DˆAˆqà× Ò  ¤°eÈ
Ð ÑSÔSˆåˆtÔÔ'Ñ(Ô(¨AÒ-Ð-°%¸4¼:Ð2EÐ2EÝœ™œˆJð Ô,ðQàÔ(ðQà-1Ô-=Ô-MÐQVÒ-VÐ-VÐ_cÔ_hÔ_vÐ-Vð Ô ×(Ò(¨Ñ/Ô/Ð/ð Ô%×3Ò3°EÈ4Ð3ÑPÔPð	 õ +0µ´	±´¸jÑ0HÈ!Ñ*LÔ*LˆDÔ'àÔ#ð #Ø"�”
ð ˜DœJÐ&Ð&Ø%*�Ô"ð Ô(ð 4Ø!%Ô!3�”ð Ôð 	KØÔ"ð KØŸš¥u¤}¸T¼[˜ÑIÔI��ØÔ$ð KØŸš¥u¤~¸d¼k˜ÑJÔJ�à”YÔ.ˆ
å�ŠÐ: {Ð:Ð:Ð:Ñ;Ô;Ð;Ý�jÑ!Ô!ð 	3Ý�KŠKÐK¨D×,=Ò,=¸jÑ,IÔ,IÐKÐKÑLÔLÐLÐLå�KŠKÐ1Ñ2Ô2Ð2Ý�ŠÐ2 jÐ2Ð2Ñ3Ô3Ð3å�5˜&Ñ!Ô!ð 	¥h¨u¬zÑ&:Ô&:ð 	Ø�JŠJ‰LŒLˆLÝ�4”> 6Ñ*Ô*ð 	"­x¸¼Ô8KÑ/LÔ/Lð 	"ØŒN×ÒÑ!Ô!Ð!à0:ˆÔÔ-å˜z¨9°dÑ;Ô;ˆàŒ?˜aÒÐØˆDŒJõ ' t¤yÔ'GÐW[Ð\Ñ\Ô\ˆ
Ý% d¤iÔ&FÐVZÐ[Ñ[Ô[ˆ	Ý& t¤yÔ'GÐW[Ð\Ñ\Ô\ˆ
Ý& t¤yÔ'GÐW[Ð\Ñ\Ô\ˆ
àˆØˆð !"Ðõ & jÑ1Ô1ð C	)ñ C	)‰LˆD�&å"1°&Ñ"9Ô"9ÐØ"Ð.Ø%Ð)<Ñ<Ð%àÐ%Ø!4�Jð &*×%9Ò%9¸%ÀÐI]ÐkvÐ%9Ñ%wÔ%wÑ"ˆF�F˜FÝ% d¤jÐ2CÀ[ÑQÔQˆOà@HÈDÔLdÐ@dÐ@d�×#Ò# F¨?Ô$;Ñ<Ô<Ð<Ðjnð õ &Ñ'Ô'ð Ý”‘”�ð Ð!Ø×-Ò-¨f¯mªm¸JÑ.GÔ.GÑHÔH�Ø—’˜vÑ&Ô&Ð&ØÐ(Ø $Ô 0× EÒ EÀmÐYZÐfjÐ EÑ kÔ k�Ø $× 4Ò 4°]Ñ CÔ C�Ø”yÔ3ð 2°{ÀlÒ7RÐ7RØ—N’N =Ñ1Ô1Ð1ØÐ!àÔ)×>Ò>¸vÈ1ÐX\Ð>Ñ]Ô]�ØÐ!ØÔ)×>Ò>¸vÈ1ÐX\Ð>Ñ]Ô]�ØÔ5ÐAØ!×?Ò?ÀÈÑOÔO�FØ×-Ò-¨fÑ5Ô5�Ø”yÔ3ð *°{ÀlÒ7RÐ7RØ—M’M &Ñ)Ô)Ð)ØÐ!Ø×-Ò-¨fÑ5Ô5�Ø”yÔ3ð +°{ÀlÒ7RÐ7RØ—N’N 6Ñ*Ô*Ð*àÔ0×CÒCÀDÈ$Ì*ÐVZÔVbÑcÔcˆDŒLàŒyÔ+ð )ØÔ'Ð3¸Ð8JÈvÐOaØ#'Ô#3Ô#BÔ#T�LØ#%�LØ7=ÀÔAYÐ7YÐ7Y¨V¨VÐ_c�L Ñ*Ø7?À4ÔC[Ð7[Ð7[¨V¨VÐae�L Ñ*Ø"×2Ò2Ý&Ð\°6ÀVÐ\Ð\È|Ð\Ð\Ø'3ð 3ñ ô �Gð
 ˜F F¨FÝ”
×&Ò&Ñ(Ô(Ð(Ñ(ð Ô-Ð9¸tÀa¹xÈ4ÔKgÑ>gÐklÒ>lÐ>lØ×+Ò+Ñ-Ô-Ð-Ø×*Ò*Ñ,Ô,Ð,Ø×+Ò+Ñ-Ô-Ð-Ø×+Ò+Ñ-Ô-Ð-à˜F F¨FÝ”
×&Ò&Ñ(Ô(Ð(ùð  $Ô/ÔBˆÔØŒ?ð 	#�w t¨WÑ5Ô5ð 	#å�D˜'Ñ"Ô"Ð"ð  ×*Ò*Ñ,Ô,ˆ
Ø×(Ò(Ñ*Ô*ˆ	Ø×*Ò*Ñ,Ô,ˆ
Ø×*Ò*Ñ,Ô,ˆ
õ �lÑ#Ô#ð 
	4Ý˜lÑ+Ô+ˆKˆKõ ˜Õ&:Ñ;Ô;ð 	4ÅÈÐVdÐfgÑ@hÔ@hÐklÒ@lÐ@lØ&Ô3ˆKˆKå˜*Ñ%Ô%ð 4Ø"×/Ò/°
Ñ;Ô;��à3�Ø˜!ÒÐÐ 5¸Ò 9Ð 9Ø/ˆKð Ô Ð,ØÐ%ØÐ&Ø”IÔ0ð 'ð 7=ÀÔ@XÐ6XÐ6X¨
¨
Ð^bˆO˜HÑ%Ø6>À$ÔBZÐ6ZÐ6Z¨
¨
Ð`dˆO˜HÑ%Ø×*Ò*ÝÐ^¨9À
Ð^Ð^ÈoÐ^Ð^ñô ˆGˆGð ˆ_ØˆGõ (¨Ñ0Ô0ˆå�j¥$Ñ'Ô'ð 	L¨Jð 	LÝ35´>À*Ñ3MÔ3M×3RÒ3RÑ3TÔ3T×3YÒ3YÑ3[Ô3[ˆGÐ(Ð/Ð/Ð/Ñ0Ð0Ý˜
¥B¤JÑ/Ô/ð 	LØ3=·?²?Ñ3DÔ3D×3IÒ3IÑ3KÔ3KˆGÐ(Ð/Ð/Ð/Ñ0Ý�4Ð/Ñ0Ô0ð 	]ØCGÔC\ˆGÐ(Ð?Ð?Ð?Ñ@Ý�4Ð1Ñ2Ô2ð 	aØEIÔE`ˆGÐ(ÐAÐAÐAÑBõ ˜Ÿš™œÑ'Ô'ð 	Ið 	IˆCØ—>’>Ð%6Ð"9Ð"9Ð"9Ñ:Ô:ð IØ8?¿ºÀCÑ8HÔ8H�Ð,Ð4Ð4¨sÐ4Ð4Ñ5øå¨)¸zÐSZÐhsÐtÑtÔtÐtr¶   c                 ó(  — |€dS t          ¦   «         r|€d}t          ||¦  «        }njt          ¦   «         rt          |¦  «        }nL| j        j        �| j        j        j        dk    s| j        j        €| j        j        dk    rt          |¦  «        }|S )úž
        Gather value of `tensors` (tensor or list/tuple of nested tensors) and convert them to numpy before
        concatenating them to `gathered`
        NÚnested_gatherÚNOr"  )	r�   rL   r„   r˜   rà   Údistributed_staterY  Ú
local_rankrD   ©rš  ÚtensorsÚnames      r´   rû  zTrainer._nested_gatherÅ  sž   € ð
 ˆ?ØˆFÝ!Ñ#Ô#ð 		2Øˆ|Ø&�Ý,¨W°dÑ;Ô;ˆGˆGÝ$Ñ&Ô&ð 	2Ý  Ñ)Ô)ˆGˆGØŒiÔ)Ð5¸$¼)Ô:UÔ:fÐjnÒ:nÐ:nØŒIÔ'Ð/°D´IÔ4HÈBÒ4NÐ4Nå(¨Ñ1Ô1ˆGØˆr¶   c                 ó.  ‡‡— t          | j        ¦  «        dk    rdnt          ˆfd„| j        D ¦   «         ¦  «        }‰                     d¦  «        }|€| j        }t          | j        ¦  «        dk    o|}|                      ‰¦  «        Š‰€4t          | j        d¦  «        rt          | j        j	        ddg¦  «        Šng Š|s|rIt          t          ˆfd	„| j        D ¦   «         ¦  «        ¦  «        }t          |¦  «        d
k    r|d         }nd}t          j        ¦   «         5  t          ¦   «         �rt          |‰¦  «        }	|s|r¦t!          |	t"          ¦  «        r6|	d         }
t          ˆfd„|	                     ¦   «         D ¦   «         ¦  «        }n|	d         }
|	d
d…         }|
                     ¦   «                              ¦   «                              ¦   «         }t-          |¦  «        }�nÞd}t!          |	t"          ¦  «        r.t          ˆfd„|	                     ¦   «         D ¦   «         ¦  «        }n|	}t-          |¦  «        }�n†|s|rÝ|                      ¦   «         5  |                      ‰g| j        j        ¦  «        }|                      |‰d|¬¦  «        \  }}ddd¦  «         n# 1 swxY w Y   |                     ¦   «                              ¦   «         }t!          |t"          ¦  «        r.t          ˆfd„|                     ¦   «         D ¦   «         ¦  «        }n°|d
d…         }n¥d}|                      ¦   «         5   |di ‰¤Ž}ddd¦  «         n# 1 swxY w Y   t!          |t"          ¦  «        r.t          ˆfd„|                     ¦   «         D ¦   «         ¦  «        }n|}| j        j        dk    r|| j        j        d
z
           | _        ddd¦  «         n# 1 swxY w Y   |r|ddfS t          |¦  «        }t          |¦  «        d
k    r|d         }|||fS )a  
        Perform an evaluation step on `model` using `inputs`.

        Subclass and override to inject custom behavior.

        Args:
            model (`nn.Module`):
                The model to evaluate.
            inputs (`dict[str, Union[torch.Tensor, Any]]`):
                The inputs and targets of the model.

                The dictionary will be unpacked before being fed to the model. Most models expect the targets under the
                argument `labels`. Check your model's documentation for all accepted arguments.
            prediction_loss_only (`bool`):
                Whether or not to return the loss only.
            ignore_keys (`list[str]`, *optional*):
                A list of keys in the output of your model (if it is a dictionary) that should be ignored when
                gathering predictions.

        Return:
            tuple[Optional[torch.Tensor], Optional[torch.Tensor], Optional[torch.Tensor]]: A tuple with the loss,
            logits and labels (each being optional).
        r   Fc              3   óF   •K  — | ]}‰                      |¦  «        d uV — Œd S r  ©r�  )rö   r	  r›  s     €r´   r
  z*Trainer.prediction_step.<locals>.<genexpr>ö  s4   øè è € ÐAvÐAvÐ`aÀ&Ç*Â*ÈQÁ-Ä-ÐW[ÐB[ÐAvÐAvÐAvÐAvÐAvÐAvr¶   Úreturn_lossNr„  Úkeys_to_ignore_at_inferenceÚpast_key_valuesc              3   óB   •K  — | ]}‰                      |¦  «        V — Œd S r  r^  )rö   r[  r›  s     €r´   r
  z*Trainer.prediction_step.<locals>.<genexpr>  s/   øè è € Ð(WÐ(W¸d¨¯ª°DÑ)9Ô)9Ð(WÐ(WÐ(WÐ(WÐ(WÐ(Wr¶   r   rÓ  c              3   ó2   •K  — | ]\  }}|‰d gz   v¯|V — ŒdS ©rÓ  Nr½   ©rö   r	  rB  rò  s      €r´   r
  z*Trainer.prediction_step.<locals>.<genexpr>  s;   øè è € Ð)oÐ)o±°°1ÈqÐXcÐgmÐfnÑXnÐOnÐOn¨!ÐOnÐOnÐOnÐOnÐ)oÐ)or¶   c              3   ó*   •K  — | ]\  }}|‰v¯	|V — Œd S r  r½   re  s      €r´   r
  z*Trainer.prediction_step.<locals>.<genexpr>  s2   øè è € Ð)dÐ)d±°°1ÈqÐXcÐOcÐOc¨!ÐOcÐOcÐOcÐOcÐ)dÐ)dr¶   T)rË  r™  c              3   ó2   •K  — | ]\  }}|‰d gz   v¯|V — ŒdS rd  r½   re  s      €r´   r
  z*Trainer.prediction_step.<locals>.<genexpr>,  s;   øè è € Ð&hÐ&h©T¨Q°ÈÐQ\Ð`fÐ_gÑQgÐHgÐHg qÐHgÐHgÐHgÐHgÐ&hÐ&hr¶   c              3   ó*   •K  — | ]\  }}|‰v¯	|V — Œd S r  r½   re  s      €r´   r
  z*Trainer.prediction_step.<locals>.<genexpr>4  s2   øè è € Ð&]Ð&]©T¨Q°ÈÐQ\ÐH\ÐH\ qÐH\ÐH\ÐH\ÐH\Ð&]Ð&]r¶   r½   )rB  r‰  r–  r�  rv   r‰  rF  r²   r>  r„  rJ   rç  rÀ   rŒ  r„   r—   r±   rå  rM  r¿  rÀ  rô   r™   r¸  Ú_get_num_items_in_batchrà   r÷   rÁ  rü  rL  r(  )rš  r²   r›  r  rò  Ú
has_labelsr_  Úloss_without_labelsr¡  Úraw_outputsrÉ  Ú	logits_mbrÓ  rÇ  r™  rÐ  s     ` `           r´   r=  zTrainer.prediction_stepØ  s  øø€ õ< " $Ô"2Ñ3Ô3°qÒ8Ð8�U�U½cÐAvÐAvÐAvÐAvÐeiÔeuÐAvÑAvÔAvÑ>vÔ>vˆ
ð —j’j Ñ/Ô/ˆØÐØÔ.ˆKÝ! $Ô"2Ñ3Ô3°qÒ8ÐH¸[Ðà×%Ò% fÑ-Ô-ˆØÐÝ�t”z 8Ñ,Ô,ð !Ý% d¤jÔ&7Ð9VÐYjÐXkÑlÔl��à �ð ð 	Ð,ð 	Ý"¥5Ð(WÐ(WÐ(WÐ(WÀdÔFVÐ(WÑ(WÔ(WÑ#WÔ#WÑXÔXˆFÝ�6‰{Œ{˜aÒÐØ œ�øàˆFåŒ]‰_Œ_ð +	Gð +	GÝ&Ñ(Ô(ñ *GÝ.¨u°fÑ=Ô=�Øð :Ð!4ð :Ý! +­tÑ4Ô4ð 4Ø"-¨fÔ"5˜Ý$)Ð)oÐ)oÐ)oÐ)o¸×8IÒ8IÑ8KÔ8KÐ)oÑ)oÔ)oÑ$oÔ$o˜	˜	à"-¨a¤.˜Ø$/°°°¤O˜	à"×.Ò.Ñ0Ô0×7Ò7Ñ9Ô9×=Ò=Ñ?Ô?�DÝ.¨yÑ9Ô9�F‘Fà�DÝ! +­tÑ4Ô4ð 0Ý$)Ð)dÐ)dÐ)dÐ)d¸×8IÒ8IÑ8KÔ8KÐ)dÑ)dÔ)dÑ$dÔ$d˜	˜	à$/˜	Ý.¨yÑ9Ô9�F‘Fàð GÐ!4ð GØ×:Ò:Ñ<Ô<ð ð Ø-1×-IÒ-IÈ6È(ÐTXÔT]ÔTdÑ-eÔ-eÐ*Ø(,×(9Ò(9Ø! 6¸$ÐSeð ):ñ )ô )™˜˜gðð ð ñ ô ð ð ð ð ð ð øøøð ð ð ð ð
  Ÿ;š;™=œ=×-Ò-Ñ/Ô/�Då! '­4Ñ0Ô0ð -Ý!&Ð&hÐ&hÐ&hÐ&h°W·]²]±_´_Ð&hÑ&hÔ&hÑ!hÔ!h˜˜à!(¨¨¨¤˜˜à�DØ×:Ò:Ñ<Ô<ð 2ð 2Ø"' % / /¨& / /˜ð2ð 2ð 2ñ 2ô 2ð 2ð 2ð 2ð 2ð 2ð 2øøøð 2ð 2ð 2ð 2å! '­4Ñ0Ô0ð )Ý!&Ð&]Ð&]Ð&]Ð&]°W·]²]±_´_Ð&]Ñ&]Ô&]Ñ!]Ô!]˜˜à!(˜à”yÔ+¨qÒ0Ð0Ø%,¨T¬YÔ-AÀAÑ-EÔ%F˜œ
ðW+	Gð +	Gð +	Gñ +	Gô +	Gð +	Gð +	Gð +	Gð +	Gð +	Gð +	Gøøøð +	Gð +	Gð +	Gð +	GðZ  ð 	&Ø˜$ Ð%Ð%å˜vÑ&Ô&ˆÝˆv‰;Œ;˜!ÒÐØ˜A”YˆFà�f˜fÐ%Ð%s]   Ä"D:OÉ>J&ÊOÊ&J*	Ê*OÊ-J*	Ê.BOÌ;	MÍOÍM	ÍOÍM	ÍA2OÏOÏOc                 ód   — t          | j        d¦  «        r| j                             |¦  «        S dS )aò  
        For models that inherit from [`PreTrainedModel`], uses that method to compute the number of floating point
        operations for every backward + forward pass. If using another model, either implement such a method in the
        model or subclass and override this method.

        Args:
            inputs (`dict[str, Union[torch.Tensor, Any]]`):
                The inputs and targets of the model.

        Returns:
            `int`: The number of floating-point operations.
        r`  r   )rF  r²   r`  r�  s     r´   r`  zTrainer.floating_point_opsD  s4   € õ �4”:Ð3Ñ4Ô4ð 	Ø”:×0Ò0°Ñ8Ô8Ð8à�1r¶   rË  c                 óD  — |                       ¦   «         sdS | j        j        €1t          | j        j        ¦  «                             ¦   «         j        }n| j        j        }|�|n| j        j        }t          ||| j        j	        d¬¦  «        }|j
        | _        d| _        dS )zE
        Initializes a git repo in `self.args.hub_model_id`.
        NT)rË  Úprivater  )r  rà   rj  r   rí   Úabsoluter[  Ú	hub_tokenr   Úhub_private_repoÚrepo_idÚpush_in_progress)rš  rË  Ú	repo_nameÚrepo_urls       r´   rl  zTrainer.init_hf_repoV  s�   € ð
 ×)Ò)Ñ+Ô+ð 	ØˆFàŒ9Ô!Ð)Ý˜TœYÔ1Ñ2Ô2×;Ò;Ñ=Ô=ÔBˆIˆIàœ	Ô.ˆIàÐ*��°´	Ô0CˆÝ˜y°¸t¼yÔ?YÐdhÐiÑiÔiˆØ$Ô,ˆÔØ $ˆÔÐÐr¶   ÚlanguageÚlicenseÚtagsrñ  Úfinetuned_fromÚtasksÚdataset_tagsÚdataset_argsc
                 ó:  — |                       ¦   «         sdS t          j                             | j        j        d¦  «        }
d}t          j                             |
¦  «        rŠt          j        |
¦  «        j	         
                    d¦  «        }|dk    }t          j        |
¦  «        j	        j        }|�8|�6t          |t          ¦  «        r|g}|D ]}||vr|                     |¦  «         Œt          j        | |||||||||	¬¦
  «
        }|                     ¦   «         }t%          |
d¦  «        5 }|                     |¦  «         ddd¦  «         n# 1 swxY w Y   |r>| j                             | j        ¦  «                             | j        j        ¦  «         dS dS )a  
        Creates a draft of a model card using the information available to the `Trainer`.

        Args:
            language (`str`, *optional*):
                The language of the model (if applicable)
            license (`str`, *optional*):
                The license of the model. Will default to the license of the pretrained model used, if the original
                model given to the `Trainer` comes from a repo on the Hub.
            tags (`str` or `list[str]`, *optional*):
                Some tags to be included in the metadata of the model card.
            model_name (`str`, *optional*):
                The name of the model.
            finetuned_from (`str`, *optional*):
                The name of the model used to fine-tune this one (if applicable). Will default to the name of the repo
                of the original model given to the `Trainer` (if it comes from the Hub).
            tasks (`str` or `list[str]`, *optional*):
                One or several task identifiers, to be included in the metadata of the model card.
            dataset_tags (`str` or `list[str]`, *optional*):
                One or several dataset tags, to be included in the metadata of the model card.
            dataset (`str` or `list[str]`, *optional*):
                One or several dataset identifiers, to be included in the metadata of the model card.
            dataset_args (`str` or `list[str]`, *optional*):
               One or several dataset arguments, to be included in the metadata of the model card.
        Nz	README.mdFÚlibrary_namer¬   )	rx  ry  rz  rñ  r{  r|  r}  rÜ  r~  Úw)r  rn  rt  rç  rà   rí   rÏ  r   rÉ  ru  r�  rz  r±   r%  r�  r)   Úfrom_trainerÚto_model_cardÚopenÚwriterZ  r,   r²   Úcreate_or_update_model_card)rš  rx  ry  rz  rñ  r{  r|  r}  rÜ  r~  Úmodel_card_filepathÚis_peft_libraryr€  Úexisting_tagsr¸  Útraining_summaryÚ
model_cardrµ  s                     r´   Úcreate_model_cardzTrainer.create_model_cardh  sð  € ðJ ×)Ò)Ñ+Ô+ð 	ØˆFå œgŸlšl¨4¬9Ô+?ÀÑMÔMÐØˆÝŒ7�>Š>Ð-Ñ.Ô.ð 	)Ý$œ>Ð*=Ñ>Ô>ÔC×GÒGÈÑWÔWˆLØ*¨fÒ4ˆOõ &œNÐ+>Ñ?Ô?ÔDÔIˆMØÐ MÐ$=Ý˜d¥CÑ(Ô(ð "Ø ˜6�DØ(ð )ð )�CØ $��ØŸš CÑ(Ô(Ð(øå*Ô7ØØØØØ!Ø)ØØ%ØØ%ð
ñ 
ô 
Ðð &×3Ò3Ñ5Ô5ˆ
ÝÐ% sÑ+Ô+ð 	 ¨qØ�GŠG�JÑÔÐð	 ð 	 ð 	 ñ 	 ô 	 ð 	 ð 	 ð 	 ð 	 ð 	 ð 	 øøøð 	 ð 	 ð 	 ð 	 ð ð 	hØÔ×)Ò)¨$¬*Ñ5Ô5×QÒQÐRVÔR[ÔRfÑgÔgÐgÐgÐgð	hð 	hs   Ä.EÅEÅEc           
      ó4  — |                       ¦   «         r| j        j        t          j        k    rd S | j        j        s"| j        �| j                             ¦   «         sd S | j        j        }t          t          t          t          g}t          t          fD ]ì}t          j                             ||¦  «        }t          j                             |¦  «        r«|                     |¦  «         t)          |¦  «        5 }t+          j        |                     ¦   «         ¦  «        }d d d ¦  «         n# 1 swxY w Y   t1          t3          |d                              ¦   «         ¦  «        ¦  «        }|                     |¦  «         Œít9          ¦   «         r'|                     t:          t<          t>          g¦  «         |D ]�}	t          j                             t          j                             ||	¦  «        ¦  «        rQtA          j!        t          j                             ||	¦  «        t          j                             ||	¦  «        ¦  «         Œ‘| j"        �| j"         #                    |¦  «         tI          j%        | j        t          j                             |tL          ¦  «        ¦  «         | j        j'        tP          j)        k    rd| j*        j+        › �}
ndtY          | j*        j-        ¦  «        › �}
t]          | j/        ||
| j        j0        ddtb          › d�g| j        j2        ¬¦  «        }|g}| j        j        t          j3        t          j4        fv rw| j        j        t          j3        k    rdntk          |¦  «        j6        }t]          | j/        |||
d	z   | j        j0        d| j        j2        ¬
¦  «        }|                     |¦  «         | j        �| j                             ¦   «         rto          |¦  «        | _        d S | j        j8                             |¦  «         d S )NÚ
weight_mapzTraining in progress, step zTraining in progress, epoch TrR  rû  ©rt  Úfolder_pathrÖ  rË  Úrun_as_futureÚignore_patternsr×  zlast-checkpointz, checkpoint)rt  r�  Úpath_in_reporÖ  rË  r‘  r×  )9r  rà   Úhub_strategyrU   ÚENDÚhub_always_pushru  Úis_donerí   rm   rn   rr   rp   rq   ro   rn  rt  rç  rB  r�  r„  ÚjsonÚloadsÚreadr¹   r@  rA  Úextendr‚   rj   rl   rk   r~  r_  rÜ   rî  rÀ   ry  rí  r'  rX   r   r}  ru  rÏ  rA  r   rj  rr  rP   rÞ  Ú
CHECKPOINTÚALL_CHECKPOINTSr   r[  rt   Újobs)rš  r/  rí   Úmodeling_filesÚ
index_fileÚ
index_pathrµ  r  Úshard_filesÚmodeling_filerÖ  Úmodel_push_jobÚ	push_jobsr“  Úcheckpoint_pushs                  r´   r-  zTrainer._push_from_checkpoint²  sÛ  € à×)Ò)Ñ+Ô+ð 	¨t¬yÔ/EÍÌÒ/XÐ/XØˆFàŒyÔ(ð 	¨TÔ-BÐ-NÐW[ÔWl×WtÒWtÑWvÔWvÐ-NØˆFà”YÔ)ˆ
å%Õ'=½|ÕM^Ð_ˆå-Õ/FÐGð 	3ð 	3ˆJÝœŸšÐ&7¸ÑDÔDˆJÝŒw�~Š~˜jÑ)Ô)ð 3Ø×%Ò% jÑ1Ô1Ð1Ý˜*Ñ%Ô%ð 1¨Ý œJ q§v¢v¡x¤xÑ0Ô0�Eð1ð 1ð 1ñ 1ô 1ð 1ð 1ð 1ð 1ð 1ð 1øøøð 1ð 1ð 1ð 1å"¥3 u¨\Ô':×'AÒ'AÑ'CÔ'CÑ#DÔ#DÑEÔE�Ø×%Ò% kÑ2Ô2Ð2øÝÑÔð 	jØ×!Ò!Õ#6Õ8LÕNgÐ"hÑiÔiÐiØ+ð 	uð 	uˆMÝŒw�~Š~�bœgŸlšlÐ+<¸mÑLÔLÑMÔMð uÝ”�BœGŸLšLÐ):¸MÑJÔJÍBÌGÏLÊLÐYcÐerÑLsÔLsÑtÔtÐtøàÔ Ð,ØÔ!×1Ò1°*Ñ=Ô=Ð=åŒ
�4”9�bœgŸlšl¨:Õ7IÑJÔJÑKÔKÐKàŒ9Ô"¥lÔ&8Ò8Ð8ØS¸4¼:Ô;QÐSÐSˆNˆNàS½CÀÄ
Ô@PÑ<QÔ<QÐSÐSˆNå&ØÔ%Ø"Ø)Ø”)Ô%ØØ!Õ&;Ð#?Ð#?Ð#?Ð@Ø”YÔ+ð
ñ 
ô 
ˆð $Ð$ˆ	àŒ9Ô!¥kÔ&<½kÔ>YÐ%ZÐZÐZà%)¤YÔ%;½{Ô?UÒ%UÐ%UÐ!Ð!Õ[_Ð`qÑ[rÔ[rÔ[wð õ ,ØÔ)Ø-Ø)Ø-°Ñ>Ø”iÔ)Ø"ØœÔ/ðñ ô ˆOð ×Ò˜_Ñ-Ô-Ð-àÔ Ð(¨DÔ,A×,IÒ,IÑ,KÔ,KÐ(Ý$2°9Ñ$=Ô$=ˆDÔ!Ð!Ð!àÔ!Ô&×-Ò-¨iÑ8Ô8Ð8Ð8Ð8s   Ã8'D+Ä+D/	Ä2D/	c                 óØ   — t          | d¦  «        sd S | j        �N| j                             ¦   «         s7t                               d¦  «         | j                             ¦   «          d S d S d S )Nru  z\Waiting for the current checkpoint push to be finished, this might take a couple of minutes.)rF  ru  r—  r"  r#  Úwait_until_doner°  s    r´   r€  zTrainer._finish_current_pushö  ss   € Ý�tÐ/Ñ0Ô0ð 	ØˆFØÔ Ð,°TÔ5J×5RÒ5RÑ5TÔ5TÐ,Ý�KŠKÐvÑwÔwÐwØÔ!×1Ò1Ñ3Ô3Ð3Ð3Ð3ð -Ð,Ð,Ð,r¶   úEnd of trainingrÖ  Úblockingr×  c           
      óR  — |                      dd¦  «        }|€\| j        j        rP| j        j        €t	          | j        j        ¦  «        j        }n%| j        j                             d¦  «        d         }|�|n| j        j        }| j        €|  	                    |¬¦  «         |  
                    d¬¦  «         |                      ¦   «         sdS t          | j        dd¦  «        �dd	|vrg |d	<   t          |d	         t          ¦  «        r|d	         g|d	<   | j        j        D ]'}||d	         vr|d	                              |¦  «         Œ( | j        dd|i|¤Ž |€| j        j        }|                      ¦   «          t+          | j        | j        j        ||| d
t,          › d�g|¬¦  «        S )u  
        Upload `self.model` and `self.processing_class` to the ðŸ¤— model hub on the repo `self.args.hub_model_id`.

        Parameters:
            commit_message (`str`, *optional*, defaults to `"End of training"`):
                Message to commit while pushing.
            blocking (`bool`, *optional*, defaults to `True`):
                Whether the function should return only when the `git push` has finished.
            token (`str`, *optional*, defaults to `None`):
                Token with write permission to overwrite Trainer's original args.
            revision (`str`, *optional*):
                The git revision to commit from. Defaults to the head of the "main" branch.
            kwargs (`dict[str, Any]`, *optional*):
                Additional keyword arguments passed along to [`~Trainer.create_model_card`].

        Returns:
            The URL of the repository where the model was pushed if `blocking=False`, or a `Future` object tracking the
            progress of the commit if `blocking=True`.
        rñ  Nú/r"  )rË  Trr  Ú
model_tagsrz  rR  rû  r�  r½   )rQ  rà   rm  rj  r   rí   r[  rà  rr  rl  rv  r  r>  r²   r±   r%  r­  r�  rŒ  rÞ  r€  r   rP   )rš  rÖ  rª  rË  r×  r¾  rñ  Ú	model_tags           r´   rk  zTrainer.push_to_hubý  sá  € ð6 —Z’Z ¨dÑ3Ô3ˆ
ØÐ $¤)Ô"7ÐØŒyÔ%Ð-Ý! $¤)Ô"6Ñ7Ô7Ô<�
�
à!œYÔ3×9Ò9¸#Ñ>Ô>¸rÔB�
ØÐ*��°´	Ô0Cˆð ÔÐ$Ø×Ò EÐÑ*Ô*Ð*ð 	�Š tˆÑ,Ô,Ð,ð ×)Ò)Ñ+Ô+ð 	ØˆFõ
 �4”:˜|¨TÑ2Ô2Ð>Ø˜VÐ#Ð#Ø!#��v‘õ ˜& œ.­#Ñ.Ô.ð 2Ø"(¨¤.Ð!1��v‘à!œZÔ2ð 5ð 5�	Ø F¨6¤NÐ2Ð2Ø˜6”N×)Ò)¨)Ñ4Ô4Ð4øàˆÔÐ?Ð?¨*Ð?¸Ð?Ð?Ð?àÐØ”yÔ-ˆHð 	×!Ò!Ñ#Ô#Ð#åØÔ%Øœ	Ô,Ø)ØØ&˜,Ø!Õ&;Ð#?Ð#?Ð#?Ð@Øð
ñ 
ô 
ð 	
r¶   c           	      óœ  — | j         }t          |¦  «        st          d¦  «        ‚|�|n|j        }| j        r| j        €t          | dd¬¦  «        \  }}|                      | j        d|¬¦  «        }t          | j
        j        ¦  «        dk    r~|| j        u ru| j        s| j        r| j
                             |¦  «        n| j
                             |d¬¦  «        }| j        r|| _        || j        ur|| _        | j        r| j        | _        | j        s[|j        r'|                     t&          j        |j        ¬	¦  «        }n-|j        r&|                     t&          j        |j        ¬	¦  «        }t1          |d
d¦  «        r|j        n|j        }	|	€t          d¦  «        ‚|                      |¦  «        }
t8                               d|› d�¦  «         t8                               d|
› �¦  «         t8                               d|	› �¦  «         d}d}d}d}d}i }t=          d|j        ¦  «        }tA          ||
|	¬¦  «        }|snd}tC          |d¦  «        r&tE          |j#        tH          ¦  «        r|j#        j        }tA          ||
|¬¦  «        }tA          ||
|¬¦  «        }tA          ||
|¬¦  «        }| %                    ¦   «          tC          | j&        d¦  «        r2tO          | j&        j%        ¦  «        r| j&         %                    ¦   «          |j(        dk    rd| _)        || j*        _+        tY          |¦  «        D �]f\  }}|  -                    ||||¬¦  «        \  }}}t1          | j        dd¦  «        }d|j.        v r|  /                    ||         ¦  «        nd}|�1| 0                    |	¦  «        }|€|nt'          j1        ||fd¬¦  «        }|�|€|nte          ||d¬¦  «        }|�|€|nte          ||d¬¦  «        }|�|€|nte          ||d¬¦  «        }| j*         3                    || j4        | j5        ¦  «        | _5        | j         j6        rb| j7        �[|�Y|�W| j
        j8        j9        }i } d|j.        v r|nd| d<   d|j.        v r|nd| d<   |  7                    tu          d(||dœ| ¤Ž|¬¦  «        }| j         j6        s|j;        �à|dz   |j;        z  dk    rÏ| <                    |  =                    |d¦  «        ¦  «         |s{| <                    |  =                    |d ¦  «        ¦  «         | <                    |  =                    |d!¦  «        ¦  «         | <                    |  =                    |d"¦  «        ¦  «         ~~~~t&          j>         ?                    ¦   «          d#\  }}}}�Œh|j(        r tC          | d$¦  «        rt�          | d$¦  «         | <                    |  =                    |d¦  «        ¦  «         |s{| <                    |  =                    |d ¦  «        ¦  «         | <                    |  =                    |d!¦  «        ¦  «         | <                    |  =                    |d"¦  «        ¦  «         | A                    ¦   «         }!|s| A                    ¦   «         nd}"|s| A                    ¦   «         nd}#|s| A                    ¦   «         nd}$| j7        �S|"�Q|#�O| j         j6        sCd|j.        v r|!nd|d<   d|j.        v r|$nd|d<   |  7                    tu          d(|"|#dœ|¤Ž¦  «        }n|€i }t…          |¦  «        }|!�,|! C                    ¦   «          D                    ¦   «         ||› d%�<   t‹          | F                    ¦   «         ¦  «        D ]7}%|% G                    |› d&�¦  «        s| H                    |%¦  «        ||› d&|%› �<   Œ8t“          |"|#||
¬'¦  «        S ))r%  z+dataloader must implement a working __len__Nr   Tr&  Fr(  r)  r+  Ú_is_accelerate_preparedz\Batch size cannot be None. Ensure the dataloader has a valid batch_size or total_batch_size.r,  r-  r  r.  r   )Úmake_multiple_ofr  r#  rñ  r#  r8  r›  )r“  r¢  r/  rÓ  r2  r3  r4  Úeval_lossesÚ
eval_predsÚeval_label_idsÚeval_inputs_ids)NNNNr(  r5  r|  r7  r½   )Jrà   ra   r%  r  rN  r.  r%   rú  r²   rB  rZ  r8  rU  r  r:  rY  r/  rR  rÅ  rÀ   r;  r÷   rS  r€  r>  r  rü  r5  r"  r#  rš  r  r<   rF  r±   r  rB   r#  r`  rp  rL  r(  re  r  rQ  r=  r>  r‰  r@  ÚcatrI   rB  r}  r†  r$  ræ   rR  rC  rS   rD  Ú
add_arraysÚ_gather_and_numpifyr  rÄ  ru  Úfinalizer]   rü  r'  r¹   rÚ  r÷  rQ  rR   )&rš  r  rÝ  r  rò  r  rà   r|  r²   rü  r5  Úlosses_hostÚ
preds_hostÚlabels_hostÚinputs_hostrY  rM  r  Úeval_losses_gathererr±  Úpreds_gathererÚlabels_gathererÚinputs_gathererr†  r›  rÓ  rÇ  r¡  r#  rP  r2  rQ  rR  Ú	eval_lossÚpredsrÙ  Ú
inputs_idsr#  s&                                         r´   r  zTrainer.prediction_loopQ  sô  € ð Œyˆå˜*Ñ%Ô%ð 	LÝÐJÑKÔKÐKà7KÐ7WÐ3Ð3Ð]aÔ]vÐð Ô$ð 	N¨¬Ð)?Ý! $¸1ÈÐMÑMÔM‰DˆAˆqà× Ò  ¤°eÈ
Ð ÑSÔSˆåˆtÔÔ'Ñ(Ô(¨AÒ-Ð-°%¸4¼:Ð2EÐ2Eð Ô,ðQØ04Ô0DðQ�Ô ×(Ò(¨Ñ/Ô/Ð/àÔ%×3Ò3°EÈ4Ð3ÑPÔPð ð Ô#ð #Ø"�”
ð ˜DœJÐ&Ð&Ø%*�Ô"ð Ô(ð 4Ø!%Ô!3�”ð Ôð 	KØÔ"ð KØŸš¥u¤}¸T¼[˜ÑIÔI��ØÔ$ð KØŸš¥u¤~¸d¼k˜ÑJÔJ�õ �zÐ#<¸eÑDÔDð'ˆJÔ'Ð'àÔ&ð 	ð ÐÝØnñô ð ð ×(Ò(¨Ñ4Ô4ˆÝ�ŠÐ: {Ð:Ð:Ð:Ñ;Ô;Ð;Ý�ŠÐ6¨Ð6Ð6Ñ7Ô7Ð7Ý�ŠÐ2 jÐ2Ð2Ñ3Ô3Ð3à.2ˆØDHˆ
ØEIˆØEIˆØ"&ˆØ "ˆå˜˜DœOÑ,Ô,ˆ
å8¸À\ÐdnÐoÑoÔoÐØ#ð 	uð  $ÐÝ�z 9Ñ-Ô-ð Aµ*¸ZÔ=OÕQmÑ2nÔ2nð AØ#-Ô#5Ô#@Ð Ý6°zÀ<ÐbrÐsÑsÔsˆNÝ7¸
ÀLÐcsÐtÑtÔtˆOÝ7¸
ÀLÐcsÐtÑtÔtˆOà�
Š
‰ŒˆÝ�4”> 6Ñ*Ô*ð 	"­x¸¼Ô8KÑ/LÔ/Lð 	"ØŒN×ÒÑ!Ô!Ð!àŒ?˜aÒÐØˆDŒJà0:ˆÔÔ-å% jÑ1Ô1ð .	[ñ .	[‰LˆD�&Ø#'×#7Ò#7¸¸vÐG[ÐitÐ#7Ñ#uÔ#uÑ ˆD�&˜&Ý% d¤jÐ2CÀ[ÑQÔQˆOà@HÈDÔLdÐ@dÐ@d�×#Ò# F¨?Ô$;Ñ<Ô<Ð<Ðjnð ð ÐØŸš ZÑ0Ô0�Ø(3Ð(;˜f˜fÅÄÈKÐY_ÐK`ÐfgÐAhÑAhÔAh�ØÐ!Ø'1Ð'9˜V˜V½}ÈZÐY_ÐosÐ?tÑ?tÔ?t�
ØÐ!Ø(3Ð(;˜f˜fÅÈ{Ð\bÐrvÐAwÑAwÔAw�ØÐ(ð #Ð*ð "�Må& {°MÐQUÐVÑVÔVð ð
  Ô0×CÒCÀDÈ$Ì*ÐVZÔVbÑcÔcˆDŒLàŒyÔ+ð 	ØÔ'Ð3¸
Ð8NÐS^ÐSjØ#'Ô#3Ô#BÔ#T�LØ#%�LØ<BÀdÔF^Ð<^Ð<^¨[¨[Ðdh�L Ñ*Ø<DÈÔH`Ð<`Ð<`¨[¨[Ðfj�L Ñ*Ø"×2Ò2Ý&Ðe°:ÈÐeÐeÐXdÐeÐeØ'3ð 3ñ ô �Gð
 ŒyÔ+ð [ØÔ,Ð8¸dÀQ¹hÈ$ÔJfÑ=fÐjkÒ=kÐ=kð %×/Ò/°×0HÒ0HÈÐVcÑ0dÔ0dÑeÔeÐeØ+ð iØ"×-Ò-¨d×.FÒ.FÀzÐS_Ñ.`Ô.`ÑaÔaÐaØ#×.Ò.¨t×/GÒ/GÈÐUeÑ/fÔ/fÑgÔgÐgØ#×.Ò.¨t×/GÒ/GÈÐUfÑ/gÔ/gÑhÔhÐhð   ¨[¸+Ý”
×&Ò&Ñ(Ô(Ð(ØDZÑA�˜Z¨°kùàŒ?ð 	#�w t¨WÑ5Ô5ð 	#å�D˜'Ñ"Ô"Ð"ð 	×'Ò'¨×(@Ò(@ÀÈmÑ(\Ô(\Ñ]Ô]Ð]Ø#ð 	aØ×%Ò% d×&>Ò&>¸zÈ<Ñ&XÔ&XÑYÔYÐYØ×&Ò& t×'?Ò'?ÀÐM]Ñ'^Ô'^Ñ_Ô_Ð_Ø×&Ò& t×'?Ò'?ÀÐM^Ñ'_Ô'_Ñ`Ô`Ð`à(×1Ò1Ñ3Ô3ˆ	Ø1EÐO�×'Ò'Ñ)Ô)Ð)È4ˆØ6JÐT�O×,Ò,Ñ.Ô.Ð.ÐPTˆ	Ø7KÐU�_×-Ò-Ñ/Ô/Ð/ÐQUˆ
ð Ô Ð,ØÐ!ØÐ%Ø”IÔ0ð &ð 6<¸tÔ?WÐ5WÐ5W¨	¨	Ð]aˆO˜HÑ%Ø6>À$ÔBZÐ6ZÐ6Z¨
¨
Ð`dˆO˜HÑ%Ø×*Ò*­>Ð+tÀeÐW`Ð+tÐ+tÐdsÐ+tÐ+tÑuÔuˆGˆGØˆ_ØˆGõ (¨Ñ0Ô0ˆàÐ Ø3<·>²>Ñ3CÔ3C×3HÒ3HÑ3JÔ3JˆGÐ(Ð/Ð/Ð/Ñ0õ ˜Ÿš™œÑ'Ô'ð 	Ið 	IˆCØ—>’>Ð%6Ð"9Ð"9Ð"9Ñ:Ô:ð IØ8?¿ºÀCÑ8HÔ8H�Ð,Ð4Ð4¨sÐ4Ð4Ñ5øå¨%¸9ÈgÐcoÐpÑpÔpÐpr¶   c                 óô   — |€dS t          ¦   «         rt          ||¦  «        }nGt          ¦   «         rt          |¦  «        }n)| j        j        t          j        k    rt          |¦  «        }t          |¦  «        S )rT  N)
r�   rL   r„   r˜   rà   rO  rh   rP  rD   rK   rY  s      r´   r¸  zTrainer._gather_and_numpify  sx   € ð
 ˆ?ØˆFÝ!Ñ#Ô#ð 	2Ý,¨W°dÑ;Ô;ˆGˆGÝ$Ñ&Ô&ð 	2Ý  Ñ)Ô)ˆGˆGØŒYÔ$­Ô(@Ò@Ð@Ý(¨Ñ1Ô1ˆGå˜gÑ&Ô&Ð&r¶   c                 óø  — |                       ¦   «         sdS ddg}t          j                             t          j                             | j        j        d¦  «        ¦  «        rdt          t          j                             | j        j        d¦  «        ¦  «        5 }|                     ¦   «         }ddd¦  «         n# 1 swxY w Y   nd}|}|D ])}||vr#| 	                    d¦  «        r||z  }Œ!|d|› �z  }Œ*||k    r‚t          t          j                             | j        j        d¦  «        d¦  «        5 }t                               d|› �¦  «         |                     |¦  «         ddd¦  «         n# 1 swxY w Y   | j                             d¦  «         t          j        d	¦  «         | j                             ¦   «         s5| j                             d
¦  «         | j                             ¦   «          dS dS )z8Add SageMaker Checkpointing patterns to .gitignore file.Nz*.sagemaker-uploadingz*.sagemaker-uploadedz
.gitignorerß  ú
r�  z"Writing .gitignore file. Content: g      à?z'Add *.sagemaker patterns to .gitignore.)r  rn  rt  rÏ  rç  ÚrepoÚ	local_dirr„  rš  rŠ  r"  rY  r…  Úgit_addrß  ÚsleepÚis_repo_cleanÚ
git_commitÚgit_push)rš  Úpatternsrµ  Úcurrent_contentÚcontentÚpatterns         r´   Ú_add_sm_patterns_to_gitignorez%Trainer._add_sm_patterns_to_gitignore  s\  € ð ×)Ò)Ñ+Ô+ð 	ØˆFà+Ð-CÐDˆõ Œ7�>Š>�"œ'Ÿ,š, t¤yÔ':¸LÑIÔIÑJÔJð 	!Ý•b”g—l’l 4¤9Ô#6¸ÑEÔEÑFÔFð +È!Ø"#§&¢&¡(¤(�ð+ð +ð +ñ +ô +ð +ð +ð +ð +ð +ð +øøøð +ð +ð +ð +øð !ˆOð "ˆØð 	.ð 	.ˆGØ˜gÐ%Ð%Ø×#Ò# DÑ)Ô)ð .Ø˜wÑ&�G�Gà˜~ G˜~˜~Ñ-�Gøð �oÒ%Ð%Ý•b”g—l’l 4¤9Ô#6¸ÑEÔEÀsÑKÔKð !ÈqÝ—’ÐKÀ'ÐKÐKÑLÔLÐLØ—’˜Ñ Ô Ð ð!ð !ð !ñ !ô !ð !ð !ð !ð !ð !ð !øøøð !ð !ð !ð !ð 	Œ	×Ò˜,Ñ'Ô'Ð'õ 	Œ
�3‰ŒˆàŒy×&Ò&Ñ(Ô(ð 	!ØŒI× Ò Ð!JÑKÔKÐKØŒI×ÒÑ Ô Ð Ð Ð ð	!ð 	!s$   ÂB:Â:B>ÃB>Ä43E3Å3E7Å:E7c                 ó:	  ‡	— i }t          d¦  «        r"| j        j        j        �| j        j        j        }d|v r1| j        j        dk    rt          d¦  «        ‚|d         | j        _        | j        j                             ¦   «         Š	t          d¦  «        r<g d¢}t          d%i ˆ	fd„|D ¦   «         ¤Ž}t          d¦  «        r| j        j        |_        ‰	 	                    d¦  «        }t          d	¦  «        s|rt          d
¦  «        ‚n/|r&| j        j        st                               d¦  «         ||_        ‰	 	                    d¦  «         d| j        j        i}| j        j        �-t          d¦  «        st          d¦  «        ‚| j        j        |d<   t          d¦  «        r||d<   n|                     ‰	¦  «         t%          | j        d¦  «        r| j        j        �s| j        j        dk    rcd| _        t-          j        t0          ¦  «        t-          j        d¦  «        k    rt3          | j        j        ¬¦  «        |d<   nt          d¦  «        ‚t5          d%i |¤Ž| _        | j        j        | _        dt=          j        | j        ¦  «        j         v r*tC          j"        | j        | j        j#        ¬¦  «        | _        tI          | j        j%        dd ¦  «        d u| _&        tI          | j        j%        dd ¦  «        d u| _'        tI          | j        j%        dd ¦  «        d u| _        | j'        ru| j        j%        j(        }dD ]?}tS          ||| j        j*         +                    |tI          ||¦  «        ¦  «        ¦  «         Œ@|j,        r| j        j-        rt          d¦  «        ‚| j&        r*tI          | j        dd ¦  «        €|  .                    ¦   «          | j        j/        r7| j&        s| j'        r)| j        j0        r| j&        rdnd}t          |› d �¦  «        ‚| j&        r5| j        j%        j        j1        d!k    r| j        j2        rt          d"¦  «        ‚| j        j/        r;| j'        r6d#tg          | j        j%        j(        j4        ¦  «        v rt          d$¦  «        ‚d S d S d S )&Nr©   r*  r   z÷The `AcceleratorConfig`'s `num_steps` is set but `gradient_accumulation_steps` is greater than 1 in the passed `TrainingArguments`If using the passed `AcceleratorConfig` is desired, do not set the `TrainingArguments` `gradient_accumulation_steps`.)Úsplit_batchesÚdispatch_batchesÚeven_batchesÚuse_seedable_samplerc                 ó<   •— i | ]}|‰                      |¦  «        “ŒS r½   )rQ  )rö   r¤  Úaccelerator_configs     €r´   rK  z>Trainer.create_accelerator_and_postprocess.<locals>.<dictcomp>Y  s*   ø€ ÐWÐWÐW¸E�5Ð,×0Ò0°Ñ7Ô7ÐWÐWÐWr¶   z1.1.0Únon_blockingrÈ  zp`non_blocking` is only supported in accelerate v0.30.0 and above. Please upgrade accelerate to use this feature.zx`non_blocking` is enabled but `dataloader_pin_memory` is not. For the best performance, it's recommended to enable both.Úgradient_accumulation_kwargsrU  z1.10.1zgParallelismConfig requires accelerate v1.10.1 and above. Please upgrade accelerate to use this feature.r�  Údataloader_configrß  Tr¥   )rß  Útorch_tp_pluginz4Requires accelerate>1.3.0 to use Tensor Parallelism.Úuse_gather_object)rß  r7  )Úlimit_all_gathersÚactivation_checkpointingzÁThe activation_checkpointing in FSDP config and the gradient_checkpointing in training arg can't be set to True simultaneously. Please use FSDP's activation_checkpointing logic when using FSDP.rS  Ú	DeepSpeedrÀ  zJ can't be used with `save_only_model` along with `load_best_model_at_end`.r‘  zo`auto_find_batch_size` isn't supported yet with DeepSpeed Zero-3. Please consider using Zero-2, Zero-1, or FSDPÚSHARDED_STATE_DICTzWsave_only_model option is not compatible with FSDP state dict type 'SHARDED_STATE_DICT'r½   )5ry   rà   rÚ  rÜ  rú  r%  Úto_dictrª   Ú	data_seedrQ  rH  r  r"  rG  rÛ  rU  r�  r‘  rF  r²   rß  r>  r   r®   Úaccelerate_versionr¦   r›   rZ  rF  r?  rº   r»   r¼   rå  r   Úeval_use_gather_objectr>  r}  rN  rU  r7  rO  rL  r�  rá  r:  r3  r)  r)  Ú
zero_stager  r%  rÜ  )
rš  Úgrad_acc_kwargsr  rÝ  rÛ  rà   r7  r¤  ÚwrapperrÚ  s
            @r´   r0  z*Trainer.create_accelerator_and_postprocessB  sW  ø€ àˆÝ" 8Ñ,Ô,ð 	X°´Ô1MÔ1jÐ1vØ"œiÔ:ÔWˆOð ˜/Ð)Ð)ØŒyÔ4°qÒ8Ð8å ðLñô ð ð
 9HÈÔ8T�”	Ô5à!œYÔ9×AÒAÑCÔCÐå" 8Ñ,Ô,ð 	Bà mÐ mÐ mÐÝ 7ð !ð !ØWÐWÐWÐWÐEVÐWÑWÔWð!ð !Ðõ ' wÑ/Ô/ð BØ.2¬iÔ.AÐ!Ô+à)×-Ò-¨nÑ=Ô=ˆÝ& xÑ0Ô0ð 
	:Øð Ý!ð Gñô ð ðð
 ð  D¤IÔ$Cð Ý—’ð Oñô ð ð .:ÐÔ*à×ÒÐ=Ñ>Ô>Ð>ð  ¤	Ô :ð
ˆð
 Œ9Ô'Ð3Ý*¨8Ñ4Ô4ð Ý!Ø}ñô ð ð *.¬Ô)EˆDÐ%Ñ&å" 8Ñ,Ô,ð 	,Ø(9ˆDÐ$Ñ%Ð%à�KŠKÐ*Ñ+Ô+Ð+õ �4”:˜yÑ)Ô)ð 	Y¨d¬jÔ.@Ð.LÐQUÔQ[ÔQcÐfgÒQgÐQgØ!%ˆDÔÝŒ}Õ/Ñ0Ô0µ7´=ÀÑ3IÔ3IÒIÐIÝ*CÈDÌJÔL^Ð*_Ñ*_Ô*_�Ð&Ñ'Ð'å Ð!WÑXÔXÐXõ 'Ð.Ð.¨Ð.Ð.ˆÔà#Ô/ÔBˆÔà¥'Ô"3°DÔ4HÑ"IÔ"IÔ"TÐTÐTÝ#,Ô#4ØÔ$¸¼	Ô8Xð$ñ $ô $ˆDÔ õ
 %,¨DÔ,<Ô,BÐDVÐX\Ñ$]Ô$]ÐeiÐ$iˆÔ!Ý& tÔ'7Ô'=¸}ÈdÑSÔSÐ[_Ð_ˆÔÝ$ TÔ%5Ô%;Ð=NÐPTÑUÔUÐ]aÐaˆÔàÔð 		ØÔ*Ô0Ô<ˆKØJð kð k�Ý˜ U¨D¬IÔ,A×,EÒ,EÀeÍWÐU`ÐbgÑMhÔMhÑ,iÔ,iÑjÔjÐjÐjØÔ3ð ¸¼	Ô8Xð Ý ð'ñô ð ð Ô$ð 	/­°´Ð<QÐSWÑ)XÔ)XÐ)`Ø×,Ò,Ñ.Ô.Ð.ð ŒIÔ%ð	uàÔ*ð	uà.2Ô.Bð	uð ”	Ô0ð	uð
 &*Ô%>ÐJ�k�kÀFˆGÝ ÐsÐsÐsÑtÔtÐtð Ô%ð	àÔ Ô&Ô7ÔBÀaÒGÐGØ”	Ô.ð Hõ ð Bñô ð ð ŒIÔ%ð	xàÔ$ð	xð %­¨DÔ,<Ô,BÔ,NÔ,^Ñ(_Ô(_Ð_Ð_åÐvÑwÔwÐwð	xð 	xð 	xð 	xð `Ð_r¶   c                 óÊ   — ddl m} | j        j        j        } ||j        j        ¦  «        |_        |j        j        |_        |j                             | j	        |¦  «         dS )zO
        Sets values in the deepspeed plugin based on the Trainer args
        r   rD  N)
rR  rE  rZ  r}  rU  rF  r„  r  rT  rà   )rš  r  rE  Ú	ds_plugins       r´   r3  z#Trainer.propagate_args_to_deepspeed½  sn   € ð 	QÐPÐPÐPÐPÐPàÔ$Ô*Ô;ˆ	à!9Ð!9¸)Ô:PÔ:WÑ!XÔ!Xˆ	ÔØ%.Ô%;Ô%Bˆ	Ô"ØÔ×5Ò5°d´iÐAUÑVÔVÐVÐVÐVr¶   c                 ó6  — | j         �rt          | j        ¦  «        röddlm} ddlm} t          | j        j        |¦  «        r$ || j        ¦  «        | j	        j
        j        _        t          | j        dd ¦  «        t          j        k    rŠ| j        j        j        j        j        rqt)          j        t,          ¦  «        t)          j        d¦  «        k    rD| j	        j
        j                             | j        j        j        j        d¬¦  «         d S d S d S d S d S d S )Nr   )Ú
PeftConfig)Úfsdp_auto_wrap_policyr  z0.27.0T)r  )rU  rµ   r²   r¬   rî  Úpeft.utils.otherrï  r±   Úactive_peft_configrZ  r}  r7  rÍ  r>  r”   rV  rý   rJ  Úbnb_4bit_quant_storager‹  r   r®   ræ  Úset_mixed_precision)rš  rî  rï  s      r´   r<  z"Trainer._fsdp_qlora_plugin_updatesÉ  s:  € ØÔñ 	¥N°4´:Ñ$>Ô$>ð 	Ø'Ð'Ð'Ð'Ð'Ð'Ø>Ð>Ð>Ð>Ð>Ð>å˜$œ*Ô7¸ÑDÔDð hØF[ÐF[Ð\`Ô\fÑFgÔFg�Ô Ô&Ô2ÔCå˜œ
Ð$9¸4Ñ@Ô@ÕDVÔDeÒeÐeØ”JÔ+Ô?ÔVÔhð få”MÕ"4Ñ5Ô5½¼ÀhÑ8OÔ8OÒOÐOàÔ Ô&Ô2×FÒFØ”JÔ+Ô?ÔVÐaeð Gñ ô ð ð ð ð	ð 	ð 	ð 	ð fÐeÐeÐeàOÐOr¶   r˜  r÷   c                 ó  — d}t          |¦  «        dk    od|d         v o| j        p| j        du}|r2	 t          d„ |D ¦   «         ¦  «        }n# t          t
          f$ r Y nw xY w|��!| j        j        rP| j        j        dk    r@| j	         
                    |                     |¦  «        ¦  «                             ¦   «         }n| j        j        dk    r|| j        j        z  }t          j        |¦  «        r’|                     |¦  «        }| j        j        dk    rK|                     ¦   «         dk    r3|                     d¦  «                             | j        j        d¦  «        }t%          | j	        dd¦  «        x}r
||j        z  }|S )a†  
        Counts the number of items in the batches to properly scale the loss.
        Args:
            batch_samples (`list`): List of batches
            device (`torch.device`): The device on which the number of items in the batch should be.
        Returns:
            None if the number of items in the batch doesn't need to be computed else the number of items in the batch
        Nr   r¡  c              3   óp   K  — | ]1}|d                                d¦  «                             ¦   «         V — Œ2dS )r¡  r¢  N)ÚnerW  )rö   r;  s     r´   r
  z2Trainer._get_num_items_in_batch.<locals>.<genexpr>ô  s@   è è € Ð(eÐ(eÈe¨%°¬/×*<Ò*<¸TÑ*BÔ*B×)GÒ)GÑ)IÔ)IÐ(eÐ(eÐ(eÐ(eÐ(eÐ(er¶   r   r"  r�  )rB  r\  rå   rW  rq  r[  rà   rÎ  r  rZ  rW  rÅ  r›  rÀ   Ú	is_tensorr“  Ú	unsqueezer«  r>  Únon_data_parallel_size)rš  r˜  r÷   r™  Úcount_num_items_in_batchÚpcs         r´   ri  zTrainer._get_num_items_in_batchÙ  sÍ  € ð "Ðå�ÑÔ Ò"ð Ø˜M¨!Ô,Ð,ðð
 Ô.ð 6ð Ô)°Ð5ð 	!ð $ð 	ðÝ%(Ð(eÐ(eÐWdÐ(eÑ(eÔ(eÑ%eÔ%eÐ"Ð"øÝ�~Ð.ð ð ð Ø�ðøøøð Ñ)ØŒyÔ6ð K¸4¼9Ô;OÐSTÒ;TÐ;TØ%)Ô%5×%<Ò%<Ð=O×=RÒ=RÐSYÑ=ZÔ=ZÑ%[Ô%[×%_Ò%_Ñ%aÔ%aÐ"Ð"Ø”” AÒ%Ð%ð &8¸4¼9¼?Ñ%JÐ"åŒÐ1Ñ2Ô2ð YØ%7×%:Ò%:¸6Ñ%BÔ%BÐ"à”9”? QÒ&Ð&Ð+=×+AÒ+AÑ+CÔ+CÀqÒ+HÐ+Hà);×)EÒ)EÀaÑ)HÔ)H×)OÒ)OÐPTÔPYÔP_ÐacÑ)dÔ)dÐ&å  Ô!1Ð3GÈÑNÔNÐN�2ð YØ);¸rÔ?XÑ)XÐ&à!Ð!s   ³A ÁA!Á A!r“  r—  c                 óÆ   — g }t          |¦  «        D ]6}	 |                     t          |¦  «        ¦  «         Œ&# t          $ r Y  nw xY w|                      ||¦  «        }||fS )z¥
        Collects a specified number of batches from the epoch iterator and optionally counts the number of items in the batches to properly scale the loss.
        )r˜  r�  r‡  ÚStopIterationri  )rš  r“  r—  r÷   r˜  r|  r™  s          r´   rO  zTrainer.get_batch_samples  sŠ   € ð ˆå�{Ñ#Ô#ð 	ð 	ˆAðØ×$Ò$¥T¨.Ñ%9Ô%9Ñ:Ô:Ð:Ð:øÝ ð ð ð Ø��ðøøøð "×9Ò9¸-ÈÑPÔPÐØÐ0Ð0Ð0s   •"8¸
AÁAr‚  c                 ó–  — |j         }|dk     }t          |¦  «        rt          |¦  «        nd}|�Rt          ||j        z  t          ||j        z  dk    ¦  «        z   d¦  «        }|rt          j        |j        |z  ¦  «        }|ry|  	                    |¦  «        }|j         dk    r"||z  t          ||z  dk    ¦  «        z   }	||z  }
n|t          j        |j        ¦  «        }	|  	                    |¦  «        |j        z  }
nE|j         dk    r#t          j        }	|}||j         z  }|j         |z  }
nt          d|j         › �¦  «        ‚|	|||
|||fS )a  
        Calculates and returns the following values:
        - `num_train_epochs`
        - `num_update_steps_per_epoch`
        - `num_examples`
        - `num_train_samples`
        - `epoch_based`
        - `len_dataloader`
        - `max_steps`
        r   Nr   zYargs.max_steps must be set to a positive value if dataloader does not have a length, was )rr  ra   rB  rš  rú  rÏ  r  r  rs  r5  ÚsysÚmaxsizer%  )rš  rà   r  r‚  rr  r…  r†  rƒ  r5  rs  r„  s              r´   r4  z#Trainer.set_initial_training_values  s°  € ð ”Nˆ	à !’mˆÝ,6°zÑ,BÔ,BÐL�˜Z™œ˜Èˆð Ð%Ý),Ø $Ô"BÑBÝ�n tÔ'GÑGÈ!ÒKÑLÔLñMàñ*ô *Ð&ð ð ZÝ œI dÔ&;Ð>XÑ&XÑYÔY�	ð ð 	Ø×,Ò,¨ZÑ8Ô8ˆLØŒ~ Ò!Ð!Ø#,Ð0JÑ#JÍSØÐ :Ñ:¸QÒ>ñNô Nñ $Ð ð
 %.Ð0FÑ$FÐ!Ð!å#'¤9¨TÔ-BÑ#CÔ#CÐ Ø$(×$5Ò$5°jÑ$AÔ$AÀDÔDYÑ$YÐ!Ð!ØŒ^˜aÒÐå"œ{ÐØ)2Ð&Ø1°D´NÑBˆLØ $¤Ð1GÑ GÐÐåð%Ø”Nð%ð %ñô ð ð
 Ø&ØØØØØð
ð 	
r¶   )NNNNNNNNNNrß   NN)r­  Nr  )NFN)F)TN)NNN)NNNNN)NNrx  ry  NNrß  )FN)NF)NNr#  )Nr)  )	NNNNNNNNN)r©  TNN)–r=  Ú
__module__Ú__qualname__Ú__doc__Útrainer_pt_utilsrÖ   r×   rØ   rÙ   rÚ   r’   r   r*   r   ÚModuler   ri   r   r   r   rå  r%  r3   r$   r!   r1   r
   rS   r¹   r9   rç  rÀ   rR  Ú	Optimizerra  ÚLambdaLRrË   r	   rÆ  r¬  ÚpropertyrÛ   Úsetterr¸  r»  rf  rÀ  rÂ  rW  rÖ  rÛ  rï  ró  rt  ru  ÚSamplerrû  rÏ  ræ  r   r  r  r!  r&  r*  r0  r8  r-  rd  rl  Ú	parameterr  ro  ÚstaticmethodrP  r.  r5  r=  rX  r  rp  rj  r7  r•  r¨  rú  rk  r  r  r  rz  r   rw  rÎ  rK  rl  rN  rþ  rÿ  r,  r*  r?  r+  r@  rC  rT   rQ   r‚  ry  r‰  r‰  r›  r±  r¸  r´  r\  rÁ  r  r  rv  rØ  rÙ  rx  rP   r{  r.  rö  rV   r#  rR   r  rû  r=  r`  rl  rŒ  r-  r€  rk  r  r¸  rÓ  r0  r3  r<  r÷   ri  r   rO  r4  r½   r¶   r´   rÕ   rÕ   1  s8  € € € € € ð\ð \ð~ lÐkÐkÐkÐkÐkÐkÐkÐkÐkÐkÐkÐkÐkà€_�[Ð+=ÀwÐdhÐiÑiÔið :>Ø,0Ø04ØW[ØY]ð Ø?CØ04ØFJØ59ØjvØaeØhlð!Vcð Vcà�_ b¤i°Ð5Ô6ðVcð Ð(Ô)ðVcð   Ô-ð	Vcð
    g¨Ð@RÐ&RÔ SÔTðVcð ˜u W¨d°3¸°<Ô.@ÐBTÐ%TÔUÔVðVcð #ØÐ)Ð+=Ð?UÐWeÐeÔfô
ðVcð ˜X c¨?Ð&:Ô;Ô<ðVcð $ HÔ-ðVcð " (¨NÐ+;¸TÐ+AÔ"BÔCðVcð ˜D Ô1Ô2ðVcð ˜( 5¤;Ô#8Ô9¸8ÀEÄKÔD\ÔDeÔ;fÐfÔgðVcð #+¨5°°e´kÔ6KÔ1LÈdÐSVÐX[ÐS[ÌnÐ1\Ô+]Ô"^ðVcð  (0°¸%¼,ÈÌÐ9UÐW\ÔWcÐ9cÔ0dÔ'eð!Vcð Vcð Vcñ jÔiðVcðp ð%˜8Ð$;Ô<ð %ð %ð %ñ „Xð%ð Ôð1ð 1ð 1ñ Ôð1ðð ð ð&<ð <ð <ð"	5ð 	5ð 	5ð<ð <ð <ð 	8ð 	8ð 	8ð	 ð 	 ð 	 ðGð Gð GðR\ð \ð \ð;ð ;Ð.@ð ;ÈxÐX[Ì}ð ;ð ;ð ;ð ;ðB EIð'ð 'Ø%ð'Ø4<¸S´Mð'à	ð'ð 'ð 'ð 'ð$0ð 0°¸Ô0Að 0ÈXÐV[ÔVaÔVfÔVnÔMoð 0ð 0ð 0ð 0ðD OSØ!Ø(,ð,ð ,àð,ð ð,ð ð	,ð
 ˜X w i°´Ô1AÔ1IÐ&IÔJÔKð,ð ð,ð ! œð,ð 
ð,ð ,ð ,ð ,ð\
 jð 
ð 
ð 
ð 
ð(+¨gð +¸(À5Ä;ÔCSÔC[Ô:\ð +ð +ð +ð +ðZ%
ð %
°¸¸sÀG¸|Ô9LÔ0Mð %
ÐYcð %
ð %
ð %
ð %
ðN
°ð 
¸Jð 
ð 
ð 
ð 
ð$ZÀð Zð Zð Zð Zð 
 °$°s´)ð 
 ð 
 ð 
 ð 
 ðAð Að AðFRð Rð RðFð Fð FðJð J¨°%¸¸U¼XÔ=OÔ=YÐ8YÔ2ZÔ)[ð Jð Jð Jð Jð  àDHðQ/ð Q/ØðQ/Ø(0°Ô(AðQ/à	ˆs�CˆxŒðQ/ð Q/ð Q/ñ „\ðQ/ðf!ð !°3ð !À5Ä;ÔCXð !ð !ð !ð !ð&K zð K°cð Kð Kð Kð Kð ðð ˜Zð °H¸S´Mð ÈSð ð ð ñ „\ðð62 e¨N¸DÀÀcÀ¼NÐ,JÔ&Kð 62ð 62ð 62ð 62ðpA¨%°ÀÀSÈ#ÀXÄÐ0NÔ*Oð AÐWZð AÐeiÐjmÐotÐjtÔeuð Að Að Að Að0a°Cð að að að aðð ð ð ð!ð !ð !ð !ðF-ð -ð -ð8Xð Xð Xð Xðx >BØ=AØ48ð	uð uà (¨¨s°D¨yÔ)9Ô :ðuð �^ T¨#¨s¨(¤^°TÐ9Ô:ðuð ' t¨C¤yÔ1ð	uð
 ðuð uð uð uðn˜Sð ð ð ð ðY°#ð Yð Yð Yð Yð imðwHð wHð wHð wHðrð ð ð(X=ð X=ð X=ð X=ðtzð zð zðxð ð ðð ð ð ð, hlð'^ð '^ð '^ð '^ðR)_ð )_ð )_ðV$"ð $"ð $"ðL:Ið :Ið :Iðx0ið 0ið 0iðd?1ð ?1ð ?1ðBq5ð q5ð q5ðf1ð 1ð 1ð(5ð 5ð 5ð6%9ð %9ð %9ðR LPØKOØØ+5Ø;?Ø=AðRð Rà˜8 ^Ð$4°d¸3À¸:Ô6FÐ$FÔGÔHðRð $ H¨d°3¸°:Ô.>Ð-?ÀÐ-FÔ$GÔHðRð ð	Rð
 ˜˜d 3œi˜Ô(ðRð ˜%  Ð 6Ô7Ô8ðRð ˜( NÐ#3°SÐ#8Ô9Ô:ðRð 
ˆw˜˜WœÐ%Ô	&ðRð Rð Rð Rðh_ð _˜˜S %˜ZÔ(ð _°h¸u´oð _ÐQUð _ð _ð _ð _ð. 5¨¬°sÐ):Ô#;ð ÀÀeÄlÐTWÐFWÔ@Xð ð ð ð ð$ d¨3°°e´lÀCÐ6GÔ0HÐ+HÔ&Ið ÈdÐSVÐX]Ð^cÔ^jÐloÐ^oÔXpÐSpÔNqð ð ð ð ð )ð )ð )ðVS.¸dÀ3ÈÈeÌlÐ\_ÐN_ÔH`ÐC`Ô>að S.ð S.ð S.ð S.ðj
ð 
ð 
ð
ð 
¸HÀT¼Nð 
ð 
ð 
ð 
ð  6:ð	\!ð \!àŒyð\!ð �S˜% ¤¨cÐ 1Ô2Ð2Ô3ð\!ð % U¤\Ô2ð	\!ð
 
Œð\!ð \!ð \!ð \!ðD  %Ø59ðQ;ð Q;àŒyðQ;ð �S˜% ¤¨cÐ 1Ô2Ð2Ô3ðQ;ð ð	Q;ð
 % U¤\Ô2ðQ;ð Q;ð Q;ð Q;ðf2 tð 2ð 2ð 2ð 2ð
0 tð 
0ð 
0ð 
0ð 
0ð6[ð 6[ X¨c¤]ð 6[È4ð 6[ð 6[ð 6[ð 6[ðpF>ð F> H¨S¤Mð F>ð F>ð F>ð F>ðP)Lð )L ¨¤ð )Lð )Lð )Lð )LðV	"ð 	"ð 	"ð Ð1FÐRWð#"ð #"à	ˆcŒð#"ð #"ð #"ð #"ðJ:ð :ð :ð :ð :ð6 FJØ+/Ø!'ð	fð fà˜u W¨d°3¸°<Ô.@Ð%@ÔAÔBðfð ˜d 3œiÔ(ðfð ð	fð
 
ˆc�5ˆjÔ	ðfð fð fð fðR hnð>tð >tØ#ð>tØ2:¸4À¼9Ô2Eð>tØadð>tà	ð>tð >tð >tð >tðH 04Ø+/Ø!'ðVuð VuàðVuð ðVuð ' tœnð	Vuð
 ˜d 3œiÔ(ðVuð ðVuð 
ðVuð Vuð Vuð Vuðpð ð ð ð0 ,0ðj&ð j&àŒyðj&ð �S˜% ¤¨cÐ 1Ô2Ð2Ô3ðj&ð #ð	j&ð
 ˜d 3œiÔ(ðj&ð 
ˆx˜œÔ% x°´Ô'=¸xÈÌÔ?UÐUÔ	Vðj&ð j&ð j&ð j&ðX¨¨c°5¸¼ÀsÐ9JÔ3KÐ.KÔ)Lð ð ð ð ð$%ð % (¨3¤-ð %ð %ð %ð %ð( #'Ø!%Ø,0Ø$(Ø(,Ø-1Ø48Ø/3Ø48ðHhð Hhà˜3”-ðHhð ˜#”ðHhð �C˜˜cœ DÐ(Ô)ð	Hhð
 ˜S”MðHhð ! œðHhð �S˜$˜sœ) TÐ)Ô*ðHhð ˜C  c¤¨DÐ0Ô1ðHhð �s˜D œI tÐ+Ô,ðHhð ˜C  c¤¨DÐ0Ô1ðHhð Hhð Hhð HhðTB9ð B9ð B9ðH4ð 4ð 4ð ):ØØ#Ø"&ðN
ð N
à  œðN
ð ðN
ð ˜Œ}ð	N
ð
 ˜3”-ðN
ð 
ðN
ð N
ð N
ð N
ðp 04Ø+/Ø!'ðxqð xqàðxqð ðxqð ' tœnð	xqð
 ˜d 3œiÔ(ðxqð ðxqð 
ðxqð xqð xqð xqðt'ð 'ð 'ð %!ð %!ð %!ð %!ðNyxð yxð yxðv
Wð 
Wð 
Wð 
Wðð ð ð 1"°Tð 1"À5Ä<ð 1"ÐT\Ð]bÐchÔcoÐqtÐctÔ]uÔTvð 1"ð 1"ð 1"ð 1"ðf1Ø&ð1Ø58ð1ØBGÄ,ð1à	ˆt�X˜e E¤L°#Ð$5Ô6Ô7Ð7Ô	8ð1ð 1ð 1ð 1ð">
Ø%ð>
Ø3=ð>
ØWZð>
ð >
ð >
ð >
ð >
ð >
r¶   rÕ   (  r  rÂ   r_  rå  r\  Úimportlib.metadatar¯   rº   r˜  r  rn  rÄ  rÿ  r~  rÿ  rh  rß  r9  Úcollections.abcr   r   r   Úpathlibr   Útypingr   r	   r
   r   r   Úintegrationsr   Úhuggingface_hub.utilsrt  r  r	  rÄ   Úsafetensors.torchrÌ  rÀ   Útorch.distributedÚdistributedr(  Úhuggingface_hubr   r   r   Ú	packagingr   r   Útorch.utils.datar   r   r   r   r   rß  r   Úconfiguration_utilsr   Údata.data_collatorr   r   r   Údebug_utilsr   r   Ú!feature_extraction_sequence_utilsr    Úfeature_extraction_utilsr!   r‚  r"   r#   Úimage_processing_utilsr$   Úintegrations.deepspeedr%   r&   r'   Úintegrations.tpur(   Ú	modelcardr)   Úmodeling_utilsr*   r+   r,   Úmodels.auto.modeling_autor-   r.   Úoptimizationr/   r0   Úprocessing_utilsr1   Úpytorch_utilsr2   Útokenization_utils_baser3   Útrainer_callbackr4   r5   r6   r7   r8   r9   r:   r;   r  r<   r=   r>   r?   r@   rA   rB   rC   rD   rE   rF   rG   rH   rI   rJ   rK   rL   rM   rN   rO   Útrainer_utilsrP   rQ   rR   rS   rT   rU   rV   rW   rX   rY   rZ   r[   r\   r]   r^   r_   r`   ra   rb   rc   rd   re   rf   r�  rg   rh   ri   rj   rk   rl   rm   rn   ro   rp   rq   rr   rs   rt   ru   rv   rw   rx   ry   rz   r{   r|   r}   r~   r   r€   r�   r‚   rƒ   r„   r…   r†   r‡   rˆ   r‰   rŠ   r‹   rŒ   r�   rŽ   r�   r�   r‘   Úutils.deprecationr’   Úutils.import_utilsr“   Úutils.quantization_configr”   rc  rh  Úutils.notebookr•   rè  Útorch_xla.core.xla_modelrÆ   Ú	xla_modelrÏ  Útorch_xla.debug.metricsrY  rY  rs  Útorch_xla.runtimeÚruntimer’  Ú	torch_xlaÚXLA_VERSIONr®   r‘  Útorch_xla.distributed.spmdÚspmdr”  Ú!smdistributed.modelparallel.torchÚmodelparallelr|  Úsmdistributed.modelparallelÚSMP_VERSIONr{  r–   r—   r˜   r™   r¬   rš   rÒ   r›   rœ   ræ  Úaccelerate.stater�   rQ  rž   rŸ   r    r¡   r¢   r£   r¤   ÚDATA_SAMPLERSr¦   Úaccelerate.data_loaderr§   r¨   rª   rµ   r¾   rÎ   ra  Ú
get_loggerr=  r"  rí  rx  r{  re  rT  r|  r°  rÕ   r½   r¶   r´   ú<module>r?     s™  ððð ð Ð Ð Ð Ø €€€Ø Ð Ð Ð Ø €€€Ø Ð Ð Ð Ø €€€Ø €€€Ø €€€Ø 	€	€	€	Ø €€€Ø 	€	€	€	Ø €€€Ø 
€
€
€
Ø €€€Ø €€€Ø €€€Ø -Ð -Ð -Ð -Ð -Ð -Ð -Ð -Ø Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð Ø @Ð @Ð @Ð @Ð @Ð @Ð @Ð @Ð @Ð @Ð @Ð @Ð @Ð @ð
ð ð ð ð ð ð -Ð ,Ð ,Ð ,Ð ,Ð ,Ø Ð Ð Ð Ø Ð Ð Ð Ø €€€Ø  Ð  Ð  Ð  Ð  Ð  Ø AÐ AÐ AÐ AÐ AÐ AÐ AÐ AÐ AÐ AØ Ð Ð Ð Ð Ð Ø Ð Ð Ð Ð Ð Ø cÐ cÐ cÐ cÐ cÐ cÐ cÐ cÐ cÐ cÐ cÐ cÐ cÐ cà Ð Ð Ð Ð Ð Ø 1Ð 1Ð 1Ð 1Ð 1Ð 1Ø \Ð \Ð \Ð \Ð \Ð \Ð \Ð \Ð \Ð \Ø <Ð <Ð <Ð <Ð <Ð <Ð <Ð <Ø GÐ GÐ GÐ GÐ GÐ GØ <Ð <Ð <Ð <Ð <Ð <Ø `Ð `Ð `Ð `Ð `Ð `Ð `Ð `Ø 6Ð 6Ð 6Ð 6Ð 6Ð 6Ø eÐ eÐ eÐ eÐ eÐ eÐ eÐ eÐ eÐ eØ 1Ð 1Ð 1Ð 1Ð 1Ð 1Ø &Ð &Ð &Ð &Ð &Ð &Ø RÐ RÐ RÐ RÐ RÐ RÐ RÐ RÐ RÐ Rðð ð ð ð ð ð ð ð 3Ð 2Ð 2Ð 2Ð 2Ð 2Ð 2Ð 2Ø ,Ð ,Ð ,Ð ,Ð ,Ð ,ðð ð ð ð ð ð =Ð <Ð <Ð <Ð <Ð <ð	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ð 	ðð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð,ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð2 KÐ JÐ JÐ JÐ JÐ JÐ JÐ JÐ JÐ Jð)ð )ð )ð )ð )ð )ð )ð )ð )ð )ð )ð )ð )ð )ð )ð )ð )ð )ð )ð )ð )ð )ð )ð )ð )ð )ð )ð )ð )ð )ð )ð )ð )ð )ð )ð )ð )ð )ð )ð )ð )ð )ð )ð )ð )ð )ð )ð )ð )ð )ð )ð )ð )ð )ð )ð )ð )ð )ð )ð )ð )ð )ð )ð )ð )ð )ð )ð )ð )ð )ð )ð )ð )ð )ð )ð )ð )ð )ð )ð )ð )ð )ð )ð )ðT /Ð .Ð .Ð .Ð .Ð .Ø (Ð (Ð (Ð (Ð (Ð (Ø 9Ð 9Ð 9Ð 9Ð 9Ð 9ð )Ð)Ð Ø,Ð à€>ÑÔð 9Ø8Ð8Ð8Ð8Ð8Ð8à 8ÐàÐÑÔð Ø€O€O€OàÐÑÔð 
#Ø)Ð)Ð)Ð)Ð)Ð)Ð)Ð)Ð)Ø)Ð)Ð)Ð)Ð)Ð)Ð)Ð)Ð)Ø"Ð"Ð"Ð"Ð"Ð"Ø4Ð4Ð4Ð4Ð4Ð4à*˜Wœ]¨;Ñ7Ô7¸=¸7¼=ÐI_Ñ;`Ô;`Ò`ÐØð 0Ø/Ð/Ð/Ð/Ð/Ð/Ð/Ð/Ð/øà"Ðð ÐÑÔð &Ø3Ð3Ð3Ð3Ð3Ð3Ð3Ð3Ð3ØFÐFÐFÐFÐFÐFà - ¤¨kÑ :Ô :¸m¸g¼mÈFÑ>SÔ>SÒ SÐàgÐgÐgÐgÐgÐgÐgÐgÐgÐgÐgÐgÐgà %ÐàÐÑÔð ØÐÐÐÐÐàÐÑÔð ?Ø:Ð:Ð:Ð:Ð:Ð:Ð:Ð:Ø<Ð<Ð<Ð<Ð<Ð<Ø1Ð1Ð1Ð1Ð1Ð1ðð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð #�O€MØ€w„}Ð'Ñ(Ô(¨=¨7¬=¸Ñ+AÔ+AÒAÐAØ>Ð>Ð>Ð>Ð>Ð>Ø<Ð<Ð<Ñ<Ñ<Ð<àÑ+Ð,Ñ,€MàÐÑÔð ?Ø>Ð>Ð>Ñ>Ñ>Ð>àÐ˜8Ñ$Ô$ð 9Ø8Ð8Ð8Ñ8Ñ8Ð8ð	ð 	ñ 	ðð ñ ð7ð 7ñ 7ð& ð Ø€M�M�Mà	ˆÕ	™HÑ	%Ô	%�ð )Ñ Ø)Ñ Ø�Ø�Ø$Ñ Ø�Ø&�ð 
€ððñ ô ðdT
ð dT
ð dT
ð dT
ð dT
ñ dT
ô dT
ñô ñdT
ð dT
ð dT
r¶   