716 lines
73 KiB
Plaintext
716 lines
73 KiB
Plaintext
|
o
|
|||
|
|
7—°hc ã @ sb d Z ddlmZ ddlZddlmZ ddlmZ ddlmZm Z m
|
|||
|
|
Z
|
|||
|
|
m Z m Z m
|
|||
|
|
Z
|
|||
|
|
mZmZ ddlmZmZmZmZmZmZmZmZmZmZmZmZmZmZm Z m Z m Z m Z m Z m
|
|||
|
|
Z
|
|||
|
|
m!Z!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z)m*Z*m+Z+m Z m,Z,m-Z-m.Z.m/Z/m0Z0m1Z1m2Z2m3Z3m4Z4m5Z5m6Z6m7Z7m8Z8m9Z9m:Z:m;Z;m<Z<m=Z=m>Z>m?Z?m@Z@mAZAmZmBZBmCZCmDZDmEZEmFZFmGZGmHZHmIZImJZJmKZKmLZLmMZMmNZNmZmOZOmPZPmZm
|
|||
|
|
Z
|
|||
|
|
m$Z$m%Z%m*Z*m=Z=mCZCmZ ddlCZCddlT ddlQm0Z0mRZR dd lSmTZT ddlZddlUZVdd
|
|||
|
|
lWmBZB ddlmZ dd lXmYZYmZZ[ d d
|
|||
|
|
d d
|
|||
|
|
d
|
|||
|
|
dœZ\ej]d d e\d�dd„ ƒZ^e0G dd„ deƒƒZ_ G dd„ de*ƒZ`G dd„ de`ƒZadS )z9
|
|||
|
|
2025.8.9
|
|||
|
|
2025.8.10
|
|||
|
|
4.55.4
|
|||
|
|
0.21.0
|
|||
|
|
__UNSLOTH_VERSIONING__
|
|||
|
|
é )ÚTensorN)Ú
|
|||
|
|
functional)ÚAnyÚListÚOptionalÚTupleÚUnionÚDictÚSetÚCallable)Or ÚAutoModelForCausalLMÚ
|
|||
|
|
AutoTokenizerÚBaseImageProcessorr Ú DPOConfigÚ
|
|||
|
|
DPOTrainerÚ DataCollatorÚDataCollatorForPreferenceÚ
|
|||
|
|
DataLoaderÚDatasetÚEvalLoopOutputÚFÚFDivergenceConstantsÚFDivergenceTypeÚFeatureExtractionMixinÚIterableDatasetÚLiteralÚ$MODEL_FOR_VISION_2_SEQ_MAPPING_NAMESr Ú PartialStateÚPathÚ
|
|||
|
|
PeftConfigÚ PeftModelÚPreTrainedModelÚPreTrainedTokenizerBaseÚProcessorMixinÚRunningMomentsÚSyncRefModelCallbackÚTrainerÚTrainerCallbackr ÚautocastÚcap_expÚcontextmanagerÚcreate_reference_modelÚ dataclassÚ defaultdictÚdisable_dropout_in_modelÚ empty_cacheÚ
|
|||
|
|
flush_leftÚ flush_rightÚgenerate_model_cardÚget_comet_experiment_urlÚget_peft_modelÚinspectÚis_comet_availableÚis_liger_kernel_availableÚis_mlflow_availableÚis_peft_availableÚis_wandb_availableÚ log_table_to_comet_experimentÚmaybe_apply_chat_templateÚmaybe_extract_promptÚnnÚ nullcontextÚosÚpadÚ
|
|||
|
|
pad_to_lengthÚpdÚpeft_module_casting_to_bf16Úprepare_deepspeedÚ prepare_fsdpÚ prepare_model_for_kbit_trainingÚrandomÚselective_log_softmaxÚshift_tokens_rightÚtextwrapÚtorchÚtqdmÚwarningsr r r r! r& r9 r@ rL )Ú*)r, Úfield)ÚVersion)r? )ÚDataCollatorForSeq2SeqÚ DataCollatorForLanguageModelingTF)Úepilogue_fusionÚ max_autotuneÚ
|
|||
|
|
shape_paddingz
|
|||
|
|
trace.enabledztriton.cudagraphs)ÚdynamicÚ fullgraphÚoptionsc
|
|||
|
|
C s¾ t j| d| jd ¡ddd�}t j| d¡ddd�}g }t||ƒD ](\}}| t j¡}t j|d| d¡d� d¡}t j
|
|||
|
|
|dd�}|| } | | ¡ q! t |¡}| | jd | jd f¡}|S )Néÿÿÿÿé r )ÚchunksÚdim)r] Úindex©r] é )
|
|||
|
|
rL ÚchunkÚreshapeÚshapeÚzipÚtoÚfloat32ÚgatherÚ unsqueezeÚsqueezeÚ logsumexpÚappendÚconcat)
|
|||
|
|
Úlogitsr^ Úchunked_logitsÚ
|
|||
|
|
chunked_indexÚall_per_token_logpsÚ chunk_logitsÚ chunk_indexÚselected_logitsÚlogsumexp_valuesÚper_token_logps© rv úQ/workspace/DS-LLM-TEMPLATE-FINETUNING/unsloth_compiled_cache/UnslothDPOTrainer.pyÚ chunked_selective_log_softmax" s
|
|||
|
|
rx c sÀ e Zd ZU dZedddid�Zee ed< edddid�Z ee
|
|||
|
|
ed < eddd
|
|||
|
|
id�Z ee
|
|||
|
|
ed <
|
|||
|
|
! ! " # $ $ % & ' ( # $ ) * + , - . / % 0 1 , d4‡ fd2d3„ Z ‡ Z
|
|||
|
|
S )5ÚUnslothDPOConfigu6.
|
|||
|
|
|
|||
|
|
Configuration class for the [`DPOTrainer`].
|
|||
|
|
|
|||
|
|
This class includes only the parameters that are specific to DPO training. For a full list of training arguments,
|
|||
|
|
please refer to the [`~transformers.TrainingArguments`] documentation. Note that default values in this class may
|
|||
|
|
differ from those in [`~transformers.TrainingArguments`].
|
|||
|
|
|
|||
|
|
Using [`~transformers.HfArgumentParser`] we can turn this class into
|
|||
|
|
[argparse](https://docs.python.org/3/library/argparse#module-argparse) arguments that can be specified on the
|
|||
|
|
command line.
|
|||
|
|
|
|||
|
|
Parameters:
|
|||
|
|
> Parameters that control the model and reference model
|
|||
|
|
|
|||
|
|
model_init_kwargs (`dict[str, Any]` or `None`, *optional*, defaults to `None`):
|
|||
|
|
Keyword arguments for `AutoModelForCausalLM.from_pretrained`, used when the `model` argument of the
|
|||
|
|
[`DPOTrainer`] is provided as a string.
|
|||
|
|
ref_model_init_kwargs (`dict[str, Any]` or `None`, *optional*, defaults to `None`):
|
|||
|
|
Keyword arguments for `AutoModelForCausalLM.from_pretrained`, used when the `ref_model` argument of the
|
|||
|
|
[`DPOTrainer`] is provided as a string.
|
|||
|
|
model_adapter_name (`str` or `None`, *optional*, defaults to `None`):
|
|||
|
|
Name of the train target PEFT adapter, when using LoRA with multiple adapters.
|
|||
|
|
ref_adapter_name (`str` or `None`, *optional*, defaults to `None`):
|
|||
|
|
Name of the reference PEFT adapter, when using LoRA with multiple adapters.
|
|||
|
|
force_use_ref_model (`bool`, *optional*, defaults to `False`):
|
|||
|
|
If you provide a PEFT model as the active model and wish to use a different model for the `ref_model`, set
|
|||
|
|
this flag to `True`.
|
|||
|
|
disable_dropout (`bool`, *optional*, defaults to `True`):
|
|||
|
|
Whether to disable dropout in the model and reference model.
|
|||
|
|
use_logits_to_keep (`bool`, *optional*, defaults to `False`):
|
|||
|
|
If `True`, only a specified number of logits are computed in the forward pass. This can be useful for
|
|||
|
|
saving memory and speeding up training by not computing the logits for all tokens, especially in scenarios
|
|||
|
|
when working with very long prompts where labels are ignored (-100).
|
|||
|
|
|
|||
|
|
> Parameters that control the data preprocessing
|
|||
|
|
|
|||
|
|
dataset_num_proc (`int` or `None`, *optional*, defaults to `None`):
|
|||
|
|
Number of processes to use for processing the dataset.
|
|||
|
|
padding_value (`int` or `None`, *optional*, defaults to `None`):
|
|||
|
|
Padding value to use. If `None`, the padding value of the tokenizer is used.
|
|||
|
|
label_pad_token_id (`int`, *optional*, defaults to `-100`):
|
|||
|
|
Padding value to use for labels.
|
|||
|
|
max_prompt_length (`int` or `None`, *optional*, defaults to `512`):
|
|||
|
|
Maximum length of the prompt.
|
|||
|
|
max_completion_length (`int` or `None`, *optional*, defaults to `None`):
|
|||
|
|
Maximum length of the completion.
|
|||
|
|
max_length (`int` or `None`, *optional*, defaults to `1024`):
|
|||
|
|
Maximum length of the full sequence (prompt + completion).
|
|||
|
|
truncation_mode (`str`, *optional*, defaults to `"keep_end"`):
|
|||
|
|
Truncation mode to use when the sequence exceeds `max_length`. Possible values are `"keep_end"` and
|
|||
|
|
`"keep_start"`.
|
|||
|
|
padding_free (`bool`, *optional*, defaults to `False`):
|
|||
|
|
Whether to perform forward passes without padding by flattening all sequences in the batch into a single
|
|||
|
|
continuous sequence. This reduces memory usage by eliminating padding overhead. Currently, this is only
|
|||
|
|
supported with the `flash_attention_2` attention implementation, which can efficiently handle the flattened
|
|||
|
|
batch structure.
|
|||
|
|
precompute_ref_log_probs (`bool`, *optional*, defaults to `False`):
|
|||
|
|
Whether to precompute the log probabilities from the reference model. Setting this to `True` allows
|
|||
|
|
training without needing the reference model during training, which can help reduce GPU memory usage. If
|
|||
|
|
set to `False` (default), the reference model will be used during training to compute log probabilities
|
|||
|
|
on-the-fly.
|
|||
|
|
precompute_ref_batch_size (`int` or `None`, *optional*, defaults to `None`):
|
|||
|
|
Batch size to use when precomputing reference model log probabilities. This can be set higher than the
|
|||
|
|
training batch size to speed up preprocessing. If `None`, defaults to `per_device_train_batch_size` for
|
|||
|
|
training and `per_device_eval_batch_size` for evaluation.
|
|||
|
|
tools (`Optional[list[Union[dict, Callable]]]`, *optional*, defaults to `None`):
|
|||
|
|
List of tools (callable functions) that will be accessible to the model. If the template does not support
|
|||
|
|
function calling, this argument will have no effect.
|
|||
|
|
|
|||
|
|
> Parameters that control the training
|
|||
|
|
|
|||
|
|
loss_type (`str` or `list[str]`, *optional*, defaults to `"sigmoid"`):
|
|||
|
|
Type of loss to use. Possible values are:
|
|||
|
|
|
|||
|
|
- `"sigmoid"`: sigmoid loss from the original [DPO](https://huggingface.co/papers/2305.18290) paper.
|
|||
|
|
- `"hinge"`: hinge loss on the normalized likelihood from the
|
|||
|
|
[SLiC](https://huggingface.co/papers/2305.10425) paper.
|
|||
|
|
- `"ipo"`: IPO loss from the [IPO](https://huggingface.co/papers/2310.12036) paper.
|
|||
|
|
- `"exo_pair"`: pairwise EXO loss from the [EXO](https://huggingface.co/papers/2402.00856) paper.
|
|||
|
|
- `"nca_pair"`: pairwise NCA loss from the [NCA](https://huggingface.co/papers/2402.05369) paper.
|
|||
|
|
- `"robust"`: unbiased estimate of the DPO loss that is robust to preference noise from the [Robust
|
|||
|
|
DPO](https://huggingface.co/papers/2403.00409) paper.
|
|||
|
|
- `"bco_pair"`: pairwise BCO loss from the [BCO](https://huggingface.co/papers/2404.04656) paper.
|
|||
|
|
- `"sppo_hard"`: SPPO loss with hard label from the [SPPO](https://huggingface.co/papers/2405.00675)
|
|||
|
|
paper.
|
|||
|
|
- `"aot"`: AOT loss for paired datasets from the [AOT](https://huggingface.co/papers/2406.05882) paper.
|
|||
|
|
- `"aot_pair"`: AOT loss for unpaired datasets from the [AOT](https://huggingface.co/papers/2406.05882)
|
|||
|
|
paper.
|
|||
|
|
- `"discopop"`: DiscoPOP (a.k.a Log-Ratio Modulated Loss, LRML) loss from the
|
|||
|
|
[DiscoPOP](https://huggingface.co/papers/2406.08414) paper.
|
|||
|
|
- `"apo_zero"`: APO-zero loss from the [APO](https://huggingface.co/papers/2408.06266) paper.
|
|||
|
|
- `"apo_down"`: APO-down loss from the [APO](https://huggingface.co/papers/2408.06266) paper.
|
|||
|
|
- `"sft"`: Negative log-likelihood loss (standard supervised fine-tuning loss).
|
|||
|
|
|
|||
|
|
Multiple loss types can be combined using comma separation (e.g., `["sigmoid", "bco_pair", "sft"]` for
|
|||
|
|
[MPO](https://huggingface.co/papers/2411.10442)). The `loss_weights` parameter can be used to specify
|
|||
|
|
corresponding weights for each loss type.
|
|||
|
|
|
|||
|
|
use_liger_loss (`bool`, *optional*, defaults to `False`):
|
|||
|
|
Whether to use Liger loss.
|
|||
|
|
base_model_attribute_name (`str`, *optional*, defaults to `"model"`):
|
|||
|
|
Name of the attribute in the model that contains the base model. This is used to get the base model from
|
|||
|
|
the model when the model does not have a `get_decoder` method in the case when `use_liger_loss` is `True`.
|
|||
|
|
beta (`float`, *optional*, defaults to `0.1`):
|
|||
|
|
Parameter controlling the deviation from the reference model. Higher β means less deviation from the
|
|||
|
|
reference model. For the IPO loss (`loss_type="ipo"`), β is the regularization parameter denoted by τ in
|
|||
|
|
the [paper](https://huggingface.co/papers/2310.12036).
|
|||
|
|
f_divergence_type (`str`, *optional*, defaults to `FDivergenceType.REVERSE_KL`):
|
|||
|
|
Type of f-divergence regularization function to compute divergence between policy and reference model.
|
|||
|
|
f_alpha_divergence_coef (`float`, *optional*, defaults to `1.0`):
|
|||
|
|
α coefficient in the α-divergence u^-α regularization function for DPO loss.
|
|||
|
|
reference_free (`bool`, *optional*, defaults to `False`):
|
|||
|
|
Whether to ignore the provided reference model and implicitly use a reference model that assigns equal
|
|||
|
|
probability to all responses.
|
|||
|
|
label_smoothing (`float`, *optional*, defaults to `0.0`):
|
|||
|
|
Robust DPO label smoothing parameter from the [cDPO report](https://ericmitchell.ai/cdpo.pdf) and [Robust
|
|||
|
|
DPO](https://huggingface.co/papers/2403.00409) paper that should be between `0.0` and `0.5`.
|
|||
|
|
use_weighting (`bool`, *optional*, defaults to `False`):
|
|||
|
|
Whether to weight the loss as done in the [WPO paper](https://huggingface.co/papers/2406.11827).
|
|||
|
|
rpo_alpha (`float`, *optional*, defaults to `None`):
|
|||
|
|
α parameter from the [RPO paper](https://huggingface.co/papers/2404.19733) (v3), which controls the
|
|||
|
|
weighting of the NLL term in the loss. If `None`, no weighting is applied and the loss is the same as the
|
|||
|
|
DPO loss. The paper recommends `rpo_alpha=1.0`.
|
|||
|
|
ld_alpha (`float` or `None`, *optional*, defaults to `None`):
|
|||
|
|
α parameter from the [LD-DPO paper](https://huggingface.co/papers/2409.06411), which controls the weighting
|
|||
|
|
of the verbose token log-probabilities in responses. If `None`, no weighting is applied to the verbose
|
|||
|
|
part, and the loss is equivalent to the standard DPO loss. The paper recommends setting `ld_alpha` between
|
|||
|
|
`0.0` and `1.0`.
|
|||
|
|
discopop_tau (`float`, *optional*, defaults to `0.05`):
|
|||
|
|
Ï„/temperature parameter from the [DiscoPOP](https://huggingface.co/papers/2406.08414) paper, which controls
|
|||
|
|
the shape of log ratio modulated loss. The paper recommends the default value `discopop_tau=0.05`.
|
|||
|
|
loss_weights (`list[float]` or `None`, *optional*, defaults to `None`):
|
|||
|
|
List of loss weights for multi-loss combinations. Used when combining multiple loss types.
|
|||
|
|
Example: `[0.8, 0.2, 1.0]` for [MPO](https://huggingface.co/papers/2411.10442). If not provided, defaults
|
|||
|
|
to equal weights (`1.0`) for all loss types.
|
|||
|
|
sync_ref_model (`bool`, *optional*, defaults to `False`):
|
|||
|
|
Whether to synchronize the reference model with the active model every `ref_model_sync_steps` steps, using
|
|||
|
|
the `ref_model_mixup_alpha` parameter. This synchronization originates from the
|
|||
|
|
[TR-DPO](https://huggingface.co/papers/2404.09656) paper.
|
|||
|
|
ref_model_mixup_alpha (`float`, *optional*, defaults to `0.6`):
|
|||
|
|
α parameter from the [TR-DPO](https://huggingface.co/papers/2404.09656) paper, which controls the mix
|
|||
|
|
between the current policy and the previous reference policy during updates. The reference policy is
|
|||
|
|
updated according to the equation: `π_ref = α * π_θ + (1 - α) * π_ref_prev`. To use this parameter, you
|
|||
|
|
must set `sync_ref_model=True`.
|
|||
|
|
ref_model_sync_steps (`int`, *optional*, defaults to `512`):
|
|||
|
|
Ï„ parameter from the [TR-DPO](https://huggingface.co/papers/2404.09656) paper, which determines how
|
|||
|
|
frequently the current policy is synchronized with the reference policy. To use this parameter, you must
|
|||
|
|
set `sync_ref_model=True`.
|
|||
|
|
|
|||
|
|
> Parameters that control the logging
|
|||
|
|
|
|||
|
|
generate_during_eval (`bool`, *optional*, defaults to `False`):
|
|||
|
|
Whether to generate and log completions from both the model and the reference model to W&B or Comet during
|
|||
|
|
evaluation.
|
|||
|
|
|
|||
|
|
NÚhelpzvLLM SamplingParams)ÚdefaultÚmetadataÚvllm_sampling_paramsrZ z8Chunk size to reduce memory usage. -1 is most efficient.Úunsloth_num_chunksz'Maximum sequence length to truncate to.Úmax_seq_lengthFÚnor[ é r éú ç-C ëâ6
|
|||
|
|
?ç{®Gáz„?çÍÌÌÌÌÌì?ç+‡ÙÎ÷ï?ç:Œ0âŽyE>ç ð?ç @Úlinearçš™™™™™¹?ÚpassiveÚwarningTÚstepsr` éô éO
|
|||
|
|
ÚO1ÚautoÚ ç Ú
|
|||
|
|
adamw_8bitÚlengthÚ
|
|||
|
|
every_saveÚlasté éœÿÿÿé é Úkeep_endÚmodelçš™™™™™©?ç333333ã?c¥ § sb |dk r t d|› d�ƒ‚|dkrtd|› d�ƒ‚|d u r(|#dkr(|$dkr(d}d }#|ˆd u r:d
|
|||
|
|
d lm}¦ t|¦ƒ d d
|
|||
|
|
ƒ}ˆtƒ jd¯i d|“d|“d|“d|“d|“d|“d|“d|“d| “d|
|
|||
|
|
“d| “d| “d|
|
|||
|
|
“d|“d |“d |“d |“d |“d |“d!|“d"|“d#|“d$|“d%|“d&|“d'|“d(|“d)| “d*| “d+| “d,| “d-| “d.|!“d/|"“d0|#“d1|$“d2|%“d3|&“d4|'“d5|(“d6|)“d7|*“d8|+“d9|,“d:|-“d;|.“d<|/“d=|0“d>|1“d?|2“d@|3“dA|4“dB|5“dC|6“dD|7“dE|8“dF|9“dG|:“dH|;“dI|<“dJ|=“dK|>“dL|?“dM|@“dN|A“dO|B“dP|C“dQ|D“dR|E“dS|F“dT|G“dU|H“dV|I“dW|J“dX|K“dY|L“dZ|M“d[|N“d\|O“d]|P“d^|Q“d_|R“d`|S“da|T“db|U“dc|V“dd|W“de|X“df|Y“dg|Z“dh|[“di|\“dj|]“dk|^“dl|_“dm|`“dn|a“do|b“dp|c“dq|d“dr|e“ds|f“dt|g“du|h“dv|i“dw|j“dx|k“dy|l“dz|m“d{|n“d||o“d}|p“d~|q“d|r“d€|s“d�|t“d‚|u“dƒ|v“d„|w“d…|x“d†|y“d‡|z“dˆ|{“d‰||“dŠ|}“d‹|~“dŒ|“d�|€“dŽ|�“d�|‚“d�|ƒ“d‘|„“d’|…“d“|†“d”|‡“d•|ˆ“d–|‰“d—|Š“d˜|‹“d™|Œ“dš|�“d›|Ž“dœ|�“d�|�“dž|‘“dŸ|’“d |““d¡|”“d¢|•“d£|–“d¤|—“d¥|˜“d¦|™“d§|š“d¨|›“d©|œ“dª|�“d«|ž“d¬|Ÿ“d| “d®|¡“|¥¤Ž |¢| _|£| _|¤| _ d S )°NgH¯¼šò×z>z Unsloth: Your learning rate of `zi` is too small and less than 1e-7! Consider increasing it, otherwise gradient updates will be close to 0!r` za` is way too larger > 1! Consider decreasing it to 1e-1, otherwise gradient updates will explode!rŽ r� Ú unsloth_training_checkpointsr€ r )Ú cpu_countr[ r� Ú
|
|||
|
|
output_dirÚoverwrite_output_dirÚdo_trainÚdo_evalÚ
|
|||
|
|
do_predictÚ
|
|||
|
|
eval_strategyÚprediction_loss_onlyÚper_device_train_batch_sizeÚper_device_eval_batch_sizeÚper_gpu_train_batch_sizeÚper_gpu_eval_batch_sizeÚgradient_accumulation_stepsÚeval_accumulation_stepsÚ
|
|||
|
|
eval_delayÚtorch_empty_cache_stepsÚ
|
|||
|
|
learning_rateÚ weight_decayÚ
|
|||
|
|
adam_beta1Ú
|
|||
|
|
adam_beta2Ú adam_epsilonÚ
|
|||
|
|
max_grad_normÚnum_train_epochsÚ max_stepsÚlr_scheduler_typeÚ warmup_ratioÚ warmup_stepsÚ log_levelÚlog_level_replicaÚlog_on_each_nodeÚ logging_dirÚlogging_strategyÚlogging_first_stepÚ
|
|||
|
|
logging_stepsÚlogging_nan_inf_filterÚ
|
|||
|
|
save_strategyÚ
|
|||
|
|
save_stepsÚsave_total_limitÚsave_safetensorsÚsave_on_each_nodeÚsave_only_modelÚ'restore_callback_states_from_checkpointÚno_cudaÚuse_cpuÚuse_mps_deviceÚseedÚ data_seedÚ
|
|||
|
|
jit_mode_evalÚuse_ipexÚbf16Úfp16Úfp16_opt_levelÚhalf_precision_backendÚbf16_full_evalÚfp16_full_evalÚtf32Ú
|
|||
|
|
local_rankÚ ddp_backendÚ
|
|||
|
|
tpu_num_coresÚtpu_metrics_debugÚdebugÚdataloader_drop_lastÚ
|
|||
|
|
eval_stepsÚdataloader_num_workersÚdataloader_prefetch_factorÚ
|
|||
|
|
past_indexÚrun_nameÚ disable_tqdmÚremove_unused_columnsÚ label_namesÚload_best_model_at_endÚmetric_for_best_modelÚgreater_is_betterÚignore_data_skipÚfsdpÚfsdp_min_num_paramsÚ fsdp_configÚ"fsdp_transformer_layer_cls_to_wrapÚaccelerator_configÚ deepspeedÚlabel_smoothing_factorÚoptimÚ
|
|||
|
|
optim_argsÚ adafactorÚgroup_by_lengthÚlength_column_nameÚ report_toÚddp_find_unused_parametersÚddp_bucket_cap_mbÚddp_broadcast_buffersÚdataloader_pin_memoryÚ dataloader_persistent_workersÚskip_memory_metricsÚuse_legacy_prediction_loopÚ push_to_hubÚresume_from_checkpointÚ hub_model_idÚ hub_strategyÚ hub_tokenÚhub_private_repoÚhub_always_pushÚ hub_revisionÚgradient_checkpointingÚ gradient_checkpointing_kwargsÚinclude_inputs_for_metricsÚeval_do_concat_batchesÚ fp16_backendÚpush_to_hub_model_idÚpush_to_hub_organizationÚpush_to_hub_tokenÚ
|
|||
|
|
mp_parametersÚauto_find_batch_sizeÚfull_determinismÚ torchdynamoÚ ray_scopeÚ ddp_timeoutÚ
|
|||
|
|
torch_compileÚtorch_compile_backendÚtorch_compile_modeÚinclude_tokens_per_secondÚ include_num_input_tokens_seenÚneftune_noise_alphaÚoptim_target_modulesÚbatch_eval_metricsÚ
|
|||
|
|
eval_on_startÚuse_liger_kernelÚliger_kernel_configÚeval_use_gather_objectÚ average_tokens_across_devicesÚmodel_init_kwargsÚref_model_init_kwargsÚmodel_adapter_nameÚref_adapter_nameÚforce_use_ref_modelÚdisable_dropoutÚuse_logits_to_keepÚdataset_num_procÚ
|
|||
|
|
padding_valueÚlabel_pad_token_idÚmax_prompt_lengthÚmax_completion_lengthÚ
|
|||
|
|
max_lengthÚtruncation_modeÚ padding_freeÚprecompute_ref_log_probsÚprecompute_ref_batch_sizeÚtoolsÚuse_liger_lossÚbase_model_attribute_nameÚbetaÚf_alpha_divergence_coefÚreference_freeÚlabel_smoothingÚ
|
|||
|
|
use_weightingÚ rpo_alphaÚld_alphaÚ discopop_tauÚ loss_weightsÚsync_ref_modelÚref_model_mixup_alphaÚref_model_sync_stepsÚgenerate_during_evalrv )
|
|||
|
|
ÚFloatingPointErrorÚ
|
|||
|
|
OverflowErrorÚmultiprocessingr¢ ÚmaxÚsuperÚ__init__r} r~ r )§Úselfr£ r¤ r¥ r¦ r§ r¨ r© rª r« r¬ r r® r¯ r° r± r² r³ r´ rµ r¶ r· r¸ r¹ rº r» r¼ r½ r¾ r¿ rÀ rÁ r rà rÄ rÅ rÆ rÇ rÈ rÉ rÊ rË rÌ rÍ rÎ rÏ rÐ rÑ rÒ rÓ rÔ rÕ rÖ r× rØ rÙ rÚ rÛ rÜ rÝ rÞ rß rà rá râ rã rä rå ræ rç rè ré rê rë rì rí rî rï rð rñ rò ró rô rõ rö r÷ rø rù rú rû rü rý rþ rÿ r r r r r r r r r r r
|
|||
|
|
r r r
|
|||
|
|
r r r r r r r r r r r r r r r r r r r r! r" r# r$ r% r& r' r( r) r* r+ r, r- r. r/ r0 r1 r2 r3 r4 r5 r6 r7 r8 r9 r: r; r< r= r> r? r@ rA rB rC r} r~ r Úkwargsr¢ ©Ú __class__rv rw rI Þ s( ) ÿþýüûúùø ÷
|
|||
|
|
ö õ ô
|
|||
|
|
óòñðïîíìëêéèçæå ä ã â á à!ß"Þ#Ý$Ü%Û&Ú'Ù(Ø)×*Ö+Õ,Ô-Ó.Ò/Ñ0Ð1Ï2Î3Í4Ì5Ë6Ê7É8È9Ç:Æ;Å<Ä=Ã>Â?Á@ÀA¿B¾C½D¼E»FºG¹H¸I·J¶KµL´M³N²O±P°Q¯R®ST¬U«VªW©X¨Y§Z¦[¥\¤]£^¢_¡` aŸbžc�dœe›fšg™h˜i—j–k•l”m“n’o‘p�q�rŽs�tŒu‹vŠw‰xˆy‡z†{…|„}ƒ~‚� �ÿ �þ �ý �ü �û �ú �ù �ø �÷
|
|||
|
|
�ö �õ �ô
|
|||
|
|
�ó �ò �ñ �ð �ï �î �í �ì �ë �ê �é �è �ç �æ �å �ä �ã �â �á �à ! �ß " �Þ #
|
|||
|
|
zUnslothDPOConfig.__init__)¤NNFFFr€ Fr[ r[ NNr� r� r r‚ rƒ r„ r… r† r‡ rˆ r‰ rZ rŠ r‹ r rŒ r� TNrŽ Fr` FrŽ r� NTFFFFFFr� r� FFFFr‘ r’ FFNrZ NNFr“ FNr NrZ NNTNFNNFr“ r NNNNr” r• NFFr– NNNNTFTFFNNr— NNFNFNFTr’ NNNr“ TFNr˜ r™ FNNFFNNFFFNFTNNNNFTFNNrš r› Nrœ r� FFNNFrž r‹ rˆ Fr” FNNrŸ NFr r› FNrZ N)Ú__name__Ú
|
|||
|
|
__module__Ú __qualname__Ú__doc__rP r} r r Ú__annotations__r~ Úintr rI Ú
|
|||
|
|
__classcell__rv rv rL rw ry 3 sj
|
|||
|
|
þþþ �Úry c s e Zd ZdZddgZ djdeeeje f de
|
|||
|
|
ee ejef de
|
|||
|
|
e d e
|
|||
|
|
e d
|
|||
|
|
e
|
|||
|
|
ee
|
|||
|
|
ef d e
|
|||
|
|
ee
|
|||
|
|
eeeee
|
|||
|
|
ef f f d e
|
|||
|
|
eeeeef d
|
|||
|
|
e
|
|||
|
|
eegef de
|
|||
|
|
ee dee
|
|||
|
|
ejj e
|
|||
|
|
ejj j f de
|
|||
|
|
ee ejj eee f f de
|
|||
|
|
eej ej gej f de
|
|||
|
|
d f‡ fdd„
|
|||
|
|
Z!dkdede de"de fdd„Z#de de de de de f
|
|||
|
|
d d „Z$de de fd d „Z%d ee
|
|||
|
|
ef d eeeeef de d!edee
|
|||
|
|
ef f
|
|||
|
|
d"d#„Z&e' $dld%eeef d ed&e
|
|||
|
|
e( d'e
|
|||
|
|
e( d(e"deeee( f f d)d*„ƒZ)e' $dld%eeef d ed&e
|
|||
|
|
e( d'e
|
|||
|
|
e( d(e"deeee( f f d+d,„ƒZ*d-d.„ Z+de,f‡ fd/d0„ Z-dmd e
|
|||
|
|
e
|
|||
|
|
de,f‡ fd1d2„
|
|||
|
|
Z.e/d3d4„ ƒZ0d5eeej1f deej ej f fd6d7„Z2e'd5eeeeej1f f d8e(deeej1f fd9d:„ƒZ3 ; dnd<ej4d=ej4d>ej4d?ej4d@edAeeej4f deej4ej4ej4f fdBdC„Z5dejd5eeeeej1f f deeej f fdDdE„Z6 dkdejd5eeeeej1f f dFe"deeej f fdGdH„Z7 Idodee ejf d5eeeeej1f f dJe8dK deej eee9f f fdLdM„Z: dpdee ejf dNeeeej e f f deej eej eee9f f f fdOdP„Z;d5eeej1f deeef fdQdR„Z< dmdee ejf dNeeeej e f f dSe"dTe
|
|||
|
|
ee deej e
|
|||
|
|
ej e
|
|||
|
|
ej f f
|
|||
|
|
dUdV„Z=dodWeee9f dJe8dK ddfdXdY„Z> Zdqd[e,d\edSe
|
|||
|
|
e" dTe
|
|||
|
|
ee d]edef ‡ fd^d_„
|
|||
|
|
Z?dmd`eee9f dae
|
|||
|
|
e9 ddf‡ fdbdc„
|
|||
|
|
Z@‡ fddde„ZA drdfe
|
|||
|
|
e d!e
|
|||
|
|
e dgeeee df fdhdi„ZB‡ ZCS )sÚ_UnslothDPOTrainerr“ ÚtrlÚdpoN©NNrž Ú ref_modelÚargsÚ
|
|||
|
|
data_collatorÚ
|
|||
|
|
train_datasetÚ eval_datasetÚprocessing_classÚcompute_metricsÚ callbacksÚ
|
|||
|
|
optimizersÚoptimizer_cls_and_kwargsÚ preprocess_logits_for_metricsÚ peft_configr c
|
|||
|
|
st t |tƒr|n|jj}ˆ d u r | d¡d }t|› d�ƒ‰ ˆd u r&t |¡‰ˆ jd ur0ˆ jˆ_n$t ˆdƒr?ˆj
|
|||
|
|
d ur?ˆj
|
|||
|
|
ˆ_nt ˆdƒrPˆj j
|
|||
|
|
d urPˆj j
|
|||
|
|
ˆ_nt dƒ‚t |tƒsa||u rat dƒ‚ˆ j
|
|||
|
|
d urpt |tƒspt d¡ t |tƒr{ˆ |ˆ ¡}ˆ jd urŠt |tƒsŠt d ¡ t |tƒr—ˆj|ˆ d
|
|||
|
|
d �}ˆ |||
|
|||
|
|
ˆ ¡}ˆ jr¯tƒ s¯tƒ s¯tƒ s¯t d ƒ‚|jjˆ_|jjt ¡ v ˆ_t ƒ oÄt |t ƒˆ_ ˆ j ˆ_ ˆ j ˆ_ ˆ j!ˆ_!|rØ|ˆ_"nˆj sÞˆ j#râd ˆ_"nt$|ƒˆ_"ˆ j%røt&|ƒ ˆj"d urøt&ˆj"ƒ ˆ j'�rt(ƒ �st)d
|
|||
|
|
ƒ‚ˆ j*dk�rt dƒ‚t+ˆ j,ˆ j-ˆ j! dd�ˆ_.d
|
|||
|
|
|j/d< |d u �r+t0ˆjd�}ˆ jˆ_ˆ j,ˆ_,ˆ j1ˆ_1ˆ j2ˆ_2ˆ j3ˆ_3ˆ j4ˆ_4ˆ j#ˆ_#ˆ j5ˆ_5ˆ j6�rf|jj7dk�r[t d¡ ˆ j8dk�rft d¡ ˆ j6ˆ_6dˆ_9dˆ_:ˆ j-ˆ_-ˆ j;ˆ_;t ˆ j*t<ƒ�r‚ˆ j*nˆ j*gˆ_*ˆ j=ˆ_=t>|jddƒˆ_?ˆ j@ˆ_@t>|jddƒˆ_Aˆj?�r¯ˆjAdk�r¯t dtB¡ ˆj*D ]!}|d v �rɈ j;d k�rÉt d |› d �tB¡ |d k�rÒt d!ƒ‚�q²tCd"d#„ ƒˆ_Dˆ jEˆ_EtFjGˆ jHiˆ_Iˆ jJˆ_Jˆ K|ˆˆ d$¡}|d u�rt |tLƒ�r ‡ ‡‡fd%d&„| M¡ D ƒ}nˆ K|ˆˆ d'¡}tNƒ jO|ˆ |||ˆ|| |
|
|||
|
|
| | d(� dˆ_Pt ˆjQd)ƒ�r5ˆjQ RˆjS¡ t ˆd*ƒ�s?tTd+ƒ‚ˆjU�rTˆjVjWjXjYd,k�rTˆj#�rTt d-ƒ‚ˆj"d u �roˆj �sfˆj#�sft d.ƒ‚ˆ jZ�rnt d/ƒ‚n$ˆjU�r|t[ˆj"ˆjVƒˆ_"nˆj\�r‰t]ˆj"ˆjVƒˆ_"n
|
|||
|
|
ˆjVj^ˆj"d
|
|||
|
|
d0�ˆ_"ˆ jZ�rªˆj#�rŸt d1ƒ‚ˆ _t`ˆj"ˆjVd2�¡ d3ˆj*v �r¸taˆjVƒˆ_bd S d S )4Nú/rZ z-DPOÚ pad_token_idÚ tokenizera `padding_value` is not specified in `DPOConfig`, and `pad_token_id` is missing in the `processing_class`. Please either set the `padding_value` argument in `DPOConfig`, or set `tokenizer.pad_token` (e.g., `tokenizer.pad_token = tokenizer.eos_token`) before instantiating the trainer.zœ`model` and `ref_model` cannot be the same object. If you want `ref_model` to be the same as `model`, you must mass a copy of it, or `None` if you use peft.z�You passed model_init_kwargs to the `DPOConfig`, but your model is already instantiated. The `model_init_kwargs` will be ignored.z�You passed ref_model_init_kwargs to the `DPOConfig`, but your ref_model is already instantiated. The `ref_model_init_kwargs` will be ignored.T)Úis_refz”`generate_during_eval=True` requires Weights and Biases, MLFlow or Comet to be installed. Please install `wandb`, `mlflow` or `comet-ml` to resolve.z‚You set `use_liger_loss=True` but the liger kernel is not available. Please install liger-kernel first: `pip install liger-kernel`Úsigmoidz{You set `use_liger_loss=True` but the loss type is not `sigmoid`. Please set `loss_type='sigmoid'` to use the liger kernel.F)Ú ignore_indexr7 Ú
|
|||
|
|
use_ref_modelÚaverage_log_probÚestimate_tokens)rf Úflash_attention_2aç Padding-free training is enabled, but the attention implementation is not set to 'flash_attention_2'. Padding-free training flattens batches into a single sequence, and 'flash_attention_2' is the only known attention mechanism that reliably supports this. Using other implementations may lead to unexpected behavior. To ensure compatibility, set `attn_implementation='flash_attention_2'` in the model configuration, or verify that your attention mechanism can handle flattened sequences.r` zÎYou are using a per_device_train_batch_size of 1 with padding-free training. Using a batch size of 1 anihilate the benefits of padding-free training. Please consider increasing the batch size to at least 2.Úoutput_router_logitsÚrouter_aux_loss_coefr” a- You set `output_router_logits` to `True` in the model config, but `router_aux_loss_coef` is set to `0.0`, meaning the auxiliary loss will not be used. Either set `router_aux_loss_coef` to a value greater than `0.0`, or set `output_router_logits` to `False` if you don't want to use the auxiliary loss.)ÚhingeÚipoÚbco_pairÚ sppo_hardÚnca_pairÚapo_zeroÚapo_downr zYou are using the z™ loss type that does not support label smoothing. The `label_smoothing` parameter will be ignored. Set `label_smoothing` to `0.0` to remove this warning.Úkto_pairzKSupport for kto_pair has been removed in DPOTrainer. Please use KTOTrainer.c S s t tƒS ©N)r- Úlistrv rv rv rw Ú<lambda>ö s z-_UnslothDPOTrainer.__init__.<locals>.<lambda>Útrainc s" i | ]
|
|||
|
|
\}}|ˆ |ˆˆ |¡“qS rv )Ú_prepare_dataset)Ú.0ÚkeyÚdataset©rZ r^ rJ rv rw Ú
|
|||
|
|
<dictcomp>ÿ s ÿÿz/_UnslothDPOTrainer.__init__.<locals>.<dictcomp>Úeval) rž rZ r[ r\ r] r^ r_ r` ra rb rc Úadd_model_tagsÚ acceleratorzXYour `Trainer` does not have an `accelerator` object. Consider upgrading `transformers`.é zrYou cannot use `precompute_ref_log_probs=True` with Deepspeed ZeRO-3. Please set `precompute_ref_log_probs=False`.z]No reference model and model is not a Peft model. Try setting `precompute_ref_log_probs=True`zYYou currently cannot use `ref_model=None` with TR-DPO method. Please provide `ref_model`.)Úevaluation_modezoYou cannot use `precompute_ref_log_probs=True` with TR-DPO method. Please set `precompute_ref_log_probs=False`.)rY r… rs )cÚ
|
|||
|
|
isinstanceÚstrÚconfigÚ
|
|||
|
|
_name_or_pathÚsplitr r
|
|||
|
|
Úfrom_pretrainedr+ Úhasattrrf rg Ú
|
|||
|
|
ValueErrorr# rN ÚwarnÚ_create_model_from_pathr$ Ú_prepare_peft_modelrC r: r6 r8 Úis_encoder_decoderÚ
|
|||
|
|
model_typer ÚkeysÚis_vision_modelr9 r Ú
|
|||
|
|
is_peft_modelr% r& r9 rY r2 r+ r( r. r5 r7 Ú ImportErrorÚ loss_typeÚLigerFusedLinearDPOLossr, r7 Ú dpo_loss_fnÚwarnings_issuedr r- r. r/ r0 r) r1 Ú_attn_implementationrª Ú _precomputed_train_ref_log_probsÚ _precomputed_eval_ref_log_probsr: rz r? ÚgetattrÚaux_loss_enabledr; Ú
|
|||
|
|
aux_loss_coefÚ UserWarningr- Ú_stored_metricsÚf_divergence_typer ÚALPHA_DIVERGENCE_COEF_KEYr8 Úf_divergence_paramsr* r} ÚdictÚitemsrH rI Úmodel_accepts_loss_kwargsrž r„ Ú
|
|||
|
|
_tag_namesÚAttributeErrorÚis_deepspeed_enabledr… ÚstateÚdeepspeed_pluginÚ
|
|||
|
|
zero_stager@ rE Úis_fsdp_enabledrF Ú
|
|||
|
|
prepare_modelÚ add_callbackr% r$ Úrunning)rJ rž rY rZ r[ r\ r] r^ r_ r` ra rb rc rd Úmodel_idÚ
|
|||
|
|
model_namer™ rL r� rw rI ; sV
|
|||
|
|
|
|||
|
|
|
|||
|
|
|
|||
|
|
ÿÿÿ
|
|||
|
|
ÿ
|
|||
|
|
ÿ
|
|||
|
|
|
|||
|
|
|
|||
|
|
|
|||
|
|
ÿ ÿü
|
|||
|
|
|
|||
|
|
ÿ ÿ û
|
|||
|
|
|
|||
|
|
|
|||
|
|
ý
|
|||
|
|
ÿ
|
|||
|
|
þõ ÿÿ ÿÿÿÿ ÿz_UnslothDPOTrainer.__init__FÚ
|
|||
|
|
model_pathrh Úreturnc C s† |s|j pi }n|jp i }| d¡}t|tjƒs |dks |du r!nt|tƒr0tt|ƒ}||d< ntd|› d�ƒ‚t j
|
|||
|
|
|fi |¤Ž}|S )z0Creates a model from a path or model identifier.Ú torch_dtyper’ NzˆInvalid `torch_dtype` passed to `DPOConfig`. Expected either 'auto' or a string representing a `torch.dtype` (e.g., 'float32'), but got Ú.) r# r$ Úgetrˆ rL Údtyper‰ r r� r r� )rJ r· rZ rh r# r¹ rž rv rv rw r‘ E s
|
|||
|
|
|
|||
|
|
|
|||
|
|
|
|||
|
|
|
|||
|
|
ÿÿ z*_UnslothDPOTrainer._create_model_from_pathc C sü d| _ tƒ s|durtdƒ‚tƒ rv|durvt|tƒr | ¡ }|dur)|js)tdƒ‚t|ddƒs5t|ddƒrYt|dƒoCdt t
|
|||
|
|
t ¡j
|
|||
|
|
ƒv }d|ji}|rP|j|d< t |fi |¤Ž}n| ||¡}t||ƒ}|jrtt|ddƒrtt|ƒ d | _ |S | ||¡}|S )
|
|||
|
|
z#Prepares a model for PEFT training.FNzvPEFT is not installed and you passed a `peft_config` in the trainer's kwargs, please install it to use the PEFT modelsa8 You passed both a ref_model and a peft_config. For training PEFT adapters with DPO there is no need to pass a reference model. Please pass `ref_model=None` in case you want to train PEFT adapters, or pass a ref_model with `force_use_ref_model=True` in DPOTrainer's init. if you want to use a different ref_model.Úis_loaded_in_8bitÚis_loaded_in_4bitr Úuse_gradient_checkpointingT)Ú _peft_has_been_casted_to_bf16r9 r� rˆ r Úmerge_and_unloadr' r rŽ rz r5 Ú signaturerG Ú
|
|||
|
|
parametersr r Ú _prepare_gradient_checkpointingr4 rÓ rD )rJ rž rY rd rZ Ú_support_gc_kwargsÚprepare_model_kwargsrv rv rw r’ ` s@ ÿ
|
|||
|
|
ÿÿ
|
|||
|
|
ÿþ
|
|||
|
|
|
|||
|
|
|
|||
|
|
þz&_UnslothDPOTrainer._prepare_peft_modelc C s6 |j rt|dƒr| ¡ |S dd„ }| ¡ |¡ |S )z4Prepare the gradienting checkpointing for the model.Úenable_input_require_gradsc S s | d¡ d S )NT)Úrequires_grad_)ÚmoduleÚinputÚoutputrv rv rw Úmake_inputs_require_grad s zT_UnslothDPOTrainer._prepare_gradient_checkpointing.<locals>.make_inputs_require_grad)r rŽ rÇ Úget_input_embeddingsÚregister_forward_hook)rJ rž rZ rÌ rv rv rw rÄ • s
|
|||
|
|
ûz2_UnslothDPOTrainer._prepare_gradient_checkpointingr€ Ú dataset_namec C s i }t |tƒr|j|d< d|d< tƒ ¡ �d t |tƒr#d|› d�|d< |jtfi |¤Ž}t |tƒr9d|› d�|d< |jtfd||jd œi|¤Ž}t |tƒrUd
|
|||
|
|
|› d�|d< |j| j s]| j
|
|||
|
|
n| j fd d g||j |j
|
|||
|
|
d
|
|||
|
|
dœdœ|¤Ž}W d ƒ |S 1 s|w Y |S )NÚnum_procé
|
|||
|
|
Úwriter_batch_sizezExtracting prompt in z datasetÚdesczApplying chat template to Ú fn_kwargs)rg r4 z Tokenizing ÚchosenÚrejectedF)r^ r- r. Úadd_special_tokens)Úremove_columnsrÔ )rˆ r r* r Úmain_process_firstÚmapr= r< r4 r– Ú tokenize_rowÚ process_rowr- r. )rJ r€ r^ rZ rÏ Ú
|
|||
|
|
map_kwargsrv rv rw r} § sF
|
|||
|
|
|
|||
|
|
|
|||
|
|
|
|||
|
|
ÿ
|
|||
|
|
ÿÿ
|
|||
|
|
ÿûý
|
|||
|
|
ö
|
|||
|
|
ï â z#_UnslothDPOTrainer._prepare_datasetTÚfeaturesr- r. r× c C sÊ |}|| d dd�d }|| d dd�d }|| d dd�d }|r8|j dur-|j g| }|jdur8||jg }||jg }||jg }|durO|| d… }|dur_|d|… }|d|… }|||dœS ) a©
|
|||
|
|
Tokenize a row of the dataset.
|
|||
|
|
|
|||
|
|
Args:
|
|||
|
|
features (`dict[str, str]`):
|
|||
|
|
Row of the dataset, should contain the keys `"prompt"`, `"chosen"`, and `"rejected"`.
|
|||
|
|
processing_class (`PreTrainedTokenizerBase`):
|
|||
|
|
Processing class used to process the data.
|
|||
|
|
max_prompt_length (`int` or `None`):
|
|||
|
|
Maximum length of the prompt sequence. If `None`, the prompt sequence is not truncated.
|
|||
|
|
max_completion_length (`int` or `None`):
|
|||
|
|
Maximum length of the completion sequences. If `None`, the completion sequences are not truncated.
|
|||
|
|
add_special_tokens (`bool`):
|
|||
|
|
Whether to add special tokens to the sequences. Typically used for encoder-decoder models. If `True`,
|
|||
|
|
the prompt sequence will have a bos token prepended and an eos token appended. In any case, the
|
|||
|
|
completion sequences will have an eos token appended.
|
|||
|
|
|
|||
|
|
Returns:
|
|||
|
|
`dict[str, list[int]]`:
|
|||
|
|
Tokenized sequences with the keys `"prompt_input_ids"`, `"chosen_input_ids"`, and
|
|||
|
|
`"rejected_input_ids".
|
|||
|
|
|
|||
|
|
Example:
|
|||
|
|
```python
|
|||
|
|
>>> from transformers import GPT2Tokenizer
|
|||
|
|
|
|||
|
|
>>> tokenizer = GPT2Tokenizer.from_pretrained("gpt2")
|
|||
|
|
>>> features = {"prompt": "The sky is", "chosen": " blue", "rejected": " green"}
|
|||
|
|
>>> DPOTrainer.tokenize_row(
|
|||
|
|
... features, tokenizer, max_prompt_length=3, max_completion_length=3, add_special_tokens=False
|
|||
|
|
... )
|
|||
|
|
{'prompt_input_ids': [464, 6766, 318], 'chosen_input_ids': [4171, 50256], 'rejected_input_ids': [4077, 50256]}
|
|||
|
|
```
|
|||
|
|
ÚpromptF©r× Ú input_idsrÕ rÖ N)Úprompt_input_idsÚchosen_input_idsÚrejected_input_ids)Ú bos_token_idÚ eos_token_id) rÞ r^ r- r. r× rg râ rã rä rv rv rw rÛ Ô s( *
|
|||
|
|
|
|||
|
|
ýz _UnslothDPOTrainer.tokenize_rowc
|
|||
|
|
C s" ||j }}|| d | d dd�}|d d }|d d } || d dd �d }
|
|||
|
|
|| d
|
|||
|
|
dd �d } |rI|jd ur>|jg| }|jd urI||jg }|
|
|||
|
|
|jg }
|
|||
|
|
| |jg } |d ur`|| d … }|d urp|
|
|||
|
|
d |… }
|
|||
|
|
| d |… } || |
|
|||
|
|
| d œ} d
|
|||
|
|
|v rƒ|d
|
|||
|
|
d | d
|
|||
|
|
< d|v r�|d d | d< | S )zt
|
|||
|
|
Same as `tokenize_row` but for vision models. Please refer to `tokenize_row` for more information.
|
|||
|
|
Úimagesrß F)rç Útextr× rá r Ú pixel_valuesrÕ rà rÖ N)râ ré rã rä Úpixel_attention_maskÚ image_sizes)rg rå ræ )
|
|||
|
|
rÞ r^ r- r. r× Ú processorrg Úprocessed_featuresrâ ré rã rä rË rv rv rw rÜ s8
|
|||
|
|
|
|||
|
|
üz _UnslothDPOTrainer.process_rowc C s | j d u r g d¢| _ d S d S )N)râ rã rä rë Úref_chosen_logpsÚref_rejected_logps)Ú_signature_columns©rJ rv rv rw Ú _set_signature_columns_if_neededJ s
|
|||
|
|
ÿz3_UnslothDPOTrainer._set_signature_columns_if_neededc s | j rƒ| jsƒ| jjp
|
|||
|
|
| jj}|| j| jj| jjddœ}| j t
|
|||
|
|
| j fi |¤Ž¡}g }g }t |dd�D ])}|
|
|||
|
|
|¡\}}| j ||f¡\}}| | ¡ ¡ | | ¡ ¡ tƒ | j ¡ q2t |¡ ¡ ¡ } t |¡ ¡ ¡ }
|
|||
|
|
| j jd| d�| _ | j jd|
|
|||
|
|
d�| _ d| _tƒ ¡ S ) z·
|
|||
|
|
Returns the training [`~torch.utils.data.DataLoader`].
|
|||
|
|
|
|||
|
|
Subclass of transformers.src.transformers.trainer.get_train_dataloader to precompute `ref_log_probs`.
|
|||
|
|
F©Ú
|
|||
|
|
batch_sizeÚ
|
|||
|
|
collate_fnÚ num_workersÚ
|
|||
|
|
pin_memoryÚshufflez!Train dataset reference log probs©ÚiterablerÓ rî ©ÚnameÚcolumnrï T)r2 rž rZ r3 rª r[ rá rü r… Úpreparer r\ rM Úcompute_ref_log_probsÚgather_for_metricsrk Úcpur/ Ú free_memoryrL ÚcatÚfloatÚnumpyÚ
|
|||
|
|
add_columnrH Úget_train_dataloader) rJ rô Údataloader_paramsÚ data_loaderrî rï Ú padded_batchÚref_chosen_logpÚref_rejected_logpÚall_ref_chosen_logpsÚall_ref_rejected_logpsrL rv rw r Y s8 û ÿ ÿ
|
|||
|
|
z'_UnslothDPOTrainer.get_train_dataloaderc s6 |du r
|
|||
|
|
| j du r
|
|||
|
|
tdƒ‚|dur|n| j }| jr”| js”| jjp#| jj}|| j| jj| jj ddœ}| j
|
|||
|
|
t |fi |¤Ž¡}g }g }t
|
|||
|
|
|dd�D ]!}| |¡\}} | j
|
|||
|
|
|| f¡\}} | | ¡ ¡ | | ¡ ¡ qGt |¡ ¡ ¡ }
|
|||
|
|
t |¡ ¡ ¡ } |jd|
|
|||
|
|
d�}|jd | d�}| j dur‘|| _ d
|
|||
|
|
| _tƒ j|d �S ) aé
|
|||
|
|
Returns the evaluation [`~torch.utils.data.DataLoader`].
|
|||
|
|
|
|||
|
|
Subclass of transformers.src.transformers.trainer.get_eval_dataloader to precompute `ref_log_probs`.
|
|||
|
|
|
|||
|
|
Args:
|
|||
|
|
eval_dataset (`torch.utils.data.Dataset`, *optional*):
|
|||
|
|
If provided, will override `self.eval_dataset`. If it is a [`~datasets.Dataset`], columns not accepted
|
|||
|
|
by the `model.forward()` method are automatically removed. It must implement `__len__`.
|
|||
|
|
Nz-Trainer: evaluation requires an eval_dataset.Fró z Eval dataset reference log probsrù rî rû rï T)r] )r] r� r2 rŸ rZ r3 r« r[ rá rü r… rþ r rM rÿ r rk r rL r r r r rH Úget_eval_dataloader) rJ r] rô r r rî rï r
|
|||
|
|
r r r
|
|||
|
|
r rL rv rw r ‡ s: û ÿ
|
|||
|
|
z&_UnslothDPOTrainer.get_eval_dataloaderc c sŽ � | j r| js| j | j¡ ¡ ntƒ �* | jr | j | j¡ dV | jr5| j | jp+d¡ W d ƒ dS W d ƒ dS 1 s@w Y dS )zWContext manager for handling null reference model (that is, peft adapter manipulation).Nr{ ) r— r& r… Ú unwrap_modelrž Údisable_adapterr? Ú set_adapterr% rñ rv rv rw Únull_ref_contextº s €ÿÿý÷"øz#_UnslothDPOTrainer.null_ref_contextÚbatchc
|
|||
|
|
C sÎ | j r
|
|||
|
|
t| jjjƒntƒ }t ¡ �F |�4 | jdu r8| ¡ � | j
|
|||
|
|
| j |dd�}W d ƒ n1 s2w Y n | j
|
|||
|
|
| j|dd�}W d ƒ n1 sKw Y W d ƒ n1 sZw Y |d |d fS )zfComputes log probabilities of the reference model for a single padded batch of a DPO specific dataset.NT)Ú is_ref_modelÚ chosen_logpsÚrejected_logps) rÀ r( r… ÚdeviceÚtyper? rL Úno_gradrY r Úconcatenated_forwardrž )rJ r Úcompte_ref_context_managerÚref_model_outputrv rv rw rÿ È s ÿ
|
|||
|
|
|
|||
|
|
ÿ€€ û€ z(_UnslothDPOTrainer.compute_ref_log_probsr+ c C s0 i }t j| d | d gdd�|d< t j| d | d gdd�|d< d| v r3t j| d | d gdd�|d< d| v rFt j| d | d gdd�|d< d| v rYt j| d | d gdd�|d< t| d jd | d
|
|||
|
|
jd ƒ}t t| d ||d �t| d
|
|||
|
|
||d �f¡|d < t t| d
|
|||
|
|
|dd �t| d |dd �f¡|d< |S )aÕ
|
|||
|
|
Concatenate the `chosen` and `rejected` inputs from the batch into a single tensor for both the prompt and
|
|||
|
|
completion sequences.
|
|||
|
|
|
|||
|
|
Args:
|
|||
|
|
batch (`dict[str, Union[list, torch.LongTensor]]`):
|
|||
|
|
A batch of input data. The batch must contain the following keys:
|
|||
|
|
|
|||
|
|
- `"prompt_input_ids"`: Tensor of shape `(batch_size, prompt_length)` representing the prompt input
|
|||
|
|
IDs.
|
|||
|
|
- `"chosen_input_ids"`: Tensor of shape `(batch_size, chosen_length)` representing the chosen
|
|||
|
|
completion input IDs.
|
|||
|
|
- `"rejected_input_ids"`: Tensor of shape `(batch_size, rejected_length)` representing the rejected
|
|||
|
|
completion input IDs.
|
|||
|
|
- `"prompt_pixel_values"` (optional): Tensor for pixel values, if available.
|
|||
|
|
- `"prompt_pixel_attention_mask"` (optional): Tensor for pixel attention masks, if available.
|
|||
|
|
|
|||
|
|
padding_value (`int`):
|
|||
|
|
The padding value to use for the concatenated completion sequences (`chosen_input_ids` and
|
|||
|
|
`rejected_input_ids`).
|
|||
|
|
|
|||
|
|
Returns:
|
|||
|
|
`dict[str, torch.LongTensor]`: A dictionary containing:
|
|||
|
|
|
|||
|
|
- `"prompt_input_ids"`: Concatenated prompt input IDs of shape `(2 * batch_size, prompt_length)`.
|
|||
|
|
- `"completion_input_ids"`: Concatenated chosen and rejected completion input IDs of shape `(2 *
|
|||
|
|
batch_size, max_completion_length)`.
|
|||
|
|
- `"prompt_attention_mask"`: Concatenated prompt attention masks of shape `(2 * batch_size,
|
|||
|
|
prompt_length)`.
|
|||
|
|
- `"completion_attention_mask"`: Concatenated chosen and rejected attention masks of shape `(2 *
|
|||
|
|
batch_size, max_completion_length)`.
|
|||
|
|
- `"pixel_values"` (optional): Concatenated pixel values if `"prompt_pixel_values"` are present.
|
|||
|
|
- `"pixel_attention_mask"` (optional): Concatenated pixel attention masks if
|
|||
|
|
`"prompt_pixel_attention_mask"` are present.
|
|||
|
|
|
|||
|
|
Notes:
|
|||
|
|
The completion input IDs and attention masks are padded to the maximum completion length of the chosen or
|
|||
|
|
rejected sequences.
|
|||
|
|
râ r r_ Úprompt_attention_maskré rê rë rã r` rä )Ú pad_valueÚcompletion_input_idsÚchosen_attention_maskÚrejected_attention_maskÚcompletion_attention_mask)rL r rG rc rB )r r+ rË r. rv rv rw Úconcatenated_inputsÕ s2 +
|
|||
|
|
ÿ
|
|||
|
|
ÿ þÿþÿz&_UnslothDPOTrainer.concatenated_inputsri r r rî rï r™ Ú model_outputc " C sô | j j}| |¡| j | |¡ }| |¡| j | |¡ } | jtjjkrLtj }
|
|||
|
|
| j
|
|||
|
|
r;tj | j
|
|||
|
|
v r;t | j
|
|||
|
|
tj ƒ}
|
|||
|
|
t
|
|||
|
|
| |
|
|||
|
|
ƒt
|
|||
|
|
||
|
|||
|
|
ƒ |
|
|||
|
|
} n<|| } | jr_tjdg| j| jd�}
|
|||
|
|
n|| }
|
|||
|
|
| | j j¡} |
|
|||
|
|
| j j¡}
|
|||
|
|
| |
|
|||
|
|
} | jtjjkrˆ| t |¡t | ¡ 8 } |dkr¨t | j| ¡ d| j t | j | ¡| j }�n/|dkrÏt | j| ¡ d| j t | j | ¡| j dd| j }�n|dk�rddl}| jdkràd | _| j| ¡ t | j| ¡| d| j ¡ | j | ¡ t | j | ¡| | j¡ }�nÇ|d
|
|||
|
|
k�r!t d| j| ¡}�n¶|d k�r3| dd| j d }�n¤|d k�rx|| }|| } | j| }| j| }t ||fd¡ ¡ ¡ }| j |¡ | j j }t | j| | ¡ t | j| | ¡ }�n_|d
|
|||
|
|
k�r™|| }|| }|d| j d |d| j d }�n>|dk�rÄ|| | j }|| | j }t |¡ dt | ¡ dt | ¡ }�n|dk�r|| }|| } tj |dd�\}}tj | dd�\}}|| }t | j| ¡ d| j t | j | ¡| j }nÕ|dk�r@|| } || }
|
|||
|
|
tj | dd�\}}tj |
|
|||
|
|
dd�\}}|| }t | j| ¡ d| j t | j | ¡| j }n—|dk�r\dt | j| ¡ }t | j| ¡} || }n{|dk�rzt | j| ¡}dt | j|| ¡ } || }n]|dk�r°|| } || }
|
|||
|
|
| |
|
|||
|
|
} | | j } t | | j!j" ¡} t | ¡ } t #| ¡} | d| | | }n'|dk�rÎ|d } |j$d }!| %|!¡}t &|¡}t &|¡}n t'd| j(› d�ƒ‚| j| |¡| |¡ ¡ }| j| |¡| |¡ ¡ }|||fS )a
|
|||
|
|
Compute the DPO loss for a batch of policy and reference model log probabilities.
|
|||
|
|
|
|||
|
|
Args:
|
|||
|
|
chosen_logps (`torch.FloatTensor`):
|
|||
|
|
Log probabilities of the model for the chosen responses. Shape: `(batch_size,)`.
|
|||
|
|
rejected_logps (`torch.FloatTensor`):
|
|||
|
|
Log probabilities of the model for the rejected responses. Shape: `(batch_size,)`.
|
|||
|
|
ref_chosen_logps (`torch.FloatTensor`):
|
|||
|
|
Log probabilities of the reference model for the chosen responses. Shape: `(batch_size,)`.
|
|||
|
|
ref_rejected_logps (`torch.FloatTensor`):
|
|||
|
|
Log probabilities of the reference model for the rejected responses. Shape: `(batch_size,)`.
|
|||
|
|
|
|||
|
|
Returns:
|
|||
|
|
A tuple of three tensors: `(losses, chosen_rewards, rejected_rewards)`. The losses tensor contains the DPO
|
|||
|
|
loss for each example in the batch. The `chosen_rewards` and `rejected_rewards` tensors contain the rewards
|
|||
|
|
for the chosen and rejected responses, respectively.
|
|||
|
|
r )r¼ r ri r` Úrobustr� Úexo_pairNgü©ñÒMbP?rq rr rs rt g à?ru Úaot_pairr_ Úaotrv rw ÚdiscopopÚsftÚnll_losszUnknown loss type: z§. Should be one of ['sigmoid', 'hinge', 'ipo', 'exo_pair', 'nca_pair', 'robust', 'bco_pair', 'sppo_hard', 'aot', 'aot_pair', 'discopop', 'apo_zero', 'apo_down', 'sft']))r… r re r9 r¥ r ÚALPHA_DIVERGENCEÚvaluer Ú ALPHA_DIVERGENCE_COEF_DEFAULTr§ r¦ r r) rL Útensorr¼ Ú
|
|||
|
|
JS_DIVERGENCEr ÚsoftplusÚ
|
|||
|
|
logsigmoidr7 r: Úmathri ÚlogÚrelur ÚmeanÚdetachr´ ÚupdateÚsortrZ r> Úexprc ÚexpandÚ
|
|||
|
|
zeros_liker� r™ )"rJ r r rî rï r™ r% r Úchosen_logratiosÚrejected_logratiosÚ
|
|||
|
|
alpha_coefrm Ú logratiosÚ
|
|||
|
|
ref_logratiosÚlossesr4 Úchosen_rewardsÚrejected_rewardsÚrewardsÚdeltaÚaÚbÚchosen_logratios_sortedÚ_Úrejected_logratios_sortedÚlogratios_sortedÚref_logratios_sortedÚ
|
|||
|
|
losses_chosenÚlosses_rejectedÚlog_ratio_modulationÚlogistic_componentÚ
|
|||
|
|
exp_componentÚsft_lossrô rv rv rw Údpo_loss" sä "ÿÿÿ ý
|
|||
|
|
|
|||
|
|
ÿ,þ
|
|||
|
|
|
|||
|
|
|
|||
|
|
|
|||
|
|
|
|||
|
|
|
|||
|
|
ÿ
|
|||
|
|
(
|
|||
|
|
|
|||
|
|
ÿþÿ
|
|||
|
|
ÿÿ
|
|||
|
|
ÿÿ
|
|||
|
|
|
|||
|
|
|
|||
|
|
|
|||
|
|
|
|||
|
|
|
|||
|
|
|
|||
|
|
|
|||
|
|
|
|||
|
|
|
|||
|
|
ÿ
|
|||
|
|
z_UnslothDPOTrainer.dpo_lossc ) C sR | j |¡}| j|| jd�}i }| jrd|d< d|v r!|d |d< d|v r+|d |d< d|v r5|d |d< |d }|d }| jrÜ| ¡ |d |d dd
|
|||
|
|
�}t|d |jj ƒ} |
|
|||
|
|
¡ | |d |j |d d d
|
|||
|
|
�}
|
|||
|
|
|
|
|||
|
|
j } d } | j s›| j
|
|||
|
|
d ur›| j | j
|
|||
|
|
¡}
|
|||
|
|
|
|
|||
|
|
¡ |d |d dd
|
|||
|
|
�}|
|
|||
|
|
|
|||
|
|
¡ | |d |j |d d d
|
|||
|
|
�}|j } n7| j sÒ| ¡ �( | ¡ |d |d dd
|
|||
|
|
�}|
|
|||
|
|
¡ | |d |j |d d d
|
|||
|
|
�}|j } W d ƒ n1 sÍw Y |d }| ¡ }�n¼tj|d |d fdd�}tj|d |d fdd�}tjt |¡|fdd�}| jd u�rˆ| j| d¡k �rˆ| jdk�rBt|||ƒ\}}}|d d …d | j…f }|d d …d | j…f }|d d …d | j…f }nO| jdk�rt|||ƒ\}}}|d d …| j d …f }|d d …| j d …f }|d d …| j d …f }t|||ƒ\}}}ntd| j› d�ƒ‚t|||ƒ\}}}| j�r®|jdd�d ¡ }|j d | ¡ d }||d< d|d< | j �rÛ|| ¡ d¡}|| ¡ d¡}| d¡| ¡ d¡d }||d< n||d< t!|dƒ�rê|
|
|||
|
|
¡ }nt"|| j#j$|ƒ}||fdd i|¤Ž}|j d d …d d …f } d } | j �sC| j
|
|||
|
|
d u�rC| j | j
|
|||
|
|
¡}
|
|||
|
|
t!|
|
|||
|
|
dƒ�r%|
|
|||
|
|
|
|||
|
|
¡ }nt"|
|
|||
|
|
| j#j$|
|
|||
|
|
ƒ}||fdd i|¤Ž}|j d d …d d …f } nA| j �s„t!|dƒ�rR|
|
|||
|
|
¡ }nt"|| j#j$|ƒ}| ¡ � ||fdd i|¤Ž}|j d d …d d …f } W d ƒ n 1 �sw Y t %|dk|| j&¡}|d d …dd …f }| '¡ } d } d } | j �sÝ| j
|
|||
|
|
d u�r¶| j | j
|
|||
|
|
¡}
|
|||
|
|
|
|
|||
|
|
'¡ } n| ¡ � | '¡ } W d ƒ n 1 �sÊw Y | j(} t!| d ƒ�rÛ| j)nd } | j*| j(| |t!| d ƒ�rì| j)nd | j �só| nd | j �sú| nd | j �s| nd d �} | \}!^}"}#}$}%}&}'|!|"|#|$|%|&|'d |'d d œ}(| j�r'|j+|(d < |(S )!N©r+ Tro ré rê rë r r# râ )Úattention_maskÚ return_dictr F)rá rW Úencoder_hidden_statesÚencoder_attention_maskÚ use_cacher` r_ Ú
|
|||
|
|
keep_startr� úUnknown truncation mode: 'ú/'. Should be one of ['keep_end', 'keep_start'].©Úas_tupleÚlogits_to_keepÚoutput_hidden_statesr Ú position_idsrW Ú get_decoderr[ rZ Úbias)re Ú ref_inputÚ
|
|||
|
|
ref_weightÚref_bias)Úlossr r Úmean_chosen_logitsÚmean_rejected_logitsr, rD rE Úaux_loss),r… r r$ r+ r¡ r“ Ú get_encoderrJ rŠ Údecoder_start_token_idrd Úlast_hidden_stater9 rY r ÚboolrL r r= r/ Úsizer0 r0 r1 r� r) ÚnonzeroÚminrc Úitemr1 rh ÚcumsumrŽ r rZ r6 Úwherer, Úget_output_embeddingsÚweightre r› rl ))rJ rž r Úunwrapped_modelÚconcatenated_batchÚ model_kwargsr r# Úencoder_outputsÚdecoder_input_idsÚdecoder_outputsÚ
|
|||
|
|
hidden_statesÚref_hidden_statesÚunwrapped_ref_modelÚref_encoder_outputsÚref_decoder_outputsÚlabelsÚ loss_maskrá rW Úfirst_compute_indexra rc Ú
|
|||
|
|
base_modelÚoutputsÚref_base_modelÚ ref_outputsÚmasked_input_idsÚlm_headrg rh Ú ref_lm_headÚ loss_outputri r r Úchosen_logits_meanÚrejected_logits_meanr, Ú aux_outputsrË rv rv rw Ú_compute_loss_ligerê sn ýþûýû
|
|||
|
|
ýû ó ÿþ þ ÿ
|
|||
|
|
|
|||
|
|
ÿþý
|
|||
|
|
|
|||
|
|
ÿÿþý
|
|||
|
|
|
|||
|
|
ÿþý ú
|
|||
|
|
|
|||
|
|
|
|||
|
|
ÿù ýø
|
|||
|
|
|
|||
|
|
z&_UnslothDPOTrainer._compute_loss_ligerr c / C s |d j d }| j|| jd�}ddi}| jrd|d< d|v r$|d |d< d |v r.|d |d < d
|
|||
|
|
|v r8|d
|
|||
|
|
|d
|
|||
|
|
< |d }|d }|d } |d
|
|||
|
|
}
|
|||
|
|
| jrh| } | j| |
|
|||
|
|
dk< |d,||| dœ|¤Ž} | j}
|
|||
|
|
|
|
|||
|
|
¡ }�n1tj || fdd�}tj ||
|
|||
|
|
fdd�}tj t
|
|||
|
|
|¡|
|
|||
|
|
fdd�}| j du�r | j | d¡k �r | j
|
|||
|
|
dkrÅt|||ƒ\}}}|dd…d| j …f }|dd…d| j …f }|dd…d| j …f }nO| j
|
|||
|
|
dk�rt|||ƒ\}}}|dd…| j d…f }|dd…| j d…f }|dd…| j d…f }t|||ƒ\}}}ntd| j
|
|||
|
|
› d�ƒ‚t|||ƒ\}}}| j�r1|jdd�d ¡ }|j d | ¡ d }||d< d|d< | j�r^|| ¡ d¡}|| ¡ d¡}| d¡| ¡ d¡d }||d< n||d< ||fi |¤Ž} | j}
|
|||
|
|
tj|ddd �} tj|ddd � ¡ }| j�r™| dd…| d…f } |dd…| d…f }|
|
|||
|
|
j dd … | j dd … k�r¸| j d }|
|
|||
|
|
dd…| d…f }
|
|||
|
|
d| | < t|
|
|||
|
|
| ƒ}d|| < tj|ddd �}| j�rí|j \}}tj||| jj| jj d �}||| ¡ < |}|dd…dd…f d¡}i }| j �rOt ¡ �B t j!|
|
|||
|
|
dd�}tj"d | dd�}|| } | | d¡| d¡ } | d|… } | |d… } tj#t $| | ¡dd �|d < W d ƒ n 1 �sJw Y | j%j&du�s\d!| j'v �r˜| j�sj|
|
|||
|
|
d|…dd…f n|
|
|||
|
|
d|… } | j�s~| d|…dd…f n| d|… }!t j(tj)| dd"�tj)|!dd"�dd#�|d$< d%| j'v �r¥|| d¡ }| j%j*du�r|�s|j dd�}"|"d|… }#|"|d… }$t |#|$¡}%tj |%|%gdd�}%| d¡}tj+||jd&� ,|¡}||% d¡k }&||" d¡k }'|&|'@ -¡ }(|& |'@ -¡ })||( j dd�}*||) j dd�}+|*| j%j*|+ }|d|… |d'< ||d… |d(< | j�rX|dkjdd�d | },|
|
|||
|
|
dd|,…f |dd|,…f .¡ }-|
|
|||
|
|
d|,d…f |d|,d…f .¡ }.n |
|
|||
|
|
d|… |d|… .¡ }-|
|
|||
|
|
|d… ||d… .¡ }.|-|d)< |.|d*< | j�r…| j/|d+< |S )-a
|
|||
|
|
Runs the given model on the given batch of inputs, concatenating the chosen and rejected inputs together.
|
|||
|
|
|
|||
|
|
We do this to avoid doing two forward passes, because it's faster for FSDP.
|
|||
|
|
|
|||
|
|
Args:
|
|||
|
|
model:
|
|||
|
|
Model to run the forward pass on.
|
|||
|
|
batch:
|
|||
|
|
Batch of input data.
|
|||
|
|
is_ref_model:
|
|||
|
|
Whether this method is being called for the reference model. If `True`, length desensitization is not
|
|||
|
|
applied.
|
|||
|
|
râ r rV r[ FTro ré rê rë r r r# )rá rW r„ r` r_ Nr\ r� r] r^ r_ ra rb rc rW rZ )ÚshiftsÚdimsr� )r r¼ )rG Úpolicy_weightsr+ )Úend_dim)rj r, rr ©r r r rj rk rl rv )0rc r$ r+ r¡ r“ r, rm rp rL r r= r/ rq r0 r0 r1 r� r) rr rs rt r1 rh ru ÚrollrI Úzerosr r¼ Úsumr; r r Ú log_softmaxrj Úclampr; rZ r< r™ Ú
|
|||
|
|
cross_entropyÚflattenr= ÚarangeÚ expand_asr r7 rl )/rJ rž r r Ú num_examplesrz r{ râ r r r# r„ rˆ rm r… rá rW r† ra rc Úseq_lenru rô Úper_token_logps_Ú all_logpsrË ÚlogprobsÚweights_adjustment_factorÚper_token_logps_adjustedÚ all_weightsÚchosen_weightsÚrejected_weightsÚ
|
|||
|
|
chosen_logitsÚ
|
|||
|
|
chosen_labelsÚcompletion_lengthsÚchosen_lengthsÚrejected_lengthsÚpublic_lengthsÚld_maskÚmaskÚ
|
|||
|
|
front_maskÚ rear_maskÚ front_logpsÚ
|
|||
|
|
rear_logpsÚ split_idxrj rk rv rv rw r Ñ s ýü þ
|
|||
|
|
ÿ
|
|||
|
|
|
|||
|
|
|
|||
|
|
|
|||
|
|
|
|||
|
|
|
|||
|
|
ÿ
|
|||
|
|
ø
|
|||
|
|
((
|
|||
|
|
ÿ
|
|||
|
|
$&
|
|||
|
|
z'_UnslothDPOTrainer.concatenated_forwardr| Ú
|
|||
|
|
train_eval)r| rƒ c C sÜ i }| j jr| ||¡}|d }|d }|d }n[| ||¡}d|v r0d|v r0|d } |d }
|
|||
|
|
n| |¡\} }
|
|||
|
|
d}d}d}t| jƒD ]1\} } | |d |d | |
|
|||
|
|
| |¡\}
|
|||
|
|
}}| jr_| j| nd }||
|
|||
|
|
| }||| }||| }qB||k ¡ }| j j
|
|||
|
|
d
|
|||
|
|
urŠ|| j j
|
|||
|
|
|d }| j r“||d }| j rŸ|| j
|
|||
|
|
|d
|
|||
|
|
}|dkr¥dnd}| j |¡ ¡ ¡ ||› d�< | j |¡ ¡ ¡ ||› d�< | j |¡ ¡ ¡ ||› d�< | j || ¡ ¡ ¡ ||› d�< | j |d ¡ ¡ ¡ ¡ ||› d�< | j |d ¡ ¡ ¡ ¡ ||› d�< | j |d ¡ ¡ ¡ ¡ ||› d�< | j |d ¡ ¡ ¡ ¡ ||› d�< | j j
|
|||
|
|
d
|
|||
|
|
u�s>d| jv �rQ| j |d ¡ ¡ ¡ ¡ ||› d �< | j �rh| j |d
|
|||
|
|
¡ ¡ ¡ ¡ ||› d
|
|||
|
|
�< | ¡ |fS ) zWCompute the DPO loss and other metrics for the given batch of inputs for train or test.ri rD rE rî rï r r r rˆ Nr, r• rl rƒ Úeval_r“ zrewards/chosenzrewards/rejectedzrewards/accuracieszrewards/marginsz logps/chosenzlogps/rejectedrj z
|
|||
|
|
logits/chosenrk zlogits/rejectedr+ )rZ r5 r’ r rÿ Ú enumerater™ rU r? r r< r; r¡ r¢ r… r r7 rt r8 )rJ rž r r¸ Úmetricsr% rC rD rE rî rï Úidxr™ Ú_lossesÚ_chosen_rewardsÚ_rejected_rewardsrx Úreward_accuraciesÚprefixrv rv rw Úget_batch_loss_metrics® sn
|
|||
|
|
|
|||
|
|
|
|||
|
|
ú
|
|||
|
|
ÿ ÿ ÿ ÿ ÿ ÿ ÿ z)_UnslothDPOTrainer.get_batch_loss_metricsÚinputsc C s~ | j r
|
|||
|
|
t| jjjƒntƒ }|� | j||dd�\}}W d ƒ n1 s$w Y | | jj¡}| j |dd� |r=||fS |S )Nr| ©r¸ )
|
|||
|
|
rÀ r( r… r r r? r re rZ Ú
|
|||
|
|
store_metrics)rJ rž rà Úreturn_outputsÚnum_items_in_batchÚ compute_loss_context_managerri r» rv rv rw Ú compute_loss s ÿ ÿz _UnslothDPOTrainer.compute_lossc C s0 | j r
|
|||
|
|
t| jjjƒntƒ }|�] |j|d |d | jd| jd�}d|v r)|d }n<| j du rT|
|
|||
|
|
¡ � | j j|d |d | jd| jd�}W d ƒ n1 sNw Y n| j j|d |d | jd| jd�}W d ƒ n1 sow Y t || j| jƒ}| j
|
|||
|
|
j|dd�}t || j| jƒ}| j
|
|||
|
|
j|dd�}||fS )zRGenerate samples from the model and reference model for the given batch of inputs.râ r T)rá rW r/ Ú do_samplerf Ú
|
|||
|
|
ref_outputN)Úskip_special_tokens)rÀ r( r… r r r? Úgenerater/ r+ rY r rž rB r^ Ú batch_decode)rJ rž r Úgenerate_context_managerÚ
|
|||
|
|
policy_outputrË Úpolicy_output_decodedÚref_output_decodedrv rv rw Úgenerate_from_model_and_ref sJ ÿû
|
|||
|
|
|
|||
|
|
|
|||
|
|
û ÿ€ û€ é z._UnslothDPOTrainer.generate_from_model_and_refr© Ú ignore_keysc s ˆ d u rt |dƒrt|jdg ƒ‰ ng ‰ | jr t| jjjƒntƒ }t
|
|||
|
|
¡ �$ |� | j ||dd�\}}W d ƒ n1 s<w Y W d ƒ n1 sKw Y | j |dd� |r`|
|
|||
|
|
¡ d d fS |d |d dœ}‡ fdd „| ¡ D ƒ} t j| | jjd
|
|||
|
|
�} t j| jd | jjd
|
|||
|
|
�}
|
|||
|
|
|
|
|||
|
|
¡ | |
|
|||
|
|
fS ) NrŠ Úkeys_to_ignore_at_inferencerƒ rÄ úeval_logits/chosenúeval_logits/rejected)rÖ r× c s g | ]
|
|||
|
|
\}}|ˆ vr|‘qS rv rv )r~ ÚkÚv©rÔ rv rw Ú
|
|||
|
|
<listcomp>l s z6_UnslothDPOTrainer.prediction_step.<locals>.<listcomp>r— r )rŽ r rŠ rÀ r( r… r r r? rL r r rÅ r8 r© r0 r™ rc ) rJ rž rà r© rÔ Úprediction_context_managerri r» Ú logits_dictrm r„ rv rÚ rw Úprediction_stepM s*
|
|||
|
|
ÿ ÿ€ þz"_UnslothDPOTrainer.prediction_stepr» c C s* | ¡ D ]\}}| j| | |¡ qd S ry )r© r¤ rk )rJ r» r¸ r r. rv rv rw rÅ r s ÿz _UnslothDPOTrainer.store_metricsrƒ Ú
|
|||
|
|
dataloaderÚ descriptionÚmetric_key_prefixc sü | j rrt|jƒ}tjt|ƒ| jjd�}|j |¡}| |¡} |
|
|||
|
|
| ¡} | | j | ¡\}
|
|||
|
|
} t
|
|||
|
|
jg d¢dd„ t|d |
|
|||
|
|
| ƒD ƒd�} d| jjv rU| jjrUt dtj| d �i¡ d
|
|||
|
|
| jjv ratd | d � d
|
|||
|
|
| jjv rr| jjrrtj| dd� tƒ |||||¡}
|
|||
|
|
|
|
|||
|
|
S )zÞ
|
|||
|
|
Overriding built-in evaluation loop to store metrics for each batch. Prediction/evaluation loop, shared by
|