329 lines
34 KiB
Plaintext
329 lines
34 KiB
Plaintext
|
o
|
|||
|
|
<—°hÝ ã @ sB d Z ddlmZ ddlZddlmZ ddlmZ ddlmZm Z m
|
|||
|
|
Z
|
|||
|
|
m Z m Z m
|
|||
|
|
Z
|
|||
|
|
mZmZ ddlmZmZmZmZmZmZmZmZmZmZmZm Z m Z m
|
|||
|
|
Z
|
|||
|
|
m Z m Z m Z m!Z!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z)m Z m*Z*m+Z+m,Z,m-Z-m.Z.m/Z/m0Z0m1Z1m2Z2m3Z3m4Z4m5Z5m6Z6m7Z7m8Z8m9Z9m:Z:m;Z;m<Z<m=Z=m>Z>m?Z?m@Z@mAZAmZmBZBmCZCmDZDmEZEmFZFmGZGmHZHmIZImJZJmKZKmZmLZLmMZMm
|
|||
|
|
Z
|
|||
|
|
m"Z"m'Z'm;Z;mDZDmZ ddlDZDddlT ddlNmOZOmPZP dd lQmRZR ddlZddlSZBdd
|
|||
|
|
lTmCZC ddlmZ dd lUmVZVmWZX d d
|
|||
|
|
d d
|
|||
|
|
d
|
|||
|
|
dœZYejZd d eYd�dd„ ƒZ[eOG dd„ de ƒƒZ\ G dd„ de'ƒZ]G dd„ de]ƒZ^dS )z9
|
|||
|
|
2025.8.9
|
|||
|
|
2025.8.10
|
|||
|
|
4.55.4
|
|||
|
|
0.21.0
|
|||
|
|
__UNSLOTH_VERSIONING__
|
|||
|
|
é )ÚTensorN)Ú
|
|||
|
|
functional)ÚAnyÚListÚOptionalÚTupleÚUnionÚDictÚSetÚCallable)GÚ AcceleratorÚBaseImageProcessorÚCallbackHandlerÚDEFAULT_CALLBACKSÚDEFAULT_PROGRESS_CALLBACKÚDataCollatorWithPaddingÚ
|
|||
|
|
DataLoaderÚDatasetÚExportableStateÚFeatureExtractionMixinÚGenerationConfigÚINVALID_LOGPROBÚOnlineTrainerStater Ú PPOConfigÚ
|
|||
|
|
PPOTrainerÚPathÚ
|
|||
|
|
PeftConfigÚ PeftModelÚPolicyAndValueWrapperÚPreTrainedTokenizerBaseÚPrinterCallbackÚProcessorMixinÚTrainerÚTrainerCallbackÚTrainerControlr Úbatch_generationÚ broadcastÚcontextmanagerÚcreate_reference_modelÚ defaultdictÚdisable_dropout_in_modelÚ empty_cacheÚ exact_divÚfirst_true_indicesÚforwardÚ
|
|||
|
|
gather_objectÚgcÚgenerate_model_cardÚget_comet_experiment_urlÚget_peft_modelÚ#get_reporting_integration_callbacksÚ
|
|||
|
|
get_rewardÚis_peft_availableÚis_rich_availableÚis_wandb_availableÚ log_table_to_comet_experimentÚ masked_meanÚ
|
|||
|
|
masked_whitenÚmathÚnnÚnpÚ nullcontextÚosÚpdÚpeft_module_casting_to_bf16Úprepare_deepspeedÚprint_rich_tableÚselective_log_softmaxÚtextwrapÚtimeÚtorchÚtruncate_responseÚunwrap_model_for_generationr r r" r6 r@ rH )Ú*)Ú dataclassÚfield)ÚVersion)r? )ÚDataCollatorForSeq2SeqÚ DataCollatorForLanguageModelingTF)Úepilogue_fusionÚ max_autotuneÚ
|
|||
|
|
shape_paddingz
|
|||
|
|
trace.enabledztriton.cudagraphs)ÚdynamicÚ fullgraphÚoptionsc
|
|||
|
|
C s¾ t j| d| jd ¡ddd�}t j| d¡ddd�}g }t||ƒD ](\}}| t j¡}t j|d| d¡d� d¡}t j
|
|||
|
|
|dd�}|| } | | ¡ q! t |¡}| | jd | jd f¡}|S )Néÿÿÿÿé r )ÚchunksÚdim)rZ Úindex©rZ é )
|
|||
|
|
rH ÚchunkÚreshapeÚshapeÚzipÚtoÚfloat32ÚgatherÚ unsqueezeÚsqueezeÚ logsumexpÚappendÚconcat)
|
|||
|
|
Úlogitsr[ Úchunked_logitsÚ
|
|||
|
|
chunked_indexÚall_per_token_logpsÚ chunk_logitsÚ chunk_indexÚselected_logitsÚlogsumexp_valuesÚper_token_logps© rs úQ/workspace/DS-LLM-TEMPLATE-FINETUNING/unsloth_compiled_cache/UnslothPPOTrainer.pyÚ chunked_selective_log_softmax" s
|
|||
|
|
ru c s e Zd ZU dZedddid�Zee ed< edddid�Z ee
|
|||
|
|
ed <
|
|||
|
|
|
|||
|
|
|
|||
|
|
|
|||
|
|
|
|||
|
|
|
|||
|
|
|
|||
|
|
|
|||
|
|
|
|||
|
|
|
|||
|
|
|
|||
|
|
|
|||
|
|
|
|||
|
|
|
|||
|
|
|
|||
|
|
|
|||
|
|
|
|||
|
|
|
|||
|
|
!
|
|||
|
|
|
|||
|
|
|
|||
|
|
"
|
|||
|
|
|
|||
|
|
|
|||
|
|
" # $
|
|||
|
|
|
|||
|
|
%
|
|||
|
|
|
|||
|
|
|
|||
|
|
&
|
|||
|
|
|
|||
|
|
|
|||
|
|
! "
|
|||
|
|
' (
|
|||
|
|
|
|||
|
|
|
|||
|
|
|
|||
|
|
|
|||
|
|
|
|||
|
|
|
|||
|
|
) * + , - . -
|
|||
|
|
/ 0 1 1 2 d5‡ fd3d4„ Z ‡ Z S )6ÚUnslothPPOConfigaþ
|
|||
|
|
|
|||
|
|
|
|||
|
|
Configuration class for the [`PPOTrainer`].
|
|||
|
|
|
|||
|
|
This class includes only the parameters that are specific to PPO training. For a full list of training arguments,
|
|||
|
|
please refer to the [`~transformers.TrainingArguments`] and [`OnPolicyConfig`] documentation. Note that default
|
|||
|
|
values in this class may differ from those in [`~transformers.TrainingArguments`].
|
|||
|
|
|
|||
|
|
Using [`~transformers.HfArgumentParser`] we can turn this class into
|
|||
|
|
[argparse](https://docs.python.org/3/library/argparse#module-argparse) arguments that can be specified on the
|
|||
|
|
command line.
|
|||
|
|
|
|||
|
|
Parameters:
|
|||
|
|
exp_name (`str`, *optional*, defaults to `os.path.basename(__file__)[:-3]`):
|
|||
|
|
Name of this experiment.
|
|||
|
|
reward_model_path (`str`, *optional*, defaults to `"EleutherAI/pythia-160m"`):
|
|||
|
|
Path to the reward model.
|
|||
|
|
model_adapter_name (`str` or `None`, *optional*, defaults to `None`):
|
|||
|
|
Name of the train target PEFT adapter, when using LoRA with multiple adapters.
|
|||
|
|
ref_adapter_name (`str` or `None`, *optional*, defaults to `None`):
|
|||
|
|
Name of the reference PEFT adapter, when using LoRA with multiple adapters.
|
|||
|
|
num_ppo_epochs (`int`, *optional*, defaults to `4`):
|
|||
|
|
Number of epochs to train.
|
|||
|
|
whiten_rewards (`bool`, *optional*, defaults to `False`):
|
|||
|
|
Whether to whiten the rewards.
|
|||
|
|
kl_coef (`float`, *optional*, defaults to `0.05`):
|
|||
|
|
KL coefficient.
|
|||
|
|
kl_estimator (`Literal["k1", "k3"]`, *optional*, defaults to `"k1"`):
|
|||
|
|
Which estimator for KL-Divergence to use from [Approximating KL
|
|||
|
|
Divergence](http://joschu.net/blog/kl-approx.html). Defaults to "k1", a straightforward, unbiased
|
|||
|
|
estimator. Can be set to "k3", an unbiased estimator with lower variance which "appears to be a strictly
|
|||
|
|
better estimator". Cannot be set to "k2", as it is used for logging purposes.
|
|||
|
|
cliprange (`float`, *optional*, defaults to `0.2`):
|
|||
|
|
Clip range.
|
|||
|
|
vf_coef (`float`, *optional*, defaults to `0.1`):
|
|||
|
|
Value function coefficient.
|
|||
|
|
cliprange_value (`float`, *optional*, defaults to `0.2`):
|
|||
|
|
Clip range for the value function.
|
|||
|
|
gamma (`float`, *optional*, defaults to `1.0`):
|
|||
|
|
Discount factor.
|
|||
|
|
lam (`float`, *optional*, defaults to `0.95`):
|
|||
|
|
Lambda value for GAE.
|
|||
|
|
ds3_gather_for_generation (`bool`, *optional*, defaults to `True`):
|
|||
|
|
This setting applies to DeepSpeed ZeRO-3. If enabled, the policy model weights are gathered for generation,
|
|||
|
|
improving generation speed. However, disabling this option allows training models that exceed the VRAM
|
|||
|
|
capacity of a single GPU, albeit at the cost of slower generation.
|
|||
|
|
|
|||
|
|
NÚhelpzvLLM SamplingParams)ÚdefaultÚmetadataÚvllm_sampling_paramsrW z8Chunk size to reduce memory usage. -1 is most efficient.Úunsloth_num_chunksFÚnorX é r éú ç-C ëâ6
|
|||
|
|
?ç{®Gáz„?çÍÌÌÌÌÌì?ç+‡ÙÎ÷ï?ç:Œ0âŽyE>ç ð?ç @Úlinearçš™™™™™¹?ÚpassiveÚwarningTÚstepsr] éô éO
|
|||
|
|
ÚO1ÚautoÚ ç Ú
|
|||
|
|
adamw_8bitÚlengthÚ
|
|||
|
|
every_saveÚlasté é@ é
|
|||
|
|
é5 çffffffæ?úEleutherAI/pythia-160mÚ
|
|||
|
|
ppo_configçš™™™™™©?Úk1çš™™™™™É?çffffffî?c£ ¥ sv |dk r t d|› d�ƒ‚|dkrtd|› d�ƒ‚|d u r(|#dkr(|$dkr(d}d }#|�d u r:d
|
|||
|
|
d lm}¤ t|¤ƒ d d
|
|||
|
|
ƒ}�|‰d
|
|||
|
|
krBtdƒ‚|‰dkrJtdƒ‚tƒ jd±i d|“d|“d|“d|“d|“d|“d|“d|“d| “d|
|
|||
|
|
“d| “d | “d |
|
|||
|
|
“d |“d |“d |“d!|“d"|“d#|“d$|“d%|“d&|“d'|“d(|“d)|“d*|“d+|“d,| “d-| “d.| “d/| “d0| “d1|!“d2|"“d3|#“d4|$“d5|%“d6|&“d7|'“d8|(“d9|)“d:|*“d;|+“d<|,“d=|-“d>|.“d?|/“d@|0“dA|1“dB|2“dC|3“dD|4“dE|5“dF|6“dG|7“dH|8“dI|9“dJ|:“dK|;“dL|<“dM|=“dN|>“dO|?“dP|@“dQ|A“dR|B“dS|C“dT|D“dU|E“dV|F“dW|G“dX|H“dY|I“dZ|J“d[|K“d\|L“d]|M“d^|N“d_|O“d`|P“da|Q“db|R“dc|S“dd|T“de|U“df|V“dg|W“dh|X“di|Y“dj|Z“dk|[“dl|\“dm|]“dn|^“do|_“dp|`“dq|a“dr|b“ds|c“dt|d“du|e“dv|f“dw|g“dx|h“dy|i“dz|j“d{|k“d||l“d}|m“d~|n“d|o“d€|p“d�|q“d‚|r“dƒ|s“d„|t“d…|u“d†|v“d‡|w“dˆ|x“d‰|y“dŠ|z“d‹|{“dŒ||“d�|}“dŽ|~“d�|“d�|€“d‘|�“d’|‚“d“|ƒ“d”|„“d•|…“d–|†“d—|‡“d˜|ˆ“d™|‰“dš|Š“d›|‹“dœ|Œ“d�|�“dž|Ž“dŸ|�“d |�“d¡|‘“d¢|’“d£|““d¤|”“d¥|•“d¦|–“d§|—“d¨|˜“d©|™“dª|š“d«|›“d¬|œ“d|�“d®|ž“d¯|Ÿ“d°| “|£¤Ž |¡| _|¢| _ d S )²NçH¯¼šò×z>z Unsloth: Your learning rate of `zi` is too small and less than 1e-7! Consider increasing it, otherwise gradient updates will be close to 0!r] za` is way too larger > 1! Consider decreasing it to 1e-1, otherwise gradient updates will explode!rŠ r‹ Ú unsloth_training_checkpointsr| r )Ú cpu_countrX r} zUUnsloth: Please set a positive non-zero temperature since your results will be wrong.r— zgUnsloth: Please set a positive non-zero temperature less than 10, since sampling will be quite erratic.Ú
|
|||
|
|
output_dirÚoverwrite_output_dirÚdo_trainÚdo_evalÚ
|
|||
|
|
do_predictÚ
|
|||
|
|
eval_strategyÚprediction_loss_onlyÚper_device_train_batch_sizeÚper_device_eval_batch_sizeÚper_gpu_train_batch_sizeÚper_gpu_eval_batch_sizeÚgradient_accumulation_stepsÚeval_accumulation_stepsÚ
|
|||
|
|
eval_delayÚtorch_empty_cache_stepsÚ
|
|||
|
|
learning_rateÚ weight_decayÚ
|
|||
|
|
adam_beta1Ú
|
|||
|
|
adam_beta2Ú adam_epsilonÚ
|
|||
|
|
max_grad_normÚnum_train_epochsÚ max_stepsÚlr_scheduler_typeÚ warmup_ratioÚ warmup_stepsÚ log_levelÚlog_level_replicaÚlog_on_each_nodeÚ logging_dirÚlogging_strategyÚlogging_first_stepÚ
|
|||
|
|
logging_stepsÚlogging_nan_inf_filterÚ
|
|||
|
|
save_strategyÚ
|
|||
|
|
save_stepsÚsave_total_limitÚsave_safetensorsÚsave_on_each_nodeÚsave_only_modelÚ'restore_callback_states_from_checkpointÚno_cudaÚuse_cpuÚuse_mps_deviceÚseedÚ data_seedÚ
|
|||
|
|
jit_mode_evalÚuse_ipexÚbf16Úfp16Úfp16_opt_levelÚhalf_precision_backendÚbf16_full_evalÚfp16_full_evalÚtf32Ú
|
|||
|
|
local_rankÚ ddp_backendÚ
|
|||
|
|
tpu_num_coresÚtpu_metrics_debugÚdebugÚdataloader_drop_lastÚ
|
|||
|
|
eval_stepsÚdataloader_num_workersÚdataloader_prefetch_factorÚ
|
|||
|
|
past_indexÚrun_nameÚ disable_tqdmÚremove_unused_columnsÚ label_namesÚload_best_model_at_endÚmetric_for_best_modelÚgreater_is_betterÚignore_data_skipÚfsdpÚfsdp_min_num_paramsÚ fsdp_configÚ"fsdp_transformer_layer_cls_to_wrapÚaccelerator_configÚ deepspeedÚlabel_smoothing_factorÚoptimÚ
|
|||
|
|
optim_argsÚ adafactorÚgroup_by_lengthÚlength_column_nameÚ report_toÚddp_find_unused_parametersÚddp_bucket_cap_mbÚddp_broadcast_buffersÚdataloader_pin_memoryÚ dataloader_persistent_workersÚskip_memory_metricsÚuse_legacy_prediction_loopÚ push_to_hubÚresume_from_checkpointÚ hub_model_idÚ hub_strategyÚ hub_tokenÚhub_private_repoÚhub_always_pushÚ hub_revisionÚgradient_checkpointingÚ gradient_checkpointing_kwargsÚinclude_inputs_for_metricsÚeval_do_concat_batchesÚ fp16_backendÚpush_to_hub_model_idÚpush_to_hub_organizationÚpush_to_hub_tokenÚ
|
|||
|
|
mp_parametersÚauto_find_batch_sizeÚfull_determinismÚ torchdynamoÚ ray_scopeÚ ddp_timeoutÚ
|
|||
|
|
torch_compileÚtorch_compile_backendÚtorch_compile_modeÚinclude_tokens_per_secondÚ include_num_input_tokens_seenÚneftune_noise_alphaÚoptim_target_modulesÚbatch_eval_metricsÚ
|
|||
|
|
eval_on_startÚuse_liger_kernelÚliger_kernel_configÚeval_use_gather_objectÚ average_tokens_across_devicesÚdataset_num_procÚnum_mini_batchesÚtotal_episodesÚ local_rollout_forward_batch_sizeÚnum_sample_generationsÚresponse_lengthÚ
|
|||
|
|
stop_tokenÚ
|
|||
|
|
stop_token_idÚ temperatureÚmissing_eos_penaltyÚsft_model_pathÚ
|
|||
|
|
world_sizeÚnum_total_batchesÚmicro_batch_sizeÚlocal_batch_sizeÚ
|
|||
|
|
batch_sizeÚlocal_mini_batch_sizeÚmini_batch_sizeÚexp_nameÚreward_model_pathÚmodel_adapter_nameÚref_adapter_nameÚnum_ppo_epochsÚwhiten_rewardsÚkl_coefÚ kl_estimatorÚ cliprangeÚvf_coefÚcliprange_valueÚgammaÚlamÚds3_gather_for_generationrs )
|
|||
|
|
ÚFloatingPointErrorÚ
|
|||
|
|
OverflowErrorÚmultiprocessingr¢ ÚmaxÚ MathErrorÚsuperÚ__init__rz r{ )¥Úselfr£ r¤ r¥ r¦ r§ r¨ r© rª r« r¬ r r® r¯ r° r± r² r³ r´ rµ r¶ r· r¸ r¹ rº r» r¼ r½ r¾ r¿ rÀ rÁ r rà rÄ rÅ rÆ rÇ rÈ rÉ rÊ rË rÌ rÍ rÎ rÏ rÐ rÑ rÒ rÓ rÔ rÕ rÖ r× rØ rÙ rÚ rÛ rÜ rÝ rÞ rß rà rá râ rã rä rå ræ rç rè ré rê rë rì rí rî rï rð rñ rò ró rô rõ rö r÷ rø rù rú rû rü rý rþ rÿ r r r r r r r r r r r
|
|||
|
|
r r r
|
|||
|
|
r r r r r r r r r r r r r r r r r r r r! r" r# r$ r% r& r' r( r) r* r+ r, r- r. r/ r0 r1 r2 r3 r4 r5 r6 r7 r8 r9 r: r; r< r= r> r? r@ rA rB rz r{ Úkwargsr¢ ©Ú __class__rs rt rI n s& ( ÿþýüûúùø ÷
|
|||
|
|
ö õ ô
|
|||
|
|
óòñðïîíìëêéèçæå ä ã â á à!ß"Þ#Ý$Ü%Û&Ú'Ù(Ø)×*Ö+Õ,Ô-Ó.Ò/Ñ0Ð1Ï2Î3Í4Ì5Ë6Ê7É8È9Ç:Æ;Å<Ä=Ã>Â?Á@ÀA¿B¾C½D¼E»FºG¹H¸I·J¶KµL´M³N²O±P°Q¯R®ST¬U«VªW©X¨Y§Z¦[¥\¤]£^¢_¡` aŸbžc�dœe›fšg™h˜i—j–k•l”m“n’o‘p�q�rŽs�tŒu‹vŠw‰xˆy‡z†{…|„}ƒ~‚� �ÿ �þ �ý �ü �û �ú �ù �ø �÷
|
|||
|
|
�ö �õ �ô
|
|||
|
|
�ó �ò �ñ �ð �ï �î �í �ì �ë �ê �é �è �ç �æ �å �ä �ã �â �á �à ! �ß "
|
|||
|
|
zUnslothPPOConfig.__init__)¢NNFFFr| FrX rX NNr} r} r r~ r r€ r� r‚ rƒ r„ r… rW r† r‡ r rˆ r‰ TNrŠ Fr] FrŠ r‹ NTFFFFFFrŒ rŒ FFFFr� rŽ FFNrW NNFr� FNr NrW NNTNFNNFr� r NNNNr� r‘ NFFr’ NNNNTFTFFNNr“ NNFNFNFTrŽ NNNr� TFNr” r• FNNFFNNFFFNFTNr] Nr– r— r˜ NNr™ Nrš NNNNNNNr› rš NNrX Frœ r� rž r‡ rž r„ rŸ TNrW )
|
|||
|
|
Ú__name__Ú
|
|||
|
|
__module__Ú __qualname__Ú__doc__rM rz r r Ú__annotations__r{ ÚintrI Ú
|
|||
|
|
__classcell__rs rs rL rt rv 3 s\
|
|||
|
|
0þþ �Ürv c sP e Zd ZddgZ d,dedeeeee e
|
|||
|
|
f de j dee j d e j d
|
|||
|
|
e
|
|||
|
|
d e j d ee d
|
|||
|
|
eee
|
|||
|
|
eee
|
|||
|
|
f f deejjejjjf deee ded ddfdd„Zdefdd„Zdefdd„Z e dd„ ƒZ d-d ee d e f‡ fd d „
|
|||
|
|
Z d d!„ Z!d.d"e fd#d$„Z"‡ fd%d&„Z# d/d'ee d(ee d)eeee df fd*d+„Z$‡ Z%S )0Ú_UnslothPPOTrainerÚtrlÚppoN©NNÚargsÚprocessing_classÚmodelÚ ref_modelÚ reward_modelÚ
|
|||
|
|
train_datasetÚ value_modelÚ
|
|||
|
|
data_collatorÚ eval_datasetÚ
|
|||
|
|
optimizersÚ callbacksÚ peft_configr Úreturnc
|
|||
|
|
C s\ ||u rt dƒ‚|| _|| _|| _|d u rt| jƒ}|jr$|jr$t dƒ‚|jr?|jdkr6|j | jj_| _nt d|j› d�ƒ‚|j | jj_| _| jj dvrRt dƒ‚t
|
|||
|
|
ƒ s]| d ur]t dƒ‚t
|
|||
|
|
ƒ r†| d ur†t | jt
|
|||
|
|
ƒrp| j ¡ | _t| j| ƒ| _|jr†t| jd d
|
|||
|
|
ƒr†t| jƒ t
|
|||
|
|
ƒ oŽt | jt
|
|||
|
|
ƒ| _|j| _|j| _|rž|| _n
|
|||
|
|
| jr¥d | _nt| jƒ| _|| _|| _t|ƒ| _|| _ || _ | | _ |
|
|||
|
|
\| _ | _ d | _!|j"d u rÖt#|j$| j ƒ|_"t%|j&d �}
|
|||
|
|
|
|
|||
|
|
| _'|
|
|||
|
|
j(|_)|j*|j& |_+t#|j*|j) ƒ|_,t#|j+|j) ƒ|_-t.|j-|j/d ƒ|_0t.|j+|j/d
|
|||
|
|
ƒ|_1|j2�r!|j1dk�s!J d|j1› d�ƒ‚t3 4|j"|j- ¡|_5t6j7t#t8 8¡ ƒ|
|
|||
|
|
j9d�}t:|dƒ ;¡ }|j<› d|j=› d|› �|_>|j=|
|
|||
|
|
j?d | _@|jAdk�rdtBd|j5|jA ƒ| _C|j+| _D| j| j| j | jfD ] }|d u�r}tE|ƒ �qrtF| j| j ƒ| _G| jjH| jG_H| jI|j5d� tJtK| jjLƒ }| d u �r£|n|| | _MtN| jM| jG| j| j | j ƒ| _O| P| jjQ�r¿tRntS¡ tTƒ | _UtV| W¡ | X¡ dd„ | jOjM| jUg D ƒd�| _Yd| _Zd | _[t| j'jYdd ƒd u| _\t| j'jYdd ƒd u| _]d | _^| jj_�r| `¡ | jja�rtbjc| jjdd d � te| jGd ƒ�r!| jG f| jg¡ th| j| jDd | j d d �| _it6 j|j=¡ |
|
|||
|
|
k| jG| j | ji¡\| _G| _ | _it6 j| j@¡ th| j |jl| j d d �| _m|
|
|||
|
|
k| jm¡| _m| j\�r‹tn| j|j*|jo|jƒ| _| jd u �r}| j�s{t d!ƒ‚d S tn| j|j*|jo|jƒ| _d S | jd u �rš| j�s™t d!ƒ‚n | j p| j'j9¡| _| j p| j'j9¡| _d S )"Nzœ`model` and `ref_model` cannot be the same object. If you want `ref_model` to be the same as `model`, you must make a copy of it, or `None` if you use peft.z5You cannot set both `stop_token` and `stop_token_id`.ÚeoszUnknown `stop_token` z9. Allowed values are: `'eos'` and `None` (no stop token).> r� Úk3zákl_estimator must be either 'k1' (straightforward, unbiased) or 'k3' (lower variance, unbiased, appears to be a strictly better estimator). See [Approximating KL Divergence](http://joschu.net/blog/kl-approx.html) for details.zvPEFT is not installed and you passed a `peft_config` in the trainer's kwargs, please install it to use the PEFT modelsÚis_loaded_in_4bitF)r® z5`batch_size` must be a multiple of `num_mini_batches`z;`local_batch_size` must be a multiple of `num_mini_batches`é zPer-rank minibatch size z is insufficient for whitening©Údevicer Ú__i£† r] )Únum_training_stepsc S s g | ] }t |tƒr|‘qS rs )Ú
|
|||
|
|
isinstancer )Ú.0Úcbrs rs rt Ú
|
|||
|
|
<listcomp>_ s
|
|||
|
|
|
|||
|
|
ÿÿz/_UnslothPPOTrainer.__init__.<locals>.<listcomp>)Úis_local_process_zeroÚis_world_process_zeroÚstateful_callbacksÚdeepspeed_pluginÚ fsdp_pluginT)Úexist_okÚadd_model_tags)r2 ÚshuffleÚ
|
|||
|
|
collate_fnÚ drop_last)r2 rz r{ z1No reference model and model is not a Peft model.)qÚ
|
|||
|
|
ValueErrorrY rZ Ú policy_modelr r) r* Ú eos_token_idÚgeneration_configr< r6 Ú ImportErrorrn r Úmerge_and_unloadr3 rÓ ÚgetattrrB Ú
|
|||
|
|
is_peft_modelr7 r8 r\ r( r] r^ ÚlenÚtrain_dataset_lenr_ r` ra Ú optimizerÚ lr_schedulerÚoptimizer_cls_and_kwargsr% rS r¸ r r® Ú acceleratorÚ
|
|||
|
|
num_processesr. rª r1 r0 r2 r, r$ r4 r3 r: r< Úceilr/ rH ÚtensorrG rk r& Úitemr5 rÏ rä Ú
|
|||
|
|
process_indexÚ
|
|||
|
|
local_seedr' rF Úsample_generations_freqÚlocal_dataloader_batch_sizer* r r[ ÚconfigÚ create_optimizer_and_schedulerr r4 rø rc r Úcallback_handlerÚ add_callbackrå r r r$ Úcontrolr rr rs ÚstateÚ current_flosÚhp_search_backendÚis_deepspeed_enabledÚis_fsdp_enabledr r Ú init_hf_repoÚ should_saver@ Úmakedirsr£ Úhasattrrx Ú
|
|||
|
|
_tag_namesr Ú
|
|||
|
|
dataloaderÚ manual_seedÚpreparer« Úeval_dataloaderrC rÔ rb )rJ rY rZ r[ r\ r] r^ r_ r` ra rb rc rd r‰ Ú time_tensorÚtime_intÚmoduleÚdefault_callbacksrs rs rt rI Ì s ÿ
|
|||
|
|
|
|||
|
|
ÿ ÿÿ
|
|||
|
|
|
|||
|
|
|
|||
|
|
|
|||
|
|
ÿ
|
|||
|
|
ÿ ÿ
|
|||
|
|
ÿ
|
|||
|
|
€ ÿÿÿý
|
|||
|
|
|
|||
|
|
û üÿ ÿ
|
|||
|
|
ÿ ÿz_UnslothPPOTrainer.__init__c C ó | j S ©N©r¡ ©rJ rs rs rt Úget_train_dataloaderž ó z'_UnslothPPOTrainer.get_train_dataloaderc C r© rª )r¤ r¬ rs rs rt Úget_eval_dataloader¡ r® z&_UnslothPPOTrainer.get_eval_dataloaderc c s” � | j r| js| j | jj¡ ¡ ntƒ �, | jr | jj | j¡ dV | jr8| jj | j p.d¡ W d ƒ dS W d ƒ dS 1 sCw Y dS )zWContext manager for handling null reference model (that is, peft adapter manipulation).Nrx )
|
|||
|
|
rƒ r8 r‰ Ú unwrap_modelr[ ÚpolicyÚdisable_adapterr? Ú set_adapterr7 r¬ rs rs rt Únull_ref_context¤ s €ÿÿý÷"øz#_UnslothPPOTrainer.null_ref_contextFr£ Ú_internal_callc sL | j }| j j| _ | jr| j}| j | _tƒ ||¡ || _ | jr$|| _d S d S rª )r[ r± rš rñ rH Ú
|
|||
|
|
save_model)rJ r£ rµ Ú backup_modelÚbackup_deepspeedrL rs rt r¶ ² s
|
|||
|
|
|
|||
|
|
ÿz _UnslothPPOTrainer.save_modelc r
|
|||
|
|
s0 | j }| j}| j}| j}| j}| j}| j}| j‰ |j}‡ fdd„} t | ƒ ƒ}
|
|||
|
|
t
|
|||
|
|
|j |j d dddd�} |
|
|||
|
|
d¡ t ¡ } |j|j|jf}
|
|||
|
|
tj|
|
|||
|
|
|d �}tj|
|
|||
|
|
|d �}tj|
|
|||
|
|
|d �}tj|
|
|||
|
|
|d �}tj|
|
|||
|
|
|d �}tj|
|
|||
|
|
|d �}tj|
|
|||
|
|
|d �}| ¡ d
|
|||
|
|
| j_d
|
|||
|
|
| j_|j| j_|j| j | j_ |j d urª|j d k r¥t | jj|j ¡| j_ n|j | j_ |j d urÆ|j d k rÁt | jj|j ¡| j_ n|j | j_ |j!d urâ|j!d k rÝt | jj|j! ¡| j_!n|j!| j_!| j" #|| j| j$¡| _$| j%rø| j| _&| j| _'t(d |jd ƒD �]î}| j jd |j) 7 _t*|
|
|||
|
|
ƒ}t +¡ ��ˆ |d ,|¡}|j-d }g }g }g }g } g } g } g } t.| j| j| j j/d
|
|||
|
|
��} t0| j1||j2|j3| ƒ\}!}"W d ƒ n 1 �sVw Y t(d
|
|||
|
|
|j-d
|
|||
|
|
|j2ƒD ]è}#||#|#|j2 … }$|!|#|#|j2 … }%|%d d …|d …f }&|"|#|#|j2 … }'t4|'|&ƒ}(~'t5ƒ |d u �r¸| 6¡ � t7|j1|%|j3ƒ})W d ƒ n 1 �s²w Y nt7||%|j3ƒ})|)j8d d …|d d…f }*|*|j d }*t4|*|&ƒ}+~)~*t5ƒ |&},| j9d u�rít:| j9|j3|&ƒ},t ;|$|,fd ¡}-t<|,|j3kƒd }.| =|¡j>}/t?|/|%|j3|ƒ\}0}1}1|0d d …|d d…f @d¡}2t?||-|j3|ƒ\}1}3}1| A|&¡ | A|,¡ | A|(¡ | A|+¡ | A|.¡ | A|3¡ | A|2¡ �qet ;|d
|
|||
|
|
¡}t ;|d
|
|||
|
|
¡}t ;|d
|
|||
|
|
¡}t ;| d
|
|||
|
|
¡} t ;| d
|
|||
|
|
¡} t ;| d
|
|||
|
|
¡} t ;| d
|
|||
|
|
¡} ~(~+~0~2~3~ t5ƒ tB C¡ tjD|| jjEkdd�}4| j jFd u�r¢| |4 | j jF8 < tjG|j-d |jd � H|j-d
|
|||
|
|
d ¡}5|5| Id ¡k}6t J||6tK¡}t J| |6tK¡} | d }7|5|7 Id ¡k}8t J| |8d
|
|||
|
|
¡} | | }9|jLdk�rè|9 n|9 M¡ d |9 }:|jN |: };|; O¡ }<tjG|< Pd
|
|||
|
|
¡|<jd �}=t Q|7|< Pd ¡k |7| ¡}>|<|=|>g | 7 < |jR�r.tS|<|8 dd�}<t J|<|8d
|
|||
|
|
¡}<d
|
|||
|
|
}?g }@|j-d }AtTt(|AƒƒD ]:}B|B|Ad k �rP| d d …|Bd f nd}C|<d d …|Bf |jU|C | d d …|Bf }D|D|jU|jV |? }?|@ A|?¡ �q=tjW|@d d d… d d�}E|E| }FtS|E|6 ƒ}Et J|E|6d
|
|||
|
|
¡}Et5ƒ W d ƒ n 1 �s£w Y t(|jƒD �]Ê}GtXjY Z|j[¡}Hd
|
|||
|
|
}It(d
|
|||
|
|
|j[|j\ƒD �]´}J|J|j\ }K|H|J|K… }Ld
|
|||
|
|
}Mt(d
|
|||
|
|
|j\|j]ƒD �]x}N| ^|¡��b |N|j] }O|L|N|O… }P|E|P }Q||P }R|!|P }S||P }T|F|P }U| |P }Vt7||S|j3ƒ\}W}X|Wj8d d …|d d…f }'|'|j d }'t4|'|Rƒ}Yt J|Y|6|P tK¡}Y|Xd d …|d d…f @d¡}Zt J|Z|8|P d
|
|||
|
|
¡}Zt _|Z|V|j` |V|j` ¡}[t a|Z|U ¡}\t a|[|U ¡}]t b|\|]¡}^dtc|^|8|P ƒ }_tc|]|\k d¡ |8|P ƒ}`|Y|T }at M|a¡}b|Q |b }c|Q t _|bd|je d|je ¡ }dt b|c|d¡}etc|e|6|P ƒ}f|f|jf|_ }g| g|g¡ | h¡ | i¡ t +¡ �p tc|d|ck d¡ |6|P ƒ}htjjjkjl|'dtjmd� ,|'jn¡}itjo|'dd�tjp|i|' dd� }jd|ad q¡ }k|k||G|I|Mf< |h||G|I|Mf< |f||G|I|Mf< |_||G|I|Mf< |`||G|I|Mf< |j q¡ ||G|I|Mf< |b q¡ ||G|I|Mf< W d ƒ n 1 �s8w Y W d ƒ n 1 �sHw Y |Md 7 }M�qÙ|Id 7 }I~W~X~'~Y~Z~[~\~]~_~`~a~b~c~d~e~f~g~h~i~j~k~U~Q~V~R~S~Tt5ƒ �qÁ�qt +¡ �� |: pd ¡ q¡ }l| pd ¡ q¡ }m|; pd ¡ q¡ }n|n| q¡ }otr| jjt ¡ | ƒ}pi }q|p|qd< | j s|l¡ q¡ t¡ |qd< | j s|m¡ q¡ t¡ |qd< | j s|n¡ q¡ t¡ |qd< | j s|o¡ q¡ t¡ |qd< | j s| q¡ ¡ q¡ t¡ |qd < | j s|¡ q¡ t¡ |qd < | j s|¡ q¡ t¡ |qd < | j s|¡ q¡ t¡ |qd < | j s|¡ q¡ t¡ |qd < | j s|¡ q¡ t¡ |qd!< | j s|¡ q¡ t¡ |qd"< | j s|¡ q¡ t¡ |qd#< | j s|¡ u¡ t¡ |qd$< ||jEk p¡ t¡ |qd%< | jv w¡ d
|
|||
|
|
|qd&< | jj|qd'< | jj| j | j_x| j jd 7 _| y|q¡ W d ƒ n 1 �s†w Y | jv h¡ | j" z|| j| j$¡| _$| j$j{�r³| j||d d(� | j" }| j | j| j$¡| _$~:~l~m~n~ ~q~;t5ƒ tB C¡ |j~d
|
|||
|
|
k�rÚ|d | j d
|
|||
|
|
k�rÚ| j€dd)� t5ƒ ~!~~~~ ~ ~ ~4~7~5~6~8~<~=~>~E~Ft5ƒ �q | j" �|| j| j$¡| _$| j$j{�r| j||d d d*� | j" }| j | j| j$¡| _$d S d S )+Nc 3 s � ˆ E d H qrª rs rs r« rs rt Úrepeat_generatorÌ s €
|
|||
|
|
ÿz2_UnslothPPOTrainer.train.<locals>.repeat_generatorr r� r„ T©Úmax_new_tokensr+ Útop_kÚtop_pÚ do_samplez===training policy===rj r r] Ú input_ids©Úgather_deepspeed3_paramsrW r\ r� F)ÚmaskÚ
|
|||
|
|
shift_mean)Úaxisg à?)rZ Údtyper} Úepsz objective/klzobjective/entropyzobjective/non_score_rewardzobjective/rlhf_rewardzobjective/scoreszpolicy/approxkl_avgzpolicy/clipfrac_avgzloss/policy_avgzloss/value_avgzval/clipfrac_avgzpolicy/entropy_avgz val/ratioz
|
|||
|
|
val/ratio_varzval/num_eos_tokensÚlrÚepisode)Útrial)Úsampling)rÉ Úmetrics)‚rY r‰ r† r[ r\ r] rZ r¡ rk Úiterr r( r+ ÚprintrG r9 r$ r® rH ÚzerosÚtrainr— Ú global_steprÈ r/ r¹ r% r… r¸ rà r< r‹ rà rÆ r” Úon_train_beginr– rš rñ Ú
|
|||
|
|
model_wrappedÚranger2 ÚnextÚno_gradrb r` rJ rB r% r± r& Ú pad_token_idrE r+ r´ r. rj r* rI Úcatr- r° r_ r5 rf rh r0 ÚcollectÚanyr~ r, ÚarangeÚrepeatre Ú masked_fillr r< Úexpr; ÚcloneÚsizeÚwherer: r; Úreversedr@ rA Ústackr> ÚrandomÚ permutationr1 r3 rª Ú
|
|||
|
|
accumulateÚclampr? ÚsquarerF r: Úfloatr= r> ÚbackwardÚstepÚ zero_gradr= r Úsoftmaxrc rÅ rg ÚsumÚmeanrS Úgather_for_metricsr� Úvarr‡ Ú get_last_lrÚepochÚlogÚ on_step_endr� Ú_save_checkpointÚon_saver' r� Úgenerate_completionsÚ on_train_end)rrJ rY r‰ r† r[ Ú
|
|||
|
|
ref_policyr] rZ rk r¹ Úiter_dataloaderr Ú
|
|||
|
|
start_timeÚ stats_shapeÚapproxkl_statsÚpg_clipfrac_statsÚ
|
|||
|
|
pg_loss_statsÚ
|
|||
|
|
vf_loss_statsÚvf_clipfrac_statsÚ
|
|||
|
|
entropy_statsÚ ratio_statsÚupdateÚdataÚqueriesÚcontext_lengthÚ responsesÚpostprocessed_responsesÚlogprobsÚ ref_logprobsÚscoresÚsequence_lengthsÚvaluesÚunwrapped_modelÚquery_responsesÚlogitssÚiÚqueryÚquery_responseÚresponserj ÚlogprobÚ
|
|||
|
|
ref_outputÚ
|
|||
|
|
ref_logitsÚ ref_logprobÚpostprocessed_responseÚ postprocessed_query_responseÚsequence_lengthÚunwrapped_value_modelÚ
|
|||
|
|
full_valueÚ_ÚvalueÚscoreÚcontain_eos_tokenÚ
|
|||
|
|
response_idxsÚ padding_maskÚsequence_lengths_p1Úpadding_mask_p1ÚlogrÚklÚnon_score_rewardÚrewardsÚ actual_startÚ
|
|||
|
|
actual_endÚ
|
|||
|
|
lastgaelamÚadvantages_reversedÚ
|
|||
|
|
gen_lengthÚtÚ
|
|||
|
|
nextvaluesÚdeltaÚ
|
|||
|
|
advantagesÚreturnsÚ
|
|||
|
|
ppo_epoch_idxÚb_indsÚ
|
|||
|
|
minibatch_idxÚmini_batch_startÚmini_batch_endÚmini_batch_indsÚgradient_accumulation_idxÚmicro_batch_startÚmicro_batch_endÚmicro_batch_indsÚ mb_advantageÚ mb_responsesÚmb_query_responsesÚ mb_logprobsÚ mb_returnÚ mb_valuesÚoutputÚ
|
|||
|
|
vpred_tempÚ new_logprobsÚvpredÚ vpredclippedÚ
|
|||
|
|
vf_losses1Ú
|
|||
|
|
vf_losses2Ú vf_loss_maxÚvf_lossÚ vf_clipfracÚ
|
|||
|
|
logprobs_diffÚratioÚ pg_lossesÚ
|
|||
|
|
pg_losses2Ú pg_loss_maxÚpg_lossÚlossÚ pg_clipfracÚ prob_distÚentropyÚapproxklÚmean_klÚ mean_entropyÚmean_non_score_rewardÚ rlhf_rewardrÆ rË rs r« rt rÏ Á sR
|
|||
|
|
û
|
|||
|
|
|
|||
|
|
|
|||
|
|
|
|||
|
|
|
|||
|
|
|
|||
|
|
|
|||
|
|
|
|||
|
|
|
|||
|
|
|
|||
|
|
|
|||
|
|
|
|||
|
|
ÿ
|
|||
|
|
û ý
|
|||
|
|
|
|||
|
|
|
|||
|
|
ÿ€
|
|||
|
|
|
|||
|
|
ÿ
|
|||
|
|
|
|||
|
|
ÿ
|
|||
|
|
|
|||
|
|
ÿ
|
|||
|
|
|
|||
|
|
|
|||
|
|
|
|||
|
|
|
|||
|
|
|
|||
|
|
$"
|
|||
|
|
&* †}
|
|||
|
|
|
|||
|
|
|
|||
|
|
|
|||
|
|
ÿ ý ÿ
|
|||
|
|
|
|||
|
|
|