361 lines
46 KiB
Plaintext
361 lines
46 KiB
Plaintext
|
§
|
|||
|
|
3$�hXš ã óÔ — d Z ddlmZ ddlZddlmZ ddlmZ ddlmZm Z m
|
|||
|
|
Z
|
|||
|
|
m Z m Z m
|
|||
|
|
Z
|
|||
|
|
mZmZ ddlmZmZmZmZmZmZm
|
|||
|
|
Z
|
|||
|
|
mZmZmZmZm Z mZmZmZm Z m Z m Z m Z m Z m!Z!mZm"Z" ddl Z ddlT ddl#m$Z$m%Z% dd l&m'Z' ddlZddl(Z)dd
|
|||
|
|
l*m+Z+ ddlmZ dd l,m-Z-m.Z/ d d
|
|||
|
|
d d
|
|||
|
|
d
|
|||
|
|
dœZ0 ej1 d d e0¬¦ « d„ ¦ « Z2e$ G d„ de¦ « ¦ « Z3 G d„ de¦ « Z4 G d„ de4¦ « Z5dS )z8
|
|||
|
|
2025.8.4
|
|||
|
|
2025.8.5
|
|||
|
|
4.55.1
|
|||
|
|
0.21.0
|
|||
|
|
__UNSLOTH_VERSIONING__
|
|||
|
|
é )ÚTensorN)Ú
|
|||
|
|
functional)ÚAnyÚListÚOptionalÚTupleÚUnionÚDictÚSetÚCallable)Ú Acceleratorr r Ú
|
|||
|
|
DDPOConfigÚDDPOStableDiffusionPipelineÚ DDPOTrainerr ÚPathÚPerPromptStatTrackerÚProjectConfigurationÚPyTorchModelHubMixinr Ú defaultdictÚfuturesÚgenerate_model_cardÚget_comet_experiment_urlÚis_wandb_availableÚloggerÚosÚset_seedÚtextwrapÚtorchÚwarnings)Ú*)Ú dataclassÚfield)ÚVersion)Ú nullcontext)ÚDataCollatorForSeq2SeqÚ DataCollatorForLanguageModelingTF)Úepilogue_fusionÚ max_autotuneÚ
|
|||
|
|
shape_paddingz
|
|||
|
|
trace.enabledztriton.cudagraphs)ÚdynamicÚ fullgraphÚoptionsc ó’ — t j | d| j d ¦ « dd¬¦ « }t j | d¦ « dd¬¦ « }g }t ||¦ « D ]‘\ }}| t j ¦ « }t j |d| d¦ « ¬¦ « d¦ « }t j
|
|||
|
|
|d¬¦ « }||z
|
|||
|
|
} | | ¦ « Œ’ t j |¦ « }| | j d | j d f¦ « }|S )Néÿÿÿÿé r )ÚchunksÚdim)r1 Úindex©r1 é )
|
|||
|
|
r ÚchunkÚreshapeÚshapeÚzipÚtoÚfloat32ÚgatherÚ unsqueezeÚsqueezeÚ logsumexpÚappendÚconcat)
|
|||
|
|
Úlogitsr2 Úchunked_logitsÚ
|
|||
|
|
chunked_indexÚall_per_token_logpsÚ chunk_logitsÚ chunk_indexÚselected_logitsÚlogsumexp_valuesÚper_token_logpss
|
|||
|
|
ú^/workspace/Fine-tuning/DS-LLM-TEMPLATE-FINETUNING/unsloth_compiled_cache/UnslothDDPOTrainer.pyÚ chunked_selective_log_softmaxrK " s5 € õ ”[ § ¢ °°F´LÀÔ4DÑ!EÔ!EÐPQÐYZÐ[Ñ[Ô[€NÝ”[ § ¢ ¨rÑ!2Ô!2¸QÀaÐHÑHÔH€MØ Ðå%(¨ ¸ Ñ%GÔ%Gð 4ð 4Ñ!ˆ �kØ#—’¥u¤}Ñ5Ô5ˆ Ý œ, |¸2À{×G\ÒG\Ð]_ÑG`ÔG`ÐaÑaÔa×iÒiÐjlÑmÔmˆÝ œ?¨<¸rÐBÑBÔBÐØ)Ð,<Ñ<ˆØ×"Ò" ?Ñ3Ô3Ð3Ð3ØÝ œ,Ð':Ñ;Ô;ÐØ-×5Ò5°v´|ÀA´ÈÌ ÐUVÌÐ6XÑYÔYÐØ Ð ó c óÞ ‡ — e Zd ZU dZ edddi¬¦ « Zee ed< edddi¬¦ « Z ee
|
|||
|
|
ed < d!ˆ fd „ Z ˆ xZ S )"ÚUnslothDDPOConfigaÎ
|
|||
|
|
|
|||
|
|
Configuration class for the [`DDPOTrainer`].
|
|||
|
|
|
|||
|
|
Using [`~transformers.HfArgumentParser`] we can turn this class into
|
|||
|
|
[argparse](https://docs.python.org/3/library/argparse#module-argparse) arguments that can be specified on the
|
|||
|
|
command line.
|
|||
|
|
|
|||
|
|
Parameters:
|
|||
|
|
exp_name (`str`, *optional*, defaults to `os.path.basename(sys.argv[0])[: -len(".py")]`):
|
|||
|
|
Name of this experiment (by default is the file name without the extension name).
|
|||
|
|
run_name (`str`, *optional*, defaults to `""`):
|
|||
|
|
Name of this run.
|
|||
|
|
seed (`int`, *optional*, defaults to `0`):
|
|||
|
|
Random seed.
|
|||
|
|
log_with (`Literal["wandb", "tensorboard"]]` or `None`, *optional*, defaults to `None`):
|
|||
|
|
Log with either 'wandb' or 'tensorboard', check
|
|||
|
|
https://huggingface.co/docs/accelerate/usage_guides/tracking for more details.
|
|||
|
|
tracker_kwargs (`Dict`, *optional*, defaults to `{}`):
|
|||
|
|
Keyword arguments for the tracker (e.g. wandb_project).
|
|||
|
|
accelerator_kwargs (`Dict`, *optional*, defaults to `{}`):
|
|||
|
|
Keyword arguments for the accelerator.
|
|||
|
|
project_kwargs (`Dict`, *optional*, defaults to `{}`):
|
|||
|
|
Keyword arguments for the accelerator project config (e.g. `logging_dir`).
|
|||
|
|
tracker_project_name (`str`, *optional*, defaults to `"trl"`):
|
|||
|
|
Name of project to use for tracking.
|
|||
|
|
logdir (`str`, *optional*, defaults to `"logs"`):
|
|||
|
|
Top-level logging directory for checkpoint saving.
|
|||
|
|
num_epochs (`int`, *optional*, defaults to `100`):
|
|||
|
|
Number of epochs to train.
|
|||
|
|
save_freq (`int`, *optional*, defaults to `1`):
|
|||
|
|
Number of epochs between saving model checkpoints.
|
|||
|
|
num_checkpoint_limit (`int`, *optional*, defaults to `5`):
|
|||
|
|
Number of checkpoints to keep before overwriting old ones.
|
|||
|
|
mixed_precision (`str`, *optional*, defaults to `"fp16"`):
|
|||
|
|
Mixed precision training.
|
|||
|
|
allow_tf32 (`bool`, *optional*, defaults to `True`):
|
|||
|
|
Allow `tf32` on Ampere GPUs.
|
|||
|
|
resume_from (`str`, *optional*, defaults to `""`):
|
|||
|
|
Resume training from a checkpoint.
|
|||
|
|
sample_num_steps (`int`, *optional*, defaults to `50`):
|
|||
|
|
Number of sampler inference steps.
|
|||
|
|
sample_eta (`float`, *optional*, defaults to `1.0`):
|
|||
|
|
Eta parameter for the DDIM sampler.
|
|||
|
|
sample_guidance_scale (`float`, *optional*, defaults to `5.0`):
|
|||
|
|
Classifier-free guidance weight.
|
|||
|
|
sample_batch_size (`int`, *optional*, defaults to `1`):
|
|||
|
|
Batch size (per GPU) to use for sampling.
|
|||
|
|
sample_num_batches_per_epoch (`int`, *optional*, defaults to `2`):
|
|||
|
|
Number of batches to sample per epoch.
|
|||
|
|
train_batch_size (`int`, *optional*, defaults to `1`):
|
|||
|
|
Batch size (per GPU) to use for training.
|
|||
|
|
train_use_8bit_adam (`bool`, *optional*, defaults to `False`):
|
|||
|
|
Use 8bit Adam optimizer from bitsandbytes.
|
|||
|
|
train_learning_rate (`float`, *optional*, defaults to `3e-4`):
|
|||
|
|
Learning rate.
|
|||
|
|
train_adam_beta1 (`float`, *optional*, defaults to `0.9`):
|
|||
|
|
Adam beta1.
|
|||
|
|
train_adam_beta2 (`float`, *optional*, defaults to `0.999`):
|
|||
|
|
Adam beta2.
|
|||
|
|
train_adam_weight_decay (`float`, *optional*, defaults to `1e-4`):
|
|||
|
|
Adam weight decay.
|
|||
|
|
train_adam_epsilon (`float`, *optional*, defaults to `1e-8`):
|
|||
|
|
Adam epsilon.
|
|||
|
|
train_gradient_accumulation_steps (`int`, *optional*, defaults to `1`):
|
|||
|
|
Number of gradient accumulation steps.
|
|||
|
|
train_max_grad_norm (`float`, *optional*, defaults to `1.0`):
|
|||
|
|
Maximum gradient norm for gradient clipping.
|
|||
|
|
train_num_inner_epochs (`int`, *optional*, defaults to `1`):
|
|||
|
|
Number of inner epochs per outer epoch.
|
|||
|
|
train_cfg (`bool`, *optional*, defaults to `True`):
|
|||
|
|
Whether to use classifier-free guidance during training.
|
|||
|
|
train_adv_clip_max (`float`, *optional*, defaults to `5.0`):
|
|||
|
|
Clip advantages to the range.
|
|||
|
|
train_clip_range (`float`, *optional*, defaults to `1e-4`):
|
|||
|
|
PPO clip range.
|
|||
|
|
train_timestep_fraction (`float`, *optional*, defaults to `1.0`):
|
|||
|
|
Fraction of timesteps to train on.
|
|||
|
|
per_prompt_stat_tracking (`bool`, *optional*, defaults to `False`):
|
|||
|
|
Whether to track statistics for each prompt separately.
|
|||
|
|
per_prompt_stat_tracking_buffer_size (`int`, *optional*, defaults to `16`):
|
|||
|
|
Number of reward values to store in the buffer for each prompt.
|
|||
|
|
per_prompt_stat_tracking_min_count (`int`, *optional*, defaults to `16`):
|
|||
|
|
Minimum number of reward values to store in the buffer.
|
|||
|
|
async_reward_computation (`bool`, *optional*, defaults to `False`):
|
|||
|
|
Whether to compute rewards asynchronously.
|
|||
|
|
max_workers (`int`, *optional*, defaults to `2`):
|
|||
|
|
Maximum number of workers to use for async reward computation.
|
|||
|
|
negative_prompts (`str`, *optional*, defaults to `""`):
|
|||
|
|
Comma-separated list of prompts to use as negative examples.
|
|||
|
|
push_to_hub (`bool`, *optional*, defaults to `False`):
|
|||
|
|
Whether to push the final model checkpoint to the Hub.
|
|||
|
|
|
|||
|
|
NÚhelpzvLLM SamplingParams)ÚdefaultÚmetadataÚvllm_sampling_paramsr. z8Chunk size to reduce memory usage. -1 is most efficient.Úunsloth_num_chunksÚ inferenceÚ éO
|
|||
|
|
ÚtrlÚlogséd r4 é Úfp16Té2 ç ð?ç @é Fç-C ëâ6
|
|||
|
|
?çÍÌÌÌÌÌì?ç+‡ÙÎ÷ï?ç{®Gáz„?ç:Œ0âŽyE>ç-C ëâ6?é c) ó: •— t ¦ « j d'i d|“d|“d|“d|“d|“d|“d|“d|“d | “d
|
|||
|
|
|
|
|||
|
|
“d | “d | “d
|
|||
|
|
|
|
|||
|
|
“d|“d|“d|“d|“d|“d|“d|“d|“d|“d|“d|“d|“d|“d|“d | “d | “d | “d | “d | “d!|!“d"|"“d#|#“d$|$“d%|%“d&|&“|)¤Ž |'| _ |(| _ d S )(NÚexp_nameÚrun_nameÚseedÚlog_withÚtracker_project_nameÚlogdirÚ
|
|||
|
|
num_epochsÚ save_freqÚnum_checkpoint_limitÚmixed_precisionÚ
|
|||
|
|
allow_tf32Ú resume_fromÚsample_num_stepsÚ
|
|||
|
|
sample_etaÚsample_guidance_scaleÚsample_batch_sizeÚ sample_num_batches_per_epochÚtrain_batch_sizeÚtrain_use_8bit_adamÚtrain_learning_rateÚtrain_adam_beta1Útrain_adam_beta2Útrain_adam_weight_decayÚtrain_adam_epsilonÚ!train_gradient_accumulation_stepsÚtrain_max_grad_normÚtrain_num_inner_epochsÚ train_cfgÚtrain_adv_clip_maxÚtrain_clip_rangeÚtrain_timestep_fractionÚper_prompt_stat_trackingÚ$per_prompt_stat_tracking_buffer_sizeÚ"per_prompt_stat_tracking_min_countÚasync_reward_computationÚ max_workersÚnegative_promptsÚ push_to_hub© )ÚsuperÚ__init__rR rS )+Úselfrh ri rj rk rl rm rn ro rp rq rr rs rt ru rv rw rx ry rz r{ r| r} r~ r r€ r� r‚ rƒ r„ r… r† r‡ rˆ r‰ rŠ r‹ rŒ r� rR rS ÚkwargsÚ __class__s+ €rJ r� zUnslothDDPOConfig.__init__› s6 ø€ ðZ �‰ŒÔð & 0ð & 0ð & 0Ø �xð& 0à �xð& 0ð �4ð& 0ð �xð & 0ð
|
|||
|
|
$8Ð#7ð & 0ð �Vð
|
|||
|
|
& 0ð $˜ð& 0ð "˜ ð& 0ð $8Ð#7ð& 0ð .˜oð& 0ð $˜ð& 0ð &˜+ð& 0ð 0Ð /ð& 0ð $˜ð & 0ð %:Ð$9ð & 0ð !2Ð 1ð!& 0ð" ,HÐ+Gð#& 0ð$ 0Ð /ð%& 0ð& #6Ð"5ð'& 0ð( #6Ð"5ð)& 0ð* 0Ð /ð+& 0ð, 0Ð /ð-& 0ð. '>Ð&=ð/& 0ð0 "4Ð!3ð1& 0ð2 1RÐ0Qð3& 0ð4 #6Ð"5ð5& 0ð6 &<Ð%;ð7& 0ð8 "˜ ð9& 0ð: "4Ð!3ð;& 0ð< 0Ð /ð=& 0ð> '>Ð&=ð?& 0ð@ (@Ð'?ðA& 0ðB 4XÐ3WðC& 0ðD 2TÐ1SðE& 0ðF (@Ð'?ðG& 0ðH &˜+ðI& 0ðJ 0Ð /ðK& 0ðL &˜+¨ðM& 0ð & 0ð & 0ðN %9ˆÔ!Ø"4ˆÔ Ð Ð rL )(rT rU rV NrW rX rY r4 rZ r[ TrU r\ r] r^ r4 r_ r4 Fr` ra rb rc rd r_ r] r4 Tr^ re r] Frf rf Fr_ rU FNr. )
|
|||
|
|
Ú__name__Ú
|
|||
|
|
__module__Ú __qualname__Ú__doc__r" rR r r Ú__annotations__rS Úintr� Ú
|
|||
|
|
__classcell__©r“ s @rJ rN rN 3 sE ø€ € € € € € ð]ð ]ð| +0¨%ØØÐ 1Ð2ð+ñ +ô +И( 3œ-ð ð ñ ð */¨ØØÐ VÐWð*ñ *ô *И #œ ð ð ñ ð ØØØØ $ØØØØ Ø ØØØ ØØ #Ø Ø'(Ø Ø #Ø #Ø Ø Ø"&Ø "Ø,-Ø !Ø!"ØØ Ø!Ø"%Ø#(Ø/1Ø-/Ø#(ØØ ØØ #Ø ðSU5ð U5ð U5ð U5ð U5ð U5ð U5ð U5ð U5ð U5rL rN c óÀ ‡ — e Zd ZdZddgZ d$dedeej e e
|
|||
|
|
e e gej f deg e e
|
|||
|
|
e f f de d e
|
|||
|
|
ee e e ge f f
|
|||
|
|
d
|
|||
|
|
„Zd%d „Zd
|
|||
|
|
edefd„Zd„ Zdej dedej fd„Zd„ Zd„ Zd„ Zd„ Zd„ Zde ee
|
|||
|
|
f fd„Zd$d e
|
|||
|
|
e fd „Z d „ Z ˆ fd „Z d&d e
|
|||
|
|
e
|
|||
|
|
d!e
|
|||
|
|
e
|
|||
|
|
d"e e
|
|||
|
|
e e
|
|||
|
|
df fd#„Z!ˆ xZ"S )'Ú_UnslothDDPOTrainerrU rW ÚddpoNÚconfigÚreward_functionÚprompt_functionÚ sd_pipelineÚimage_samples_hookc óÚ
|
|||
|
|
— t j dt ¦ « |€t j d¦ « || _ || _ || _ || _ t d i | j j ¤Ž}| j j �rJt j t j
|
|||
|
|
| j j ¦ « ¦ « | j _ dt j | j j ¦ « vrÏt t! d„ t j | j j ¦ « ¦ « ¦ « }t% |¦ « dk r t' d| j j › �¦ « ‚t) d„ |D ¦ « ¦ « }t j | j j d|d › �¦ « | j _ |d d z |_ t/ | j j | j j z ¦ « | _ t7 d | j j | j j || j j | j z d
|
|||
|
|
œ| j j ¤Ž| _ | ! ¦ « \ } }
|
|||
|
|
| st' |
|
|||
|
|
¦ « ‚|j d uo
|
|||
|
|
|j d k } | j j" rg| j # | j j$ | s"tK | & ¦ « ¬ ¦ « n| & ¦ « | j j' ¬
|
|||
|
|
¦ « tQ j) d|› �¦ « tU | j j+ d¬¦ « || _, | j, - d | j j. ddd¬¦ « | j j dk r
|
|||
|
|
t^ j0 } n)| j j dk r
|
|||
|
|
t^ j1 } n t^ j2 } | j, j3 4 | j j5 | ¬¦ « | j, j6 4 | j j5 | ¬¦ « | j, j7 4 | j j5 | ¬¦ « | j, 8 ¦ « }
|
|||
|
|
| j 9 | j: ¦ « | j ; | j< ¦ « | j j= rdt^ j> j? j@ _= | A t… |
|
|||
|
|
t ¦ « s|
|
|||
|
|
C ¦ « n|
|
|||
|
|
¦ « | _D | j, 6 | j, E | j jF €dgn | j jF ddd| j, jE jG ¬¦ « jH 4 | j j5 ¦ « ¦ « d | _I |jJ r t— |jL |jM ¦ « | _N | j, jO p | j jO | _O t¡ | j, d¦ « rj| j, jQ r^| j R |
|
|||
|
|
| jD ¦ « \ }| _D t t! d „ | C ¦ « ¦ « ¦ « | _S n-| j R |
|
|||
|
|
| jD ¦ « \ | _S | _D | j jT r t« jV |jW ¬ ¦ « | _X |j rrtQ j) d |j › �¦ « | j Y |j ¦ « t/ |j Z d ¦ « d ¦ « d z | _[ d S d| _[ d S )!Nz@DDPOTrainer is deprecated and will be removed in version 0.23.0.z8No image_samples_hook provided; no images will be loggedÚ checkpoint_c ó
|
|||
|
|
— d| v S )Nr¥ rŽ )Úxs rJ ú<lambda>z._UnslothDDPOTrainer.__init__.<locals>.<lambda> s € -°1Ð"4€ rL r zNo checkpoints found in c ó^ — g | ]*}t | d ¦ « d ¦ « ‘Œ+S )Ú_r. )r™ Úsplit)Ú.0r§ s rJ ú
|
|||
|
|
<listcomp>z0_UnslothDDPOTrainer.__init__.<locals>.<listcomp> s/ € Ð,XÐ,XÐ,XÀqS°·²¸± ´ ¸bÔ1AÑ-BÔ-BÐ,XÐ,XÐ,XrL r. r4 )rk rq Úproject_configÚgradient_accumulation_stepsÚ tensorboard)Úddpo_trainer_config)rŸ Ú init_kwargsÚ
|
|||
|
|
T)Údevice_specificFÚTimestep)ÚpositionÚdisableÚleaveÚdescÚ
|
|||
|
|
dynamic_ncolsr[ Úbf16)ÚdtyperU ÚptÚ
|
|||
|
|
max_length©Úreturn_tensorsÚpaddingÚ
|
|||
|
|
truncationr¾ Úuse_lorac ó — | j S ©N)Ú
|
|||
|
|
requires_grad)Úps rJ r¨ z._UnslothDDPOTrainer.__init__.<locals>.<lambda>| s € ¸!¼/€ rL )r‹ zResuming from rª rŽ )\r ÚwarnÚDeprecationWarningÚ prompt_fnÚ reward_fnrŸ Úimage_samples_callbackr Úproject_kwargsrs r ÚpathÚnormpathÚ
|
|||
|
|
expanduserÚbasenameÚlistÚfilterÚlistdirÚlenÚ
|
|||
|
|
ValueErrorÚsortedÚjoinÚ iterationr™ rt r† Únum_train_timestepsr
|
|||
|
|
rk rq r€ Úaccelerator_kwargsÚ acceleratorÚ
|
|||
|
|
_config_checkÚis_main_processÚ
|
|||
|
|
init_trackersrl ÚdictÚto_dictÚtracker_kwargsr Úinfor rj r¢ Úset_progress_bar_configÚis_local_main_processr Úfloat16Úbfloat16r: Úvaer9 ÚdeviceÚ text_encoderÚunetÚget_trainable_layersÚ register_save_state_pre_hookÚ_save_model_hookÚ register_load_state_pre_hookÚ_load_model_hookrr ÚbackendsÚcudaÚmatmulÚ_setup_optimizerÚ
|
|||
|
|
isinstanceÚ
|
|||
|
|
parametersÚ optimizerÚ tokenizerrŒ Úmodel_max_lengthÚ input_idsÚneg_prompt_embedr‡ r rˆ r‰ Ú stat_trackerÚautocastÚhasattrrà ÚprepareÚtrainable_layersrŠ r ÚThreadPoolExecutorr‹ ÚexecutorÚ
|
|||
|
|
load_stater« Ú first_epoch)r‘ rŸ r r¡ r¢ r£ Úaccelerator_project_configÚ checkpointsÚcheckpoint_numbersÚis_okayÚmessageÚis_using_tensorboardÚinference_dtyper rë s rJ r� z _UnslothDDPOTrainer.__init__ø s‡ € õ Œ
|
|||
|
|
Ø NÝ ñ
|
|||
|
|
ô
|
|||
|
|
ð
|
|||
|
|
ð Ð %Ý ŒMÐTÑ UÔ UÐ Uà(ˆŒØ(ˆŒØ ˆŒ Ø&8ˆÔ#å%9Ð%WÐ%W¸D¼KÔ<VÐ%WÐ%WÐ"à Œ;Ô "ñ RÝ&(¤g×&6Ò&6µr´w×7IÒ7IÈ$Ì+ÔJaÑ7bÔ7bÑ&cÔ&cˆDŒKÔ #Ø ¥B¤G×$4Ò$4°T´[Ô5LÑ$MÔ$MÐMÐMå "ÝØ4Ð4Ýœ
|
|||
|
|
4¤;Ô#:Ñ;Ô;ñô ñ ô � õ �{Ñ#Ô# qÒ(Ð(Ý$Ð%YÀÄ Ô@WÐ%YÐ%YÑZÔZÐZÝ%+Ð,XÐ,XÈKÐ,XÑ,XÔ,XÑ%YÔ%YÐ"Ý*,¬'¯,ª,Ø”KÔ+Ø:Ð"4°RÔ"8Ð:Ð:ñ+ô +�” Ô'ð
|
|||
|
|
8JÈ"Ô7MÐPQÑ7QÐ*Ô4õ $' t¤{Ô'CÀdÄkÔFiÑ'iÑ#jÔ#jˆÔ å&ð
|
|||
|
|
Ø”[Ô)Ø œKÔ 7Ø5ð )-¬ Ô(UÐX\ÔXpÑ(pð
|
|||
|
|
ð
|
|||
|
|
ð ŒkÔ,ð
|
|||
|
|
ð
|
|||
|
|
ˆÔð ×-Ò-Ñ/Ô/ш�Øð &Ý ˜WÑ%Ô%Ð %à %œ°dÐ :Ð _¸v¼ÐR_Ò?_Ð à Ô Ô +ð Ø Ô × *Ò *Ø” Ô0ØI]Ðs•t°·²Ñ0@Ô0@ÐAÑAÔAÐAÐci×cqÒcqÑcsÔcsØ œKÔ 6ð
|
|||
|
|
+ñ
|
|||
|
|
ô
|
|||
|
|
ð
|
|||
|
|
õ Œ �M˜�M�MÑ"Ô"Ð"å�”Ô!°4Ð8Ñ8Ô8Ð8à&ˆÔà Ô×0Ò0ØØ Ô(Ô>Ð>ØØØ ð 1ñ
|
|||
|
|
ô
|
|||
|
|
ð
|
|||
|
|
ð Ô Ô +¨vÒ 5Ð 5Ý #œmˆOˆOØ
|
|||
|
|
Ô
|
|||
|
|
Ô
|
|||
|
|
-°Ò
|
|||
|
|
7Ð
|
|||
|
|
7Ý #œnˆOˆOå #œmˆOà ÔÔ × Ò Ô 0Ô 7¸Ð ÑOÔOÐOØ ÔÔ%×(Ò(¨Ô)9Ô)@ÈÐ(ÑXÔXÐXØ ÔÔ × Ò Ô!1Ô!8ÀÐ ÑPÔPÐPà Ô+×@Ò@ÑBÔBÐà Ô×5Ò5°dÔ6KÑLÔLÐLØ Ô×5Ò5°dÔ6KÑLÔLÐLð Œ;Ô !ð 9Ø48�EŒNÔ Ô &Ô 1à ×.Ò.Ý1;Ð<LÍdÑ1SÔ1SÐ iÐ × 'Ò 'Ñ )Ô )Ð )ÐYiñ
|
|||
|
|
ô
|
|||
|
|
ˆŒð !%Ô 0× =Ò =Ø Ô × &Ò &Ø œ Ô4Ð<���À$Ä+ÔB^Ø #Ø$Ø Ø Ô+Ô5ÔFð
|
|||
|
|
'ñ
|
|||
|
|
ô
|
|||
|
|
ô Ÿš˜4Ô+Ô2Ñ3Ô3ñ!
|
|||
|
|
ô !
|
|||
|
|
ð ô!
|
|||
|
|
ˆÔ ð Ô *ð Ý 4ØÔ;ØÔ9ñ!ô !ˆDÔ ð Ô(Ô1ÐN°TÔ5EÔ5NˆŒ
|
|||
|
|
å �4Ô# ZÑ 0Ô 0ð o°TÔ5EÔ5Nð oØ#'Ô#3×#;Ò#;Ð<LÈdÌnÑ#]Ô#]Ñ ˆD�$”.Ý$(Ð0IÐ0IÈ4Ï?Ê?ÑK\ÔK\Ñ)]Ô)]Ñ$^Ô$^ˆDÔ !Ð !à48Ô4D×4LÒ4LÐM]Ð_cÔ_mÑ4nÔ4nÑ 1ˆDÔ ! 4¤>à Œ;Ô /ð WÝ #Ô 6À6ÔCUÐ VÑ VÔ VˆDŒMà Ô ð !Ý ŒKÐ=¨Ô);Ð=Ð=Ñ >Ô >Ð >Ø Ô × 'Ò '¨Ô(:Ñ ;Ô ;Ð ;Ý " 6Ô#5×#;Ò#;¸CÑ#@Ô#@ÀÔ#DÑ EÔ EÈÑ IˆDÔ Ð Ð à ˆDÔ Ð Ð rL Fc ó( ‡ — |s[g }|D ]U\ }}}‰ |||¦ « \ }}| t j |‰ j j ¬¦ « |f¦ « ŒVn,‰ j ˆ fd„|¦ « }ˆ fd„|D ¦ « }t |Ž S )N©ré c ó •— ‰j | Ž S rÅ )rË )r§ r‘ s €rJ r¨ z5_UnslothDDPOTrainer.compute_rewards.<locals>.<lambda>– s ø€ °.°$´.À!Ð2D€ rL c ó¢ •— g | ]K\ }}t j | ¦ « ‰j j ¬ ¦ « | ¦ « f‘ŒLS ©r
|
|||
|
|
)r Ú as_tensorÚresultrÜ ré )r¬ ÚrewardÚreward_metadatar‘ s €rJ r z7_UnslothDDPOTrainer.compute_rewards.<locals>.<listcomp>— s\ ø€ ð ð ð á+�F˜Oõ ” § ¢ ¡ ¤ ¸Ô9IÔ9PÐQÑQÔQÐSb×SiÒSiÑSkÔSkÐlðð ð rL ) rË r? r r rÜ ré r Úmapr8 ) r‘ Úprompt_image_pairsÚis_asyncÚrewardsÚimagesÚpromptsÚprompt_metadatar r s ` rJ Úcompute_rewardsz#_UnslothDDPOTrainer.compute_rewardsŠ sØ ø€ Øð ؈GØ4Fð
|
|||
|
|
ð
|
|||
|
|
Ñ0�˜ Ø*.¯.ª.¸ÀÈ/Ñ*ZÔ*ZÑ'�˜Ø—’å œ¨°tÔ7GÔ7NÐOÑOÔOØ'ðñô ð ð ð
|
|||
|
|
ð ”m×'Ò'Ð(DÐ(DÐ(DÐ(DÐFXÑYÔYˆGðð ð ð à/6ðñ ô ˆGõ
|
|||
|
|
�Gˆ}Ð rL ÚepochÚ global_stepc ó¬ ‡ ‡‡‡‡— ‰ ‰ j j ‰ j j ¬¦ « \ Š}ˆfd„‰d ¦ « D ¦ « Š‰ |‰ j j ¬¦ « \ }}t |¦ « D ](\ }}| || || g¦ « Œ)‰ j �'‰ ||‰ j
|
|||
|
|
j d ¦ « t j
|
|||
|
|
|¦ « }‰ j
|
|||
|
|
|¦ « ¦ « ¦ « }‰ j
|
|||
|
|
||| ¦ « | ¦ « dœ|¬¦ « ‰ j j r�‰ j
|
|||
|
|
‰d ¦ « ¦ « ¦ « }‰ j j |d ¬
|
|||
|
|
¦ « } ‰ j | |¦ « }
|
|||
|
|
n/|| ¦ « z
|
|||
|
|
| ¦ « d z z }
|
|||
|
|
t j |
|
|||
|
|
¦ « ‰ j
|
|||
|
|
j d ¦ « ‰ j
|
|||
|
|
j ‰ j
|
|||
|
|
j ¦ « ‰d
|
|||
|
|
< ‰d= ‰d j \ } ŠtC ‰ j j" ¦ « D �]O} t j# | ‰ j
|
|||
|
|
j ¬¦ « Šˆfd„‰ $ ¦ « D ¦ « Št j% ˆˆ fd„tC | ¦ « D ¦ « ¦ « }
|
|||
|
|
dD ]=}‰| t j& | ‰ j
|
|||
|
|
j ¬¦ « dd…df |
|
|||
|
|
f ‰|< Œ>‰ ¦ « Š‰ ' ¦ « }ˆ fd„|D ¦ « }tQ |Ž }ˆfd„|D ¦ « }‰ j j) * ¦ « ‰ + | |||¦ « }‰ j
|
|||
|
|
j, st[ d¦ « ‚�ŒQ|dk r8|‰ j j. z dk r%‰ j
|
|||
|
|
j/ r‰ j
|
|||
|
|
0 ¦ « |S )a
|
|||
|
|
Perform a single step of training.
|
|||
|
|
|
|||
|
|
Args:
|
|||
|
|
epoch (int): The current epoch.
|
|||
|
|
global_step (int): The current global step.
|
|||
|
|
|
|||
|
|
Side Effects:
|
|||
|
|
- Model weights are updated
|
|||
|
|
- Logs the statistics to the accelerator trackers.
|
|||
|
|
- If `self.image_samples_callback` is not None, it will be called with the prompt_image_pairs, global_step,
|
|||
|
|
and the accelerator tracker.
|
|||
|
|
|
|||
|
|
Returns:
|
|||
|
|
global_step (int): The updated global step.
|
|||
|
|
|
|||
|
|
)Ú
|
|||
|
|
iterationsÚ
|
|||
|
|
batch_sizec óT •‡— i | ]#Š‰t j ˆfd „‰D ¦ « ¦ « “Œ$S )c ó •— g | ]
|
|||
|
|
}|‰ ‘Œ S rŽ rŽ )r¬ ÚsÚks €rJ r z7_UnslothDDPOTrainer.step.<locals>.<dictcomp>.<listcomp>¶ s ø€ Ð 7Ð 7Ð 7¨! 1¤Ð 7Ð 7Ð 7rL )r Úcat)r¬ r% Úsampless @€rJ ú
|
|||
|
|
<dictcomp>z,_UnslothDDPOTrainer.step.<locals>.<dictcomp>¶ s; øø€ ÐTÐTÐT¸Q�1•e”iÐ 7Ð 7Ð 7Ð 7¨wÐ 7Ñ 7Ô 7Ñ8Ô8ÐTÐTÐTrL r )r N)r r Ú reward_meanÚ
|
|||
|
|
reward_std©ÚstepÚ
|
|||
|
|
prompt_idsT)Úskip_special_tokensrd r. Ú
|
|||
|
|
advantagesÚ timestepsr
|
|||
|
|
c ó( •— i | ]\ }}||‰ “ŒS rŽ rŽ )r¬ r% ÚvÚperms €rJ r( z,_UnslothDDPOTrainer.step.<locals>.<dictcomp>ä s# ø€ Ð>Ð>Ð>¡d a¨�q˜!˜Dœ'Ð>Ð>Ð>rL c óP •— g | ]"}t j ‰‰j j ¬ ¦ « ‘Œ#S r )r ÚrandpermrÜ ré )r¬ rª Ú
|
|||
|
|
num_timestepsr‘ s €€rJ r z,_UnslothDDPOTrainer.step.<locals>.<listcomp>é s/ ø€ ÐpÐpÐpÐST•”
|
|||
|
|
°dÔ6FÔ6MÐNÑNÔNÐpÐpÐprL )r0 ÚlatentsÚ next_latentsÚ log_probsc ó\ •— g | ](} |j d ‰j j g|j dd… ¢R Ž ‘Œ)S )r. r4 N)r6 rŸ ry r7 )r¬ r2 r‘ s €rJ r z,_UnslothDDPOTrainer.step.<locals>.<listcomp>õ sB ø€ Ð rÐ rÐ rÐ]^˜y˜qœy¨¨T¬[Ô-IÐ XÈAÌGÐTUÐTVÐTVÌKÐ XÐ XÐ XÐ rÐ rÐ rrL c óJ •— g | ] }t t ‰|¦ « ¦ « ‘Œ S rŽ )rà r8 )r¬ Ú
|
|||
|
|
row_valuesÚ
|
|||
|
|
original_keyss €rJ r z,_UnslothDDPOTrainer.step.<locals>.<listcomp>ú s+ ø€ Ð hÐ hÐ hÈ
|
|||
|
|
�t¥C¨
|
|||
|
|
°zÑ$BÔ$BÑ CÔ CÐ hÐ hÐ hrL zsOptimization step should have been performed by this point. Please check calculated gradient accumulation settings.)1Ú_generate_samplesrŸ rx rw Úkeysr rŠ Ú enumerateÚextendrÌ rÜ Útrackersr r& r; ÚcpuÚnumpyÚlogÚmeanÚstdr‡ r¢ rø Ú batch_decoderü Úupdater r6 Ú
|
|||
|
|
num_processesÚ
|
|||
|
|
process_indexr9 ré r7 Úranger‚ r5 ÚitemsÚstackÚarangeÚvaluesr8 rë ÚtrainÚ_train_batched_samplesÚsync_gradientsrÖ ro rÞ Ú
|
|||
|
|
save_state)r‘ r r Úprompt_image_datar Úrewards_metadataÚiÚ
|
|||
|
|
image_datar- r r/ Útotal_batch_sizeÚ inner_epochÚpermsÚkeyÚoriginal_valuesÚreshaped_valuesÚtransposed_valuesÚsamples_batchedr6 r= r3 r' s ` @@@@rJ r, z_UnslothDDPOTrainer.stepž s~ øøøøø€ ð$ &*×%;Ò%;Ø”{Ô?Ø”{Ô4ð &<ñ &
|
|||
|
|
ô &
|
|||
|
|
Ñ"ˆÐ"ð UÐTÐTÐTÀ'È!Ä*Ç/Â/ÑBSÔBSÐTÑTÔTˆØ$(×$8Ò$8Ø ¨¬ Ô(Lð %9ñ %
|
|||
|
|
ô %
|
|||
|
|
Ñ!ˆÐ!õ 'Ð'8Ñ 9Ô 9ð Að A‰MˆAˆzØ × Ò ˜w qœzÐ+;¸AÔ+>Ð ?Ñ @Ô @Ð @Ð @à Ô &Ð 2Ø × 'Ò 'Ð(9¸;ÈÔHXÔHaÐbcÔHdÑ eÔ eÐ eå”)˜GÑ$Ô$ˆØÔ"×)Ò)¨'Ñ2Ô2×6Ò6Ñ8Ô8×>Ò>Ñ@Ô@ˆà Ô× Ò à!Ø Ø &Ÿ|š|™~œ~Ø %Ÿkšk™mœmð
|
|||
|
|
ð
|
|||
|
|
ð ð ñ
|
|||
|
|
ô
|
|||
|
|
ð
|
|||
|
|
ð Œ;Ô /ð Mà Ô)×0Ò0°¸ Ô1FÑGÔG×KÒKÑMÔM×SÒSÑUÔUˆJØÔ&Ô0×=Ò=¸jÐ^bÐ=ÑcÔcˆGØ Ô*×1Ò1°'¸7ÑCÔCˆJˆJà! G§L¢L¡N¤NÑ2°w·{²{±}´}ÀtÑ7KÑLˆJõ
|
|||
|
|
ŒO˜JÑ 'Ô 'ß
|
|||
|
|
ŠW�TÔ%Ô3°RÑ
|
|||
|
|
8Ô
|
|||
|
|
8¸Ô9IÔ9Wô
|
|||
|
|
Yç
|
|||
|
|
ŠR�Ô Ô'Ñ
|
|||
|
|
(Ô
|
|||
|
|
(ð � Ñ ð
|
|||
|
|
�LÐ !à*1°+Ô*>Ô*DÑ'И-å ¤Ô!CÑDÔDð ! ñ ! ˆKå”>Ð"2¸4Ô;KÔ;RÐSÑSÔSˆDØ>Ð>Ð>Ð>¨g¯mªm©o¬oÐ>Ñ>Ô>ˆGõ ”KØpÐpÐpÐpÐpÕX]Ð^nÑXoÔXoÐpÑpÔpñô ˆEð Mð
|
|||
|
|
ð
|
|||
|
|
�Ø & sœ|Ý”LÐ!1¸$Ô:JÔ:QÐRÑRÔRÐSTÐSTÐSTÐVZÐSZÔ[Øðô �˜‘ � ð
|
|||
|
|
$ŸLšL™NœNˆMØ %ŸnšnÑ .Ô .ˆOà rÐ rÐ rÐ rÐbqÐ rÑ rÔ rˆOõ !$ _Ð 5Ð à hÐ hÐ hÐ hÐVgÐ hÑ hÔ hˆOà Ô Ô !× 'Ò 'Ñ )Ô )Ð )Ø ×5Ò5°kÀ5È+ÐWfÑgÔgˆKàÔ#Ô2ð
|
|||
|
|
Ý ð Jñô ð ñ
|
|||
|
|
ð
|
|||
|
|
�AŠ:ˆ:˜% $¤+Ô"7Ñ7¸1Ò<Ð<ÀÔAQÔAaÐ<Ø Ô × 'Ò 'Ñ )Ô )Ð )àÐrL c ó¦ — | ¦ « 5 | j j r{| j t j |gdz ¦ « t j |gdz ¦ « |¦ « j }| d¦ « \ }} || j j | |z
|
|||
|
|
z z }n!| j |||¦ « j }| j
|
|||
|
|
|||| j j |¬¦ « }
|
|||
|
|
|
|
|||
|
|
j } ddd¦ « n # 1 swxY w Y t j
|
|||
|
|
|| j j | j j ¦ « }t j | |z
|
|||
|
|
¦ « } | || j j | ¦ « }
|
|||
|
|
dt j | |z
|
|||
|
|
dz ¦ « z }t j t j | dz
|
|||
|
|
¦ « | j j k ¦ « ¦ « }|
|
|||
|
|
||fS )a‚
|
|||
|
|
Calculate the loss for a batch of an unpacked sample
|
|||
|
|
|
|||
|
|
Args:
|
|||
|
|
latents (torch.Tensor):
|
|||
|
|
The latents sampled from the diffusion model, shape: [batch_size, num_channels_latents, height, width]
|
|||
|
|
timesteps (torch.Tensor):
|
|||
|
|
The timesteps sampled from the diffusion model, shape: [batch_size]
|
|||
|
|
next_latents (torch.Tensor):
|
|||
|
|
The next latents sampled from the diffusion model, shape: [batch_size, num_channels_latents, height,
|
|||
|
|
width]
|
|||
|
|
log_probs (torch.Tensor):
|
|||
|
|
The log probabilities of the latents, shape: [batch_size]
|
|||
|
|
advantages (torch.Tensor):
|
|||
|
|
The advantages of the latents, shape: [batch_size]
|
|||
|
|
embeds (torch.Tensor):
|
|||
|
|
The embeddings of the prompts, shape: [2*batch_size or batch_size, ...] Note: the "or" is because if
|
|||
|
|
train_cfg is True, the expectation is that negative prompts are concatenated to the embeds
|
|||
|
|
|
|||
|
|
Returns:
|
|||
|
|
loss (torch.Tensor), approx_kl (torch.Tensor), clipfrac (torch.Tensor) (all of these are of shape (1,))
|
|||
|
|
r_ )ÚetaÚ prev_sampleNg à?r] )rý rŸ rƒ r¢ rë r r& Úsampler5 rv Úscheduler_stepru r9 Úclampr„ ÚexpÚlossr… rF ÚabsÚfloat)r‘ r7 r0 r8 r9 r/ ÚembedsÚ
|
|||
|
|
noise_predÚnoise_pred_uncondÚnoise_pred_textÚscheduler_step_outputÚlog_probÚratiorh Ú approx_klÚclipfracs rJ Úcalculate_lossz"_UnslothDDPOTrainer.calculate_loss s € ð. �]Š]‰_Œ_ð 7ð 7ØŒ{Ô$ð
|
|||
|
|
Ø !Ô -× 2Ò 2Ý”I˜w˜i¨!™mÑ,Ô,Ý”I˜y˜k¨A™oÑ.Ô.Øñ ô ô ð ð
|
|||
|
|
6@×5EÒ5EÀaÑ5HÔ5HÑ2Ð! ?Ø .°´Ô1RØ#Ð&7Ñ7ñ2ñ �
|
|||
|
|
�
|
|||
|
|
ð "Ô -× 2Ò 2ØØ Øñ ô ô ð ð %)Ô$4×$CÒ$CØØØØ”KÔ*Ø (ð %Dñ %ô %Ð !ð -Ô6ˆHð7 7ð 7ð 7ñ 7ô 7ð 7ð 7ð 7ð 7ð 7ð 7øøøð 7ð 7ð 7ð 7õ: ”[Ø Ø
|
|||
|
|
Œ[Ô
|
|||
|
|
+Ð +Ø ŒKÔ *ñ
|
|||
|
|
ô
|
|||
|
|
ˆ
|
|||
|
|
õ ” ˜( YÑ.Ñ/Ô/ˆà�yŠy˜ T¤[Ô%AÀ5ÑIÔIˆà�%œ* h°Ñ&:¸qÑ%@ÑAÔAÑAˆ å”:�uœy¨°©Ñ 5Ô 5¸¼ Ô8TÒ T× [Ò [Ñ ]Ô ]Ñ^Ô^ˆà�Y Ð(Ð(s •CC:Ã:C>ÄC>r/ Ú
|
|||
|
|
clip_rangerq c óœ — | |z }| t j |d|z
|
|||
|
|
d|z ¦ « z }t j t j ||¦ « ¦ « S )Nr] )r rf rF Úmaximum)r‘ r/ ru rq Úunclipped_lossÚ clipped_losss rJ rh z_UnslothDDPOTrainer.lossM s[ € ð %˜ uÑ,ˆØ"�{¥U¤[Ø Ø �*Ñ Ø �*Ñ ñ&
|
|||
|
|
ô &
|
|||
|
|
ñ
|
|||
|
|
ˆ õ
|
|||
|
|
Œz�%œ-¨¸ ÑEÔEÑFÔFÐFrL c óæ — | j j rdd l}|j j }nt
|
|||
|
|
j j } ||| j j | j j | j j f| j j
|
|||
|
|
| j j ¬¦ « S )Nr )ÚlrÚbetasÚ weight_decayÚeps) rŸ rz Ú bitsandbytesÚoptimÚ AdamW8bitr ÚAdamWr{ r| r} r~ r )r‘ Útrainable_layers_parametersr Ú
|
|||
|
|
optimizer_clss rJ rô z$_UnslothDDPOTrainer._setup_optimizer[ sx € Ø Œ;Ô *ð .Ø Ð Ð Ð à (Ô .Ô 8ˆMˆMå !œKÔ -ˆMà ˆ}Ø 'ØŒ{Ô.Ø”;Ô/°´Ô1MÐNØ œÔ<Ø” Ô.ð
|
|||
|
|
ñ
|
|||
|
|
ô
|
|||
|
|
ð
|
|||
|
|
rL c óf — | j |||¦ « | ¦ « d S rÅ )r¢ Úsave_checkpointÚpop)r‘ ÚmodelsÚweightsÚ
|
|||
|
|
output_dirs rJ rî z$_UnslothDDPOTrainer._save_model_hookk s. € Ø Ô×(Ò(¨°¸*ÑEÔEÐEØ� Š ‰
|
|||
|
|
Œ
|
|||
|
|
ˆ
|
|||
|
|
ˆ
|
|||
|
|
ˆ
|
|||
|
|
rL c ód — | j ||¦ « | ¦ « d S rÅ )r¢ Úload_checkpointr‡ )r‘ rˆ Ú input_dirs rJ rð z$_UnslothDDPOTrainer._load_model_hooko s, € Ø Ô×(Ò(¨°Ñ;Ô;Ð;Ø�
|
|||
|
|
Š
|
|||
|
|
‰ Œ ˆ ˆ ˆ rL c ó
|
|||
|
|
‡ — g }g }‰ j j ¦ « ‰ j |dd¦ « }t |¦ « D �]°}t
|
|||
|
|
ˆ fd„t |¦ « D ¦ « Ž \ }}‰ j |ddd‰ j j j ¬¦ « j
|
|||
|
|
‰ j j ¦ « } ‰ j
|