273 lines
35 KiB
Plaintext
273 lines
35 KiB
Plaintext
|
§
|
|||
|
|
Y¢°hXo ã óL — d Z ddlmZ ddlZddlmZ ddlmZ ddlmZm Z m
|
|||
|
|
Z
|
|||
|
|
m Z m Z m
|
|||
|
|
Z
|
|||
|
|
mZmZ ddlmZmZmZmZmZmZm
|
|||
|
|
Z
|
|||
|
|
mZmZmZm Z mZmZmZmZm Z m Z m Z m Z mZm Z ddl Z ddlT ddl!m"Z"m#Z# dd l$m%Z% ddlZddl&Z'dd
|
|||
|
|
l(m)Z) ddlmZ dd l*m+Z+m,Z- d d
|
|||
|
|
d d
|
|||
|
|
d
|
|||
|
|
dœZ. ej/ d d e.¬¦ « d„ ¦ « Z0e" G d„ de¦ « ¦ « Z1 G d„ de¦ « Z2 G d„ de2¦ « Z3 e4e d¦ « r3ddl5Z5 G d„ de5j6 ¦ « Z7 e j8 e7d¦ « ¦ « dS dS )z9
|
|||
|
|
2025.8.9
|
|||
|
|
2025.8.10
|
|||
|
|
4.55.4
|
|||
|
|
0.21.0
|
|||
|
|
__UNSLOTH_VERSIONING__
|
|||
|
|
é )ÚTensorN)Ú
|
|||
|
|
functional)ÚAnyÚListÚOptionalÚTupleÚUnionÚDictÚSetÚCallable)Ú AcceleratorÚAlignPropConfigÚAlignPropTrainerr r ÚDDPOStableDiffusionPipeliner ÚPathÚProjectConfigurationÚPyTorchModelHubMixinr Ú defaultdictÚgenerate_model_cardÚget_comet_experiment_urlÚis_wandb_availableÚloggerÚosÚset_seedÚtextwrapÚtorchÚwarnings)Ú*)Ú dataclassÚfield)ÚVersion)Ú nullcontext)ÚDataCollatorForSeq2SeqÚ DataCollatorForLanguageModelingTF)Úepilogue_fusionÚ max_autotuneÚ
|
|||
|
|
shape_paddingz
|
|||
|
|
trace.enabledztriton.cudagraphs)ÚdynamicÚ fullgraphÚoptionsc ó’ — t j | d| j d ¦ « dd¬¦ « }t j | d¦ « dd¬¦ « }g }t ||¦ « D ]‘\ }}| t j ¦ « }t j |d| d¦ « ¬¦ « d¦ « }t j
|
|||
|
|
|d¬¦ « }||z
|
|||
|
|
} | | ¦ « Œ’ t j |¦ « }| | j d | j d f¦ « }|S )Néÿÿÿÿé r )ÚchunksÚdim)r/ Úindex)r/ é )
|
|||
|
|
r ÚchunkÚreshapeÚshapeÚzipÚtoÚfloat32ÚgatherÚ unsqueezeÚsqueezeÚ logsumexpÚappendÚconcat)
|
|||
|
|
Úlogitsr0 Úchunked_logitsÚ
|
|||
|
|
chunked_indexÚall_per_token_logpsÚ chunk_logitsÚ chunk_indexÚselected_logitsÚlogsumexp_valuesÚper_token_logpss
|
|||
|
|
úa/workspace/DS-LLM-TEMPLATE-FINETUNING/notebooks/unsloth_compiled_cache/UnslothAlignPropTrainer.pyÚ chunked_selective_log_softmaxrH " s5 € õ ”[ § ¢ °°F´LÀÔ4DÑ!EÔ!EÐPQÐYZÐ[Ñ[Ô[€NÝ”[ § ¢ ¨rÑ!2Ô!2¸QÀaÐHÑHÔH€MØ Ðå%(¨ ¸ Ñ%GÔ%Gð 4ð 4Ñ!ˆ �kØ#—’¥u¤}Ñ5Ô5ˆ Ý œ, |¸2À{×G\ÒG\Ð]_ÑG`ÔG`ÐaÑaÔa×iÒiÐjlÑmÔmˆÝ œ?¨<¸rÐBÑBÔBÐØ)Ð,<Ñ<ˆØ×"Ò" ?Ñ3Ô3Ð3Ð3ØÝ œ,Ð':Ñ;Ô;ÐØ-×5Ò5°v´|ÀA´ÈÌ ÐUVÌÐ6XÑYÔYÐØ Ð ó c óÌ ‡ — e Zd ZU dZ edddi¬¦ « Zee ed< edddi¬¦ « Z ee
|
|||
|
|
ed < d ˆ fd „ Z ˆ xZ S )!ÚUnslothAlignPropConfiga´
|
|||
|
|
|
|||
|
|
Configuration class for the [`AlignPropTrainer`].
|
|||
|
|
|
|||
|
|
Using [`~transformers.HfArgumentParser`] we can turn this class into
|
|||
|
|
[argparse](https://docs.python.org/3/library/argparse#module-argparse) arguments that can be specified on the
|
|||
|
|
command line.
|
|||
|
|
|
|||
|
|
Parameters:
|
|||
|
|
exp_name (`str`, *optional*, defaults to `os.path.basename(sys.argv[0])[: -len(".py")]`):
|
|||
|
|
Name of this experiment (defaults to the file name without the extension).
|
|||
|
|
run_name (`str`, *optional*, defaults to `""`):
|
|||
|
|
Name of this run.
|
|||
|
|
seed (`int`, *optional*, defaults to `0`):
|
|||
|
|
Random seed for reproducibility.
|
|||
|
|
log_with (`str` or `None`, *optional*, defaults to `None`):
|
|||
|
|
Log with either `"wandb"` or `"tensorboard"`. Check
|
|||
|
|
[tracking](https://huggingface.co/docs/accelerate/usage_guides/tracking) for more details.
|
|||
|
|
log_image_freq (`int`, *optional*, defaults to `1`):
|
|||
|
|
Frequency for logging images.
|
|||
|
|
tracker_kwargs (`dict[str, Any]`, *optional*, defaults to `{}`):
|
|||
|
|
Keyword arguments for the tracker (e.g., `wandb_project`).
|
|||
|
|
accelerator_kwargs (`dict[str, Any]`, *optional*, defaults to `{}`):
|
|||
|
|
Keyword arguments for the accelerator.
|
|||
|
|
project_kwargs (`dict[str, Any]`, *optional*, defaults to `{}`):
|
|||
|
|
Keyword arguments for the accelerator project config (e.g., `logging_dir`).
|
|||
|
|
tracker_project_name (`str`, *optional*, defaults to `"trl"`):
|
|||
|
|
Name of project to use for tracking.
|
|||
|
|
logdir (`str`, *optional*, defaults to `"logs"`):
|
|||
|
|
Top-level logging directory for checkpoint saving.
|
|||
|
|
num_epochs (`int`, *optional*, defaults to `100`):
|
|||
|
|
Number of epochs to train.
|
|||
|
|
save_freq (`int`, *optional*, defaults to `1`):
|
|||
|
|
Number of epochs between saving model checkpoints.
|
|||
|
|
num_checkpoint_limit (`int`, *optional*, defaults to `5`):
|
|||
|
|
Number of checkpoints to keep before overwriting old ones.
|
|||
|
|
mixed_precision (`str`, *optional*, defaults to `"fp16"`):
|
|||
|
|
Mixed precision training.
|
|||
|
|
allow_tf32 (`bool`, *optional*, defaults to `True`):
|
|||
|
|
Allow `tf32` on Ampere GPUs.
|
|||
|
|
resume_from (`str`, *optional*, defaults to `""`):
|
|||
|
|
Path to resume training from a checkpoint.
|
|||
|
|
sample_num_steps (`int`, *optional*, defaults to `50`):
|
|||
|
|
Number of sampler inference steps.
|
|||
|
|
sample_eta (`float`, *optional*, defaults to `1.0`):
|
|||
|
|
Eta parameter for the DDIM sampler.
|
|||
|
|
sample_guidance_scale (`float`, *optional*, defaults to `5.0`):
|
|||
|
|
Classifier-free guidance weight.
|
|||
|
|
train_batch_size (`int`, *optional*, defaults to `1`):
|
|||
|
|
Batch size for training.
|
|||
|
|
train_use_8bit_adam (`bool`, *optional*, defaults to `False`):
|
|||
|
|
Whether to use the 8bit Adam optimizer from `bitsandbytes`.
|
|||
|
|
train_learning_rate (`float`, *optional*, defaults to `1e-3`):
|
|||
|
|
Learning rate.
|
|||
|
|
train_adam_beta1 (`float`, *optional*, defaults to `0.9`):
|
|||
|
|
Beta1 for Adam optimizer.
|
|||
|
|
train_adam_beta2 (`float`, *optional*, defaults to `0.999`):
|
|||
|
|
Beta2 for Adam optimizer.
|
|||
|
|
train_adam_weight_decay (`float`, *optional*, defaults to `1e-4`):
|
|||
|
|
Weight decay for Adam optimizer.
|
|||
|
|
train_adam_epsilon (`float`, *optional*, defaults to `1e-8`):
|
|||
|
|
Epsilon value for Adam optimizer.
|
|||
|
|
train_gradient_accumulation_steps (`int`, *optional*, defaults to `1`):
|
|||
|
|
Number of gradient accumulation steps.
|
|||
|
|
train_max_grad_norm (`float`, *optional*, defaults to `1.0`):
|
|||
|
|
Maximum gradient norm for gradient clipping.
|
|||
|
|
negative_prompts (`str` or `None`, *optional*, defaults to `None`):
|
|||
|
|
Comma-separated list of prompts to use as negative examples.
|
|||
|
|
truncated_backprop_rand (`bool`, *optional*, defaults to `True`):
|
|||
|
|
If `True`, randomized truncation to different diffusion timesteps is used.
|
|||
|
|
truncated_backprop_timestep (`int`, *optional*, defaults to `49`):
|
|||
|
|
Absolute timestep to which the gradients are backpropagated. Used only if `truncated_backprop_rand=False`.
|
|||
|
|
truncated_rand_backprop_minmax (`tuple[int, int]`, *optional*, defaults to `(0, 50)`):
|
|||
|
|
Range of diffusion timesteps for randomized truncated backpropagation.
|
|||
|
|
push_to_hub (`bool`, *optional*, defaults to `False`):
|
|||
|
|
Whether to push the final model to the Hub.
|
|||
|
|
|
|||
|
|
NÚhelpzvLLM SamplingParams)ÚdefaultÚmetadataÚvllm_sampling_paramsr, z8Chunk size to reduce memory usage. -1 is most efficient.Úunsloth_num_chunksÚipykernel_launcherÚ éO
|
|||
|
|
r1 ÚtrlÚlogséd é Úfp16Té2 ç ð?ç @Fç-C ëâ6
|
|||
|
|
?çÍÌÌÌÌÌì?ç+‡ÙÎ÷ï?ç{®Gáz„?ç:Œ0âŽyE>é é1 c ó •— t ¦ « j d i d|“d|“d|“d|“d|“d|“d|“d|“d | “d
|
|||
|
|
|
|
|||
|
|
“d | “d | “d
|
|||
|
|
|
|
|||
|
|
“d|“d|“d|“d|“d|“d|“d|“d|“d|“d|“d|“d|“d|“d|“d | “d | “| ¤Ž | | _ | | _ d S ) NÚexp_nameÚrun_nameÚseedÚlog_withÚlog_image_freqÚtracker_project_nameÚlogdirÚ
|
|||
|
|
num_epochsÚ save_freqÚnum_checkpoint_limitÚmixed_precisionÚ
|
|||
|
|
allow_tf32Ú resume_fromÚsample_num_stepsÚ
|
|||
|
|
sample_etaÚsample_guidance_scaleÚtrain_batch_sizeÚtrain_use_8bit_adamÚtrain_learning_rateÚtrain_adam_beta1Útrain_adam_beta2Útrain_adam_weight_decayÚtrain_adam_epsilonÚ!train_gradient_accumulation_stepsÚtrain_max_grad_normÚnegative_promptsÚtruncated_backprop_randÚtruncated_backprop_timestepÚ push_to_hub© )ÚsuperÚ__init__rO rP )"Úselfrd re rf rg rh ri rj rk rl rm rn ro rp rq rr rs rt ru rv rw rx ry rz r{ r| r} r~ r r€ rO rP ÚkwargsÚ __class__s" €rG rƒ z UnslothAlignPropConfig.__init__Œ s¯ ø€ ðJ �‰ŒÔð 0ð 0ð 0Ø �xð 0à �xð 0ð �4ð 0ð �xð 0ð
|
|||
|
|
,˜^ð 0ð $8Ð#7ð
|
|||
|
|
0ð �Vð 0ð $˜ð 0ð "˜ ð 0ð $8Ð#7ð 0ð .˜oð 0ð $˜ð 0ð &˜+ð 0ð 0Ð /ð 0ð $˜ð 0ð %:Ð$9ð! 0ð" 0Ð /ð# 0ð$ #6Ð"5ð% 0ð& #6Ð"5ð' 0ð( 0Ð /ð) 0ð* 0Ð /ð+ 0ð, '>Ð&=ð- 0ð. "4Ð!3ð/ 0ð0 1RÐ0Qð1 0ð2 #6Ð"5ð3 0ð4 0Ð /ð5 0ð6 '>Ð&=ð7 0ð8 +FÐ*Eð9 0ð: &˜+¨ð; 0ð 0ð 0ð< %9ˆÔ!Ø"4ˆÔ Ð Ð rI ) rQ rR rS Nr1 rT rU rV r1 rW rX TrR rY rZ r[ r1 Fr\ r] r^ r_ r` ra rZ NTrb FNr, )
|
|||
|
|
Ú__name__Ú
|
|||
|
|
__module__Ú __qualname__Ú__doc__r rO r r Ú__annotations__rP Úintrƒ Ú
|
|||
|
|
__classcell__©r† s @rG rK rK 3 s* ø€ € € € € € ðMð Mð\ +0¨%ØØÐ 1Ð2ð+ñ +ô +И( 3œ-ð ð ñ ð */¨ØØÐ VÐWð*ñ *ô *И #œ ð ð ñ ð (ØØØØØ $ØØØØ Ø ØØØ ØØ #Ø Ø #Ø #Ø Ø Ø"&Ø "Ø,-Ø !Ø Ø"&Ø&(ØØ #Ø ðAD5ð D5ð D5ð D5ð D5ð D5ð D5ð D5ð D5ð D5rI rK c óž ‡ — e Zd ZdZddgZ d!dedeej e e
|
|||
|
|
e e gej f deg e e
|
|||
|
|
e f f de d e
|
|||
|
|
ee e e ge f f
|
|||
|
|
d
|
|||
|
|
„Zd „ Zd ed
|
|||
|
|
efd„Zd„ Zdej dedej fd„Zd„ Zd„ Zd„ Zd"d„Zd!de
|
|||
|
|
e fd„Zd„ Zˆ fd „Z d#d e
|
|||
|
|
e
|
|||
|
|
d e
|
|||
|
|
e
|
|||
|
|
d e e
|
|||
|
|
e e
|
|||
|
|
df fd „Z ˆ xZ S )$Ú_UnslothAlignPropTrainerrR rT Ú alignpropNÚconfigÚreward_functionÚprompt_functionÚ sd_pipelineÚimage_samples_hookc ó‚ — t j dt ¦ « |€t j d¦ « || _ || _ || _ || _ t d i | j j ¤Ž}| j j �rJt j t j
|
|||
|
|
| j j ¦ « ¦ « | j _ dt j | j j ¦ « vrÏt t! d„ t j | j j ¦ « ¦ « ¦ « }t% |¦ « dk r t' d| j j › �¦ « ‚t) d„ |D ¦ « ¦ « }t j | j j d|d › �¦ « | j _ |d d z |_ t/ d | j j | j j || j j d
|
|||
|
|
œ| j j ¤Ž| _ |j d uo
|
|||
|
|
|j d k } | j j rg| j | j j | s"tA | ! ¦ « ¬ ¦ « n| ! ¦ « | j j" ¬
|
|||
|
|
¦ « tG j$ d|› �¦ « tK | j j& d¬¦ « || _' | j' ( d | j j) ddd¬¦ « | j j dk r
|
|||
|
|
tT j+ }
|
|||
|
|
n)| j j dk r
|
|||
|
|
tT j, }
|
|||
|
|
n tT j- }
|
|||
|
|
| j' j. / | j j0 |
|
|||
|
|
¬¦ « | j' j1 / | j j0 |
|
|||
|
|
¬¦ « | j' j2 / | j j0 |
|
|||
|
|
¬¦ « | j' 3 ¦ « } | j 4 | j5 ¦ « | j 6 | j7 ¦ « | j j8 rdtT j9 j: j; _8 | < t{ | t ¦ « s| > ¦ « n| ¦ « | _? | j' 1 | j' @ | j jA €dgn | j jA ddd| j' j@ jB ¬¦ « jC / | j j0 ¦ « ¦ « d | _D | j' jE p | j jE | _E t� | j' d¦ « rj| j' jG r^| j H | | j? ¦ « \ } | _? t t! d „ | > ¦ « ¦ « ¦ « | _I n-| j H | | j? ¦ « \ | _I | _? |j rrtG j$ d |j › �¦ « | j J |j ¦ « t— |j L d ¦ « d ¦ « d z | _M d S d| _M d S ) NzEAlignPropTrainer is deprecated and will be removed in version 0.23.0.z8No image_samples_hook provided; no images will be loggedÚ checkpoint_c ó
|
|||
|
|
— d| v S )Nr˜ r� )Úxs rG ú<lambda>z3_UnslothAlignPropTrainer.__init__.<locals>.<lambda>õ s € -°1Ð"4€ rI r zNo checkpoints found in c ó^ — g | ]*}t | d ¦ « d ¦ « ‘Œ+S )Ú_r, )rŒ Úsplit)Ú.0rš s rG ú
|
|||
|
|
<listcomp>z5_UnslothAlignPropTrainer.__init__.<locals>.<listcomp>û s/ € Ð,XÐ,XÐ,XÀqS°·²¸± ´ ¸bÔ1AÑ-BÔ-BÐ,XÐ,XÐ,XrI r, r1 )rg rn Úproject_configÚgradient_accumulation_stepsÚ tensorboard)Úalignprop_trainer_config)r’ Ú init_kwargsÚ
|
|||
|
|
T)Údevice_specificFÚTimestep)ÚpositionÚdisableÚleaveÚdescÚ
|
|||
|
|
dynamic_ncolsrX Úbf16)ÚdtyperR ÚptÚ
|
|||
|
|
max_length©Úreturn_tensorsÚpaddingÚ
|
|||
|
|
truncationr± Úuse_lorac ó — | j S ©N)Ú
|
|||
|
|
requires_grad)Úps rG r› z3_UnslothAlignPropTrainer.__init__.<locals>.<lambda>R s € ¸!¼/€ rI zResuming from r� r� )Nr ÚwarnÚDeprecationWarningÚ prompt_fnÚ reward_fnr’ Úimage_samples_callbackr Úproject_kwargsrp r ÚpathÚnormpathÚ
|
|||
|
|
expanduserÚbasenameÚlistÚfilterÚlistdirÚlenÚ
|
|||
|
|
ValueErrorÚsortedÚjoinÚ iterationr
|
|||
|
|
rg rn r{ Úaccelerator_kwargsÚ acceleratorÚis_main_processÚ
|
|||
|
|
init_trackersri ÚdictÚto_dictÚtracker_kwargsr Úinfor rf r• Úset_progress_bar_configÚis_local_main_processr Úfloat16Úbfloat16r7 Úvaer6 ÚdeviceÚ text_encoderÚunetÚget_trainable_layersÚ register_save_state_pre_hookÚ_save_model_hookÚ register_load_state_pre_hookÚ_load_model_hookro ÚbackendsÚcudaÚmatmulÚ_setup_optimizerÚ
|
|||
|
|
isinstanceÚ
|
|||
|
|
parametersÚ optimizerÚ tokenizerr} Úmodel_max_lengthÚ input_idsÚneg_prompt_embedÚautocastÚhasattrr¶ ÚprepareÚtrainable_layersÚ
|
|||
|
|
load_staterŒ rž Ú first_epoch)
|
|||
|
|
r„ r’ r“ r” r• r– Úaccelerator_project_configÚ checkpointsÚcheckpoint_numbersÚis_using_tensorboardÚinference_dtyperð rÜ s
|
|||
|
|
rG rƒ z!_UnslothAlignPropTrainer.__init__Ù sà € õ Œ
|
|||
|
|
Ø SÝ ñ
|
|||
|
|
ô
|
|||
|
|
ð
|
|||
|
|
ð Ð %Ý ŒMÐTÑ UÔ UÐ Uà(ˆŒØ(ˆŒØ ˆŒ Ø&8ˆÔ#å%9Ð%WÐ%W¸D¼KÔ<VÐ%WÐ%WÐ"à Œ;Ô "ñ RÝ&(¤g×&6Ò&6µr´w×7IÒ7IÈ$Ì+ÔJaÑ7bÔ7bÑ&cÔ&cˆDŒKÔ #Ø ¥B¤G×$4Ò$4°T´[Ô5LÑ$MÔ$MÐMÐMå "ÝØ4Ð4Ýœ
|
|||
|
|
4¤;Ô#:Ñ;Ô;ñô ñ ô � õ �{Ñ#Ô# qÒ(Ð(Ý$Ð%YÀÄ Ô@WÐ%YÐ%YÑZÔZÐZÝ%+Ð,XÐ,XÈKÐ,XÑ,XÔ,XÑ%YÔ%YÐ"Ý*,¬'¯,ª,Ø”KÔ+Ø:Ð"4°RÔ"8Ð:Ð:ñ+ô +�” Ô'ð
|
|||
|
|
8JÈ"Ô7MÐPQÑ7QÐ*Ô4å&ð
|
|||
|
|
Ø”[Ô)Ø œKÔ 7Ø5ð )-¬ Ô(Uð
|
|||
|
|
ð
|
|||
|
|
ð ŒkÔ,ð
|
|||
|
|
ð
|
|||
|
|
ˆÔð &œ°dÐ :Ð _¸v¼ÐR_Ò?_Ð à Ô Ô +ð Ø Ô × *Ò *Ø” Ô0à+ð&•t°V·^²^Ñ5EÔ5EÐFÑFÔFÐFà—^’^Ñ%Ô%Ø œKÔ 6ð
|
|||
|
|
+ñ
|
|||
|
|
ô
|
|||
|
|
ð
|
|||
|
|
õ Œ �M˜�M�MÑ"Ô"Ð"å�”Ô!°4Ð8Ñ8Ô8Ð8à&ˆÔà Ô×0Ò0ØØ Ô(Ô>Ð>ØØØ ð 1ñ
|
|||
|
|
ô
|
|||
|
|
ð
|
|||
|
|
ð Ô Ô +¨vÒ 5Ð 5Ý #œmˆOˆOØ
|
|||
|
|
Ô
|
|||
|
|
Ô
|
|||
|
|
-°Ò
|
|||
|
|
7Ð
|
|||
|
|
7Ý #œnˆOˆOå #œmˆOà ÔÔ × Ò Ô 0Ô 7¸Ð ÑOÔOÐOØ ÔÔ%×(Ò(¨Ô)9Ô)@ÈÐ(ÑXÔXÐXØ ÔÔ × Ò Ô!1Ô!8ÀÐ ÑPÔPÐPà Ô+×@Ò@ÑBÔBÐà Ô×5Ò5°dÔ6KÑLÔLÐLØ Ô×5Ò5°dÔ6KÑLÔLÐLð Œ;Ô !ð 9Ø48�EŒNÔ Ô &Ô 1à ×.Ò.Ý1;Ð<LÍdÑ1SÔ1SÐ iÐ × 'Ò 'Ñ )Ô )Ð )ÐYiñ
|
|||
|
|
ô
|
|||
|
|
ˆŒð !%Ô 0× =Ò =Ø Ô × &Ò &Ø œ Ô4Ð<���À$Ä+ÔB^Ø #Ø$Ø Ø Ô+Ô5ÔFð
|
|||
|
|
'ñ
|
|||
|
|
ô
|
|||
|
|
ô Ÿš˜4Ô+Ô2Ñ3Ô3ñ!
|
|||
|
|
ô !
|
|||
|
|
ð ô!
|
|||
|
|
ˆÔ ð Ô(Ô1ÐN°TÔ5EÔ5NˆŒ
|
|||
|
|
å �4Ô# ZÑ 0Ô 0ð o°TÔ5EÔ5Nð oØ#'Ô#3×#;Ò#;Ð<LÈdÌnÑ#]Ô#]Ñ ˆD�$”.Ý$(Ð0IÐ0IÈ4Ï?Ê?ÑK\ÔK\Ñ)]Ô)]Ñ$^Ô$^ˆDÔ !Ð !à48Ô4D×4LÒ4LÐM]Ð_cÔ_mÑ4nÔ4nÑ 1ˆDÔ ! 4¤>à Ô ð !Ý ŒKÐ=¨Ô);Ð=Ð=Ñ >Ô >Ð >Ø Ô × 'Ò '¨Ô(:Ñ ;Ô ;Ð ;Ý " 6Ô#5×#;Ò#;¸CÑ#@Ô#@ÀÔ#DÑ EÔ EÈÑ IˆDÔ Ð Ð à ˆDÔ Ð Ð rI c ó^ — | |d |d |d ¦ « \ }}|S )NÚimagesÚpromptsÚprompt_metadata)r¾ )r„ Úprompt_image_pairsÚrewardÚreward_metadatas rG Úcompute_rewardsz(_UnslothAlignPropTrainer.compute_rewards] s: € Ø"&§.¢.Ø ˜xÔ (Ð*<¸YÔ*GÐI[Ð\mÔInñ#
|
|||
|
|
ô #
|
|||
|
|
Ñ ˆ�ð ˆ
|
|||
|
|
rI ÚepochÚ global_stepc
|
|||
|
|
óÄ — t t ¦ « }| j j ¦ « t | j j ¦ « D �]q}| j | j j ¦ « 5 |
|
|||
|
|
¦ « 5 t j ¦ « 5 |
|
|||
|
|
| j j ¬¦ « }| |¦ « }||d< | j |¦ « ¦ « ¦ « ¦ « }| |¦ « }| j |¦ « | j j r]| j t1 | j t ¦ « s| j ¦ « n| j | j j ¦ « | j ¦ « | j ¦ « ddd¦ « n # 1 swxY w Y ddd¦ « n # 1 swxY w Y ddd¦ « n # 1 swxY w Y |d | ¦ « ¦ « |d | ! ¦ « ¦ « |d | " ¦ « ¦ « �Œs| j j r‡d„ | # ¦ « D ¦ « }| j $ |d¬ ¦ « }| % d
|
|||
|
|
|i¦ « | j & ||¬ ¦ « |d z
|
|||
|
|
}t t ¦ « }ntO d
|
|||
|
|
¦ « ‚| j( �:|| j j) z dk r'| ( ||| j j* d ¦ « |dk r8|| j j+ z dk r%| j j, r| j - ¦ « |S )a
|
|||
|
|
Perform a single step of training.
|
|||
|
|
|
|||
|
|
Args:
|
|||
|
|
epoch (int): The current epoch.
|
|||
|
|
global_step (int): The current global step.
|
|||
|
|
|
|||
|
|
Side Effects:
|
|||
|
|
- Model weights are updated
|
|||
|
|
- Logs the statistics to the accelerator trackers.
|
|||
|
|
- If `self.image_samples_callback` is not None, it will be called with the prompt_image_pairs, global_step,
|
|||
|
|
and the accelerator tracker.
|
|||
|
|
|
|||
|
|
Returns:
|
|||
|
|
global_step (int): The updated global step.
|
|||
|
|
)Ú
|
|||
|
|
batch_sizeÚrewardsNÚ reward_meanÚ
|
|||
|
|
reward_stdÚlossc ób — i | ],\ }}|t j t j |¦ « ¦ « “Œ-S r� )r ÚmeanÚtensor)rŸ ÚkÚvs rG ú
|
|||
|
|
<dictcomp>z1_UnslothAlignPropTrainer.step.<locals>.<dictcomp>š s0 € ÐLÐLÐL±t°q¸!�A•u”z¥%¤,¨q¡/¤/Ñ2Ô2ÐLÐLÐLrI r )Ú reductionr )Ústepr1 zsOptimization step should have been performed by this point. Please check calculated gradient accumulation settings.r ).r rÅ r• rÜ ÚtrainÚranger’ r{ rÎ Ú
|
|||
|
|
accumulaterí r Ú enable_gradÚ_generate_samplesrt rÿ r8 ÚdetachÚcpuÚnumpyÚcalculate_lossÚbackwardÚsync_gradientsÚclip_grad_norm_ræ rð rç r| rè r Ú zero_gradr< r ÚstdÚitemÚitemsÚreduceÚupdateÚlogrÉ r¿ rh Útrackersrl rÏ Ú
|
|||
|
|
save_state) r„ r r rÔ r� rü r Ú rewards_visr s rG r z _UnslothAlignPropTrainer.stepc sN € õ" �4Ñ Ô ˆà ÔÔ ×#Ò#Ñ%Ô%Ð%å�t”{ÔDÑEÔEð -ñ -ˆAØÔ!×,Ò,¨TÔ-=Ô-BÑCÔCð
|
|||
|
|
+ð
|
|||
|
|
+ÀTÇ]Â]Á_Ä_ð
|
|||
|
|
+ð
|
|||
|
|
+ÕV[ÔVgÑViÔVið
|
|||
|
|
+ð
|
|||
|
|
+Ø%)×%;Ò%;Ø #œ{Ô ;ð &<ñ &ô &Ð"ð ×.Ò.Ð/AÑBÔB�à07Ð" 9Ñ-à "Ô .× 5Ò 5°gÑ >Ô >× EÒ EÑ GÔ G× KÒ KÑ MÔ M× SÒ SÑ UÔ U� à×*Ò*¨7Ñ3Ô3�àÔ ×)Ò)¨$Ñ/Ô/Ð/àÔ#Ô2ð ØÔ$×4Ò4å )¨$Ô*?ÅÑ FÔ Fð3˜Ô-×8Ò8Ñ:Ô:Ð:à !Ô 2Ø œ Ô7ñ ô ð ð ”×#Ò#Ñ%Ô%Ð%Ø”×(Ò(Ñ*Ô*Ð*ð1
|
|||
|
|
+ð
|
|||
|
|
+ð
|
|||
|
|
+ñ
|
|||
|
|
+ô
|
|||
|
|
+ð
|
|||
|
|
+ð
|
|||
|
|
+ð
|
|||
|
|
+ð
|
|||
|
|
+ð
|
|||
|
|
+ð
|
|||
|
|
+øøøð
|
|||
|
|
+ð
|
|||
|
|
+ð
|
|||
|
|
+ð
|
|||
|
|
+ð
|
|||
|
|
+ð
|
|||
|
|
+ð
|
|||
|
|
+ñ
|
|||
|
|
+ô
|
|||
|
|
+ð
|
|||
|
|
+ð
|
|||
|
|
+ð
|
|||
|
|
+ð
|
|||
|
|
+ð
|
|||
|
|
+ð
|
|||
|
|
+øøøð
|
|||
|
|
+ð
|
|||
|
|
+ð
|
|||
|
|
+ð
|
|||
|
|
+ð
|
|||
|
|
+ð
|
|||
|
|
+ð
|
|||
|
|
+ñ
|
|||
|
|
+ô
|
|||
|
|
+ð
|
|||
|
|
+ð
|
|||
|
|
+ð
|
|||
|
|
+ð
|
|||
|
|
+ð
|
|||
|
|
+ð
|
|||
|
|
+øøøð
|
|||
|
|
+ð
|
|||
|
|
+ð
|
|||
|
|
+ð
|
|||
|
|
+ð4
|
|||
|
|
� Ô × &Ò & {×'7Ò'7Ñ'9Ô'9Ñ :Ô :Ð :Ø � Ô × %Ò % k§o¢oÑ&7Ô&7Ñ 8Ô 8Ð 8Ø �ŒL× Ò § ¢ ¡ ¤ Ñ ,Ô ,Ð ,Ñ ,ð Ô Ô *ð àLÐL¸t¿zºz¹|¼|ÐLÑLÔLˆDØÔ#×*Ò*¨4¸6Ð*ÑBÔBˆDØ �KŠK˜ %Ð(Ñ )Ô )Ð )Ø Ô × Ò ¨KÐ Ñ 8Ô 8Ð 8Ø ˜1Ñ ˆKÝ �tÑ$Ô$ˆDˆDå ð Fñô ð
|