918 lines
146 KiB
Plaintext
918 lines
146 KiB
Plaintext
|
§
|
|||
|
|
4$�h¼] ã ó — d Z ddlmZ ddlZddlmZ ddlmZ ddlmZm Z m
|
|||
|
|
Z
|
|||
|
|
m Z m Z m
|
|||
|
|
Z
|
|||
|
|
mZmZ ddlmZmZmZmZmZmZmZmZmZmZmZmZm
|
|||
|
|
Z
|
|||
|
|
m Z m Z m Z m Z m Z m!Z!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z'm Z m(Z(m)Z)m*Z*m+Z+m,Z,m-Z-m.Z.m/Z/m0Z0m1Z1m2Z2m3Z3m4Z4m5Z5m6Z6m7Z7m8Z8m9Z9m:Z:m;Z;m<Z<m=Z=m>Z>m?Z?m@Z@mAZAmBZBmCZCmDZDmZmEZEmFZFmGZGmHZHmIZImJZJmKZKmLZLmMZMmNZNmOZOmPZPmQZQmRZRmSZSmTZTmUZUmZmVZVmWZWmXZXmYZYmZmZm Z m*Z*m+Z+m,Z,m2Z2m3Z3m8Z8m:Z:mAZAmDZDmEZEmFZFmGZGmLZLmNZNmZmVZVmXZXm1Z1mFZFmNZNmPZPmZmVZVmNZNmZm Z mFZFmMZMmNZNmRZRmSZSmTZTmZmWZWm
|
|||
|
|
Z
|
|||
|
|
m Z m&Z&m<Z<mFZFmNZNmZmZmZmFZFmNZNmZmZmNZNmZmZm&Z&m2Z2mFZFmNZNmZ ddlFZFddlT ddlZm[Z[m\Z\ dd l]m^Z^ ddlZddl_Z`dd
|
|||
|
|
lamEZE ddlmZ dd lVmbZbmcZd d d
|
|||
|
|
d d
|
|||
|
|
d
|
|||
|
|
dœZe ejf d d ee¬¦ « d„ ¦ « Zgd„ Zh G d„ deji jj ¦ « Zk d d„Zl ejf d d ee¬¦ « d„ ¦ « Zmd„ Zne[ G d„ de¦ « ¦ « Zo G d„ de&¦ « Zp G d „ d ep¦ « ZqdS ) z8
|
|||
|
|
2025.8.4
|
|||
|
|
2025.8.5
|
|||
|
|
4.55.1
|
|||
|
|
0.21.0
|
|||
|
|
__UNSLOTH_VERSIONING__
|
|||
|
|
é )ÚTensorN)Ú
|
|||
|
|
functional)ÚAnyÚListÚOptionalÚTupleÚUnionÚDictÚSetÚCallable)ˆr Ú
|
|||
|
|
AutoConfigÚ"AutoModelForSequenceClassificationÚ
|
|||
|
|
AutoProcessorÚ
|
|||
|
|
AutoTokenizerÚ
|
|||
|
|
DataLoaderÚDatasetÚFSDPÚ
|
|||
|
|
GRPOConfigÚ GRPOTrainerÚGenerationConfigÚIterableDatasetr ÚPathÚ
|
|||
|
|
PeftConfigÚPreTrainedModelÚPreTrainedTokenizerBaseÚProcessorMixinÚ
|
|||
|
|
RepeatSamplerÚ
|
|||
|
|
RewardFuncÚSamplerÚSequenceÚSyncRefModelCallbackÚTrainerÚTrainerCallbackr Ú
|
|||
|
|
VLLMClientÚ_ForwardRedirectionÚapply_chat_templateÚbroadcast_object_listÚcopyÚdatasetsÚ defaultdictÚdequeÚdisable_dropout_in_modelÚentropy_from_logitsÚgatherÚ
|
|||
|
|
gather_objectÚgenerate_model_cardÚget_comet_experiment_urlÚidentityÚinspectÚis_conversationalÚis_datasets_availableÚis_flash_attn_2_availableÚis_liger_kernel_availableÚis_peft_availableÚ
|
|||
|
|
is_peft_modelÚis_rich_availableÚis_vllm_availableÚis_wandb_availableÚmaybe_apply_chat_templateÚnanmaxÚnanminÚnanstdÚnnÚ nullcontextÚosÚpadÚpartialÚprepare_deepspeedÚ prepare_fsdpÚ print_prompt_completions_sampleÚprofiling_contextÚprofiling_decoratorÚreÚ seed_workerÚselective_log_softmaxÚset_seedÚshuffle_sequence_dictÚsplit_pixel_values_by_gridÚsplit_tensor_dictÚtextwrapÚtorchÚ transformersÚ unsplit_pixel_values_by_gridÚunwrap_model_for_generationÚwarningsr r r r& r' r( r. r/ r4 r6 r= r@ rB rC rD rI rK rS rT rV r- rC rK rM rS rT rK r r rC rJ rK rO rP rQ rS rU r r r" r8 rC rK rS r rA rC rK r rA rK rS r r" r. rC rK rS )Ú*)Ú dataclassÚfield)ÚVersion)rB )ÚDataCollatorForSeq2SeqÚ DataCollatorForLanguageModelingTF)Úepilogue_fusionÚ max_autotuneÚ
|
|||
|
|
shape_paddingz
|
|||
|
|
trace.enabledztriton.cudagraphs)ÚdynamicÚ fullgraphÚoptionsc ó’ — t j | d| j d ¦ « dd¬¦ « }t j | d¦ « dd¬¦ « }g }t ||¦ « D ]‘\ }}| t j ¦ « }t j |d| d¦ « ¬¦ « d¦ « }t j
|
|||
|
|
|d¬¦ « }||z
|
|||
|
|
} | | ¦ « Œ’ t j |¦ « }| | j d | j d f¦ « }|S )Néÿÿÿÿé r ©ÚchunksÚdim©ri Úindex©ri é )
|
|||
|
|
rS ÚchunkÚreshapeÚshapeÚzipÚtoÚfloat32r. Ú unsqueezeÚsqueezeÚ logsumexpÚappendÚconcat)
|
|||
|
|
Úlogitsrk Úchunked_logitsÚ
|
|||
|
|
chunked_indexÚall_per_token_logpsÚ chunk_logitsÚ chunk_indexÚselected_logitsÚlogsumexp_valuesÚper_token_logpss
|
|||
|
|
ú^/workspace/Fine-tuning/DS-LLM-TEMPLATE-FINETUNING/unsloth_compiled_cache/UnslothGRPOTrainer.pyÚ chunked_selective_log_softmaxrƒ " s5 € õ ”[ § ¢ °°F´LÀÔ4DÑ!EÔ!EÐPQÐYZÐ[Ñ[Ô[€NÝ”[ § ¢ ¨rÑ!2Ô!2¸QÀaÐHÑHÔH€MØ Ðå%(¨ ¸ Ñ%GÔ%Gð 4ð 4Ñ!ˆ �kØ#—’¥u¤}Ñ5Ô5ˆ Ý œ, |¸2À{×G\ÒG\Ð]_ÑG`ÔG`ÐaÑaÔa×iÒiÐjlÑmÔmˆÝ œ?¨<¸rÐBÑBÔBÐØ)Ð,<Ñ<ˆØ×"Ò" ?Ñ3Ô3Ð3Ð3ØÝ œ,Ð':Ñ;Ô;ÐØ-×5Ò5°v´|ÀA´ÈÌ ÐUVÌÐ6XÑYÔYÐØ Ð ó c óL — | dd¦ « }| dd¦ « } | dd¦ « }
|
|||
|
|
| dd¦ « } | dd ¦ « } | d d
|
|||
|
|
¦ « }
|
|||
|
|
| d d ¦ « }| d
|
|||
|
|
d ¦ « }| dd ¦ « }| d¦ « }|dk r||z }|dk r||z }|dk r|t j ||z ¦ « z }| t j ¦ « }|
|
|||
|
|
d
|
|||
|
|
k r||
|
|||
|
|
z }t j |d|¬¦ « d¦ « }|t j |d¬¦ « z
|
|||
|
|
}t j ¦ « 5 |d k r¯| €
|
|||
|
|
J d¦ « ‚|dk r| |z } |dk r| |z } |dk r| t j | |z ¦ « z } | t j ¦ « } |
|
|||
|
|
d
|
|||
|
|
k r| |
|
|||
|
|
z } t j | d|¬¦ « d¦ « }|t j | d¬¦ « z
|
|||
|
|
} |�£|dk r||z }|dk r||z }|dk r|t j ||z ¦ « z }| t j ¦ « }|
|
|||
|
|
d
|
|||
|
|
k r||
|
|||
|
|
z }t j |d|¬¦ « d¦ « }|t j |d¬¦ « z
|
|||
|
|
} d d d ¦ « n # 1 swxY w Y |d k r!t j
|
|||
|
|
||z
|
|||
|
|
¦ « ||z
|
|||
|
|
z
|
|||
|
|
d
|
|||
|
|
z
|
|||
|
|
}nd }|�t j
|
|||
|
|
||z
|
|||
|
|
¦ « }n)t j
|
|||
|
|
|| ¦ « z
|
|||
|
|
¦ « }t j |d| z
|
|||
|
|
d|
|
|||
|
|
z ¦ « }| �-t j || ¬¦ « | d¦ « z }n|| d¦ « z } || d¦ « z }t j
|
|||
|
|
||¦ « } |d k r| ||z z } | t j ¦ « }| d¦ « } |dk rU| |z d¦ « | d¦ « d
|
|||
|
|
¬¦ « z ¦ « } n�|dk rA| |z ¦ « | ¦ « d
|
|||
|
|
¬¦ « z } nI|dk r1| |z ¦ « | d¦ « | z z } nt# d|› �¦ « ‚t j ¦ « 5 | ¦ « } ||z d¦ « | z } | ¦ « }!d d d ¦ « n # 1 swxY w Y | | |!fS ©NÚ loss_typeÚgrpoÚ epsilon_lowçš™™™™™É?Ú epsilon_highÚmax_completion_lengthi ÚdeltaÚ temperatureç ð?Úlogit_scale_multiplyç Úlogit_scale_divideÚlogit_softcappingre r rj rl z.ref_logits should not be None when beta != 0.0rm ©Úmax©ÚminÚbnpoÚdr_grpoúUnknown loss type: ©Úgetrt rS Útanhrr rs r. ru rv Úno_gradÚexpÚdetachÚclampr— ÚsumÚmeanÚsizeÚ
|
|||
|
|
ValueErrorÚinference_mode©"Ú
|
|||
|
|
ref_logitsÚ
|
|||
|
|
new_logitsÚ
|
|||
|
|
old_logitsÚ input_idsÚmaskÚbetaÚ
|
|||
|
|
advantagesÚkwargsr‡ r‰ r‹ rŒ r� rŽ r� r’ r“ Únew_xÚnewÚref_xÚrefÚold_xÚoldÚkl_iÚcoef_1Úcoef_2Úloss_1Úloss_2Úloss_iÚn_mask_per_rewardÚlossÚcompletion_lengthÚmean_kl_per_rewardÚmean_kls" r‚ Úgrpo_compute_lossrÁ 4 sN € ð —
|
|||
|
|
’
|
|||
|
|
˜;¨Ñ/Ô/€IØ—*’*˜]¨CÑ0Ô0€KØ—:’:˜n¨cÑ2Ô2€LØ "ŸJšJÐ'>ÀÑ EÔ EÐØ �JŠJ�w Ñ %Ô %€EØ—*’*˜]¨CÑ0Ô0€KØ!Ÿ:š:Ð&<¸cÑBÔBÐØ!Ÿ:š:Ð&:¸CÑ@Ô@ÐØ!Ÿ:š:Ð&9¸3Ñ?Ô?Ðà×#Ò# BÑ'Ô'€Ið ˜qÒ Ð ¨zÐ<PÑ/P *ؘqÒ Ð ¨zÐ<NÑ/N *ؘqÒ Ð ¨z½E¼JÀzÐTeÑGeÑ<fÔ<fÑ/f *à— ’ �uœ}Ñ-Ô-€Jà�cÒШ
|
|||
|
|
°[Ñ(@˜:Ý ŒL˜¨2°yÐ AÑ AÔ A× IÒ IÈ"Ñ MÔ M€EØ
|
|||
|
|
•%”/ *°BÐ7Ñ7Ô7Ñ
|
|||
|
|
7€Cõ
|
|||
|
|
Œ ‰ Œ ð
|
|||
|
|
ð
|
|||
|
|
Ø �3Š;ˆ;Ø Ð)Ð)Ð+[Ñ)Ô)Ð)ð $ qÒ(Ð(°zÐDXÑ7X¨*Ø! qÒ(Ð(°zÐDVÑ7V¨*Ø qÒ(Ð(°zÅEÄJÈzÐ\mÑOmÑDnÔDnÑ7n¨*à#Ÿ š ¥u¤}Ñ5Ô5ˆJà˜cÒ!Ð!°
|
|||
|
|
¸[Ñ0H :Ý”L °2¸yÐIÑIÔI×QÒQÐRTÑUÔUˆEØ�%œ/¨*¸BÐ?Ñ?Ô?Ñ?ˆCØ à Ð !à# qÒ(Ð(°zÐDXÑ7X¨*Ø! qÒ(Ð(°zÐDVÑ7V¨*Ø qÒ(Ð(°zÅEÄJÈzÐ\mÑOmÑDnÔDnÑ7n¨*à#Ÿ š ¥u¤}Ñ5Ô5ˆJà˜cÒ!Ð!°
|
|||
|
|
¸[Ñ0H :Ý”L °2¸yÐIÑIÔI×QÒQÐRTÑUÔUˆEØ�%œ/¨*¸BÐ?Ñ?Ô?Ñ?ˆCØ ð7
|
|||
|
|
ð
|
|||
|
|
ð
|
|||
|
|
ñ
|
|||
|
|
ô
|
|||
|
|
ð
|
|||
|
|
ð
|
|||
|
|
ð
|
|||
|
|
ð
|
|||
|
|
ð
|
|||
|
|
ð
|
|||
|
|
øøøð
|
|||
|
|
ð
|
|||
|
|
ð
|
|||
|
|
ð
|
|||
|
|
ð8 ð ˆs‚{€{ÝŒy˜˜s™Ñ#Ô# s¨S¡yÑ1°CÑ7ˆˆð ˆð Ð Ý”˜3 ™9Ñ%Ô%ˆˆå”˜3 §¢¡ ¤ Ñ-Ñ.Ô.ˆÝ
|
|||
|
|
Œ[˜ [¡ °!°lÑ2BÑ
|
|||
|
|
CÔ
|
|||
|
|
C€Fà ÐÝ”˜V¨Ð/Ñ/Ô/°*×2FÒ2FÀqÑ2IÔ2IÑIˆˆà˜*×.Ò.¨qÑ1Ô1Ñ1ˆØð �j×*Ò*¨1Ñ-Ô-Ñ
|
|||
|
|
-€FÝŒi˜ Ñ'Ô'Ð
|
|||
|
|
'€FØ ˆs‚{€{ؘ$ ™+Ñ%ˆà �7Š7•5”=Ñ !Ô !€DØ Ÿš ™ œ Ðð �FÒÐؘ$‘ ×#Ò# BÑ'Ô'¨$¯(ª(°2©,¬,×*<Ò*<ÀÐ*<Ñ*EÔ*EÑE×KÒKÑMÔMˆˆØ �fÒ Ð Ø˜‘
|
|||
|
|
×"Ò"Ñ$Ô$ t§x¢x¡z¤z×'7Ò'7¸CÐ'7Ñ'@Ô'@Ñ@ˆˆØ �iÒ Ð Ø˜‘
|
|||
|
|
×"Ò"Ñ$Ô$¨¯ ª °A©¬Ð9NÑ(NÑOˆˆåÐ:¨yÐ:Ð:Ñ;Ô;Ð;õ
|
|||
|
|
|
|||
|
|
Ô Ñ Ô ð ,ð ,Ø -× 2Ò 2Ñ 4Ô 4ÐØ " T™k× .Ò .¨qÑ 1Ô 1Ð4EÑ EÐØ$×)Ò)Ñ+Ô+ˆð,ð ,ð ,ñ ,ô ,ð ,ð ,ð ,ð ,ð ,ð ,øøøð ,ð ,ð ,ð ,ð à Ð" GÐ +Ð+ó& ÆE K;Ë;K?ÌK?ÕAVÖVÖ Vc ó> — e Zd Zedd„¦ « Z ed„ ¦ « ZdS )ÚUnslothEfficientGRPONrm c óô ‡‡‡ ‡ ‡ ‡ ‡!— ‰ €i Š ˆˆ ˆfd„Š! |j } t j |¦ « }
|
|||
|
|
t j d| ¬¦ « Š t j d| ¬¦ « Š t j d| ¬¦ « Š ˆ ˆ ˆ ˆ!fd„} t j |ddt
|
|||
|
|
¬¦ « }t j |
|
|||
|
|
|
|
|||
|
|
d¬¦ « }t j ||
|
|||
|
|
d¬¦ « }|�t j ||
|
|||
|
|
d¬¦ « }nd g|
|
|||
|
|
z }t j ||
|
|||
|
|
d¬¦ « }t j ||
|
|||
|
|
d¬¦ « }t j ||
|
|||
|
|
d¬¦ « }t j ||
|
|||
|
|
d¬¦ « }| �| ¦ « nd }t |||||||¦ « D ] \ }}}}}} } |||||| | ||¦ « Œ |
|
|||
|
|
|
|
|||
|
|
¦ « ‰ |
|
|||
|
|
¦ « ‰ |
|
|||
|
|
¦ « ‰ |
|
|||
|
|
¦ « |
|
|||
|
|
|
|
|||
|
|
¦ « ‰ ‰ ‰ fS )
|
|||
|
|
Nc ó0 •— t j | ‰ ¦ « ¦ « }|d d …d d…d d …f }t j ¦ « 5 ‰dk r9t j |‰ ¦ « ¦ « }|d d …d d…d d …f }nd }|�9t j |‰ ¦ « ¦ « } | d d …d d…d d …f } nd } d d d ¦ « n # 1 swxY w Y t ||| ||‰|fi ‰¤Ž\ }
|
|||
|
|
} } |
|
|||
|
|
|z }
|
|||
|
|
|
|
|||
|
|
|
|
|||
|
|
¦ « | | ffS )Nre r‘ )rS ÚmatmulÚtrž rÁ r )Únew_hidden_statesÚold_hidden_statesÚref_hidden_statesr« r¬ r® Úscalingr© r¨ rª r½ r¾ rÀ Ú scaled_lossr Ú extra_kwargsÚlm_heads €€€r‚ Ú compute_lossz2UnslothEfficientGRPO.forward.<locals>.compute_lossµ s‡ ø€ Ý œ Ð&7¸¿º¹¼ÑEÔEˆJØ# A A A s¨ s¨A¨A¨A IÔ.ˆJÝ” ‘ ” ð
|
|||
|
|
|
|||
|
|
&ð
|
|||
|
|
|
|||
|
|
&ؘ3’;�;Ý!&¤ Ð.?ÀÇÂÁÄÑ!MÔ!M�JØ!+¨A¨A¨A¨s°¨s°A°A°A¨IÔ!6�J�Jà!%�JØ$Ð0Ý!&¤ Ð.?ÀÇÂÁÄÑ!MÔ!M�JØ!+¨A¨A¨A¨s°¨s°A°A°A¨IÔ!6�J�Jà!%�Jð
|
|||
|
|
|
|||
|
|
&ð
|
|||
|
|
|
|||
|
|
&ð
|
|||
|
|
|
|||
|
|
&ñ
|
|||
|
|
|
|||
|
|
&ô
|
|||
|
|
|
|||
|
|
&ð
|
|||
|
|
|
|||
|
|
&ð
|
|||
|
|
|
|||
|
|
&ð
|
|||
|
|
|
|||
|
|
&ð
|
|||
|
|
|
|||
|
|
&ð
|
|||
|
|
|
|||
|
|
&ð
|
|||
|
|
|
|||
|
|
&øøøð
|
|||
|
|
|
|||
|
|
&ð
|
|||
|
|
|
|||
|
|
&ð
|
|||
|
|
|
|||
|
|
&ð
|
|||
|
|
|
|||
|
|
&õ" 0AØØØØØØØð 0ð 0ð ð 0ð 0Ñ ,ˆDÐ# Wð ™.ˆKà §¢¡ ¤ Ð0AÀ7Ð LÐLÐ Ls Á
|
|||
|
|
A?CÃC à C rm ©Údevicec ó •— t j ‰dd¬¦ « | ||||||¦ « \ \ }\ } \ }
|
|||
|
|
} } ‰ |
|
|||
|
|
¦ « ‰
|
|||
|
|
| ¦ « ‰ | ¦ « ||d d …< d S )N©r T)ÚargnumsÚhas_aux)rS ÚfuncÚgrad_and_valueÚadd_)Únew_hidden_states_jÚold_hidden_states_jÚref_hidden_states_jÚ input_ids_jÚmask_jÚ advantages_jrÌ Ú
|
|||
|
|
grad_inputs_jÚchunk_grad_inputÚ
|
|||
|
|
chunk_lossÚ
|
|||
|
|
unscaled_lossÚchunk_completion_lengthÚ
|
|||
|
|
chunk_mean_klÚ accumulated_completion_lengthÚaccumulated_lossÚaccumulated_mean_klrÐ s €€€€r‚ Úaccumulate_chunkz6UnslothEfficientGRPO.forward.<locals>.accumulate_chunkà sÇ ø€ õ kpÔjt÷ kDò kDØ Ø Ø ð kDñ kô kð "Ð#6Ð8KÈ[ÐZ`ÐbnÐpwñ kyô kyÑ gÑ Ð
|
|||
|
|
Ñ!g *Ñ.f¨}Ð>UÐWdð
|
|||
|
|
|
|||
|
|
× .Ò .¨}Ñ =Ô =Ð =Ø )× .Ò .Ð/FÑ GÔ GÐ GØ × .Ò .¨}Ñ =Ô =Ð =Ø /ˆM˜!˜!˜!Ñ Ð Ð r„ T)rb ra rc r rg r� ) rÒ rS Ú
|
|||
|
|
empty_likeÚzerosÚcompileÚtorch_compile_optionsrn Ú get_scalerq Údiv_Úsave_for_backward)"ÚctxÚ_new_hidden_statesÚ_old_hidden_statesÚ_ref_hidden_statesrÏ Ú
|
|||
|
|
_input_idsÚ_maskÚ _advantagesr ÚscalerÚn_chunksrÎ rÒ Ú grad_inputsré Úgrad_inputs_chunksrÉ rÊ rË r« r¬ r® rÌ rà rÚ rÛ rÜ rÝ rÞ rß ræ rç rè rÐ s" ` ` ` @@@@r‚ Úforwardz UnslothEfficientGRPO.forward± sº øøøøøøø€ à Ð Ø ˆLð" Mð " Mð " Mð " Mð " Mð " Mð " MðF
|
|||
|
|
à"Ô)ˆÝÔ&Ð'9Ñ:Ô:ˆ Ý(-¬ °AÀÐ(GÑ(GÔ(GÐÝ(-¬ °AÀÐ(GÑ(GÔ(GÐ%Ý(-¬ °AÀÐ(GÑ(GÔ(GÐð 0ð 0ð 0ð 0ð 0ð 0ð 0ð 0ð&
|
|||
|
|
å œ=Ø Ø àÝ+ð
|
|||
|
|
ñ
|
|||
|
|
ô
|
|||
|
|
Ðõ #œ[¨ÀhÐVWÐ XÑ XÔ XÐÝ "œ[Ð);ÀhÐVWÐ XÑ XÔ XÐØ Ð )Ý!&¤Ð-?È(ÐZ[Ð!\Ñ!\Ô!\Ð Ð à!% ¨Ñ 1Ð Ý "œ[Ð);ÀhÐVWÐ XÑ XÔ XÐÝ "œ[¨ÀhÐVWÐ XÑ XÔ Xˆ Ý "œ[¨ÀhÐVWÐ XÑ XÔ XˆÝ "œ[¨ÀhÐVWÐ XÑ XÔ Xˆ
|
|||
|
|
ð )/Ð(:�&×"Ò"Ñ$Ô$Ð$Àˆõ
|
|||
|
|
Ð"Ð$5Ð7HÐJ[Ð]fÐhlÐnxÑ yÔ yð ð Ñ ˆ]Ð /Ð1DÐFYÐ\gÐioÐq}ð
|
|||
|
|
Ð Ø#Ø#Ø#ØØØ ØØ ñ
|
|||
|
|
ô
|
|||
|
|
ð
|
|||
|
|
ð
|
|||
|
|
ð
|
|||
|
|
à×*Ò*¨8Ñ4Ô4Ð4Ø×*Ò*¨8Ñ4Ô4Ð4Ø%×*Ò*¨8Ñ4Ô4Ð4Ø×*Ò*¨8Ñ4Ô4Ð4Ø × Ò ˜kÑ*Ô*Ð*à Ø )Ø ð
|
|||
|
|
ð
|
|||
|
|
r„ c ó. — | j \ }|d d d d d d d d d d f S ©N)Ú
|
|||
|
|
saved_tensors)rñ Ú grad_outputÚdcompletion_lengthÚdmean_klÚ
|
|||
|
|
grad_inputs r‚ Úbackwardz UnslothEfficientGRPO.backward2 s* € àÔ)‰
|
|||
|
|
ˆØ˜D $¨¨d°D¸$ÀÀdÈDÐRVÐWÐWr„ )Nrm N)Ú__name__Ú
|
|||
|
|
__module__Ú __qualname__Ú staticmethodrü r © r„ r‚ rÄ rÄ ¯ sU € € € € € àð}
|
|||
|
|
ð }
|
|||
|
|
ð }
|
|||
|
|
ñ „\ð}
|
|||
|
|
ð| àðXð Xñ „\ðXð €Dr„ rÄ re c óþ ‡— |j \ Š} ˆfd„t d‰dz ¦ « D ¦ « }
|
|||
|
|
|dk r‰}|
|
|||
|
|
t t j |
|
|||
|
|
|¦ « t |
|
|||
|
|
¦ « dz
|
|||
|
|
¦ « }t
|
|||
|
|
| d¦ « svt j dd¦ « dk r t j n t j | _
|
|||
|
|
t j dd¦ « d k rt j | _
|
|||
|
|
d t j d
|
|||
|
|
< |d d …| d …f } | j ¦ « j } t j | j j j | j
|
|||
|
|
¬ ¦ « 5 t j ¦ « 5 | j | j d ¬
|
|||
|
|
¦ « ¦ « 5 | |||dz ¬¦ « j }
|
|||
|
|
d d d ¦ « n # 1 swxY w Y d d d ¦ « n # 1 swxY w Y | |||dz ¬¦ « j }t4 |||
|
|||
|
|
| | ||| j | j j ||¦ « \ }}}d d d ¦ « n # 1 swxY w Y dt j d
|
|||
|
|
< |||fS )Nc ó& •— g | ]
|
|||
|
|
}‰|z d k ¯ |‘ŒS rÔ r )Ú.0ÚiÚbszs €r‚ ú
|
|||
|
|
<listcomp>z)grpo_accumulated_loss.<locals>.<listcomp>G s" ø€ Ð<Ð<Ð<�Q¨s°Q©w¸!ª|¨|ˆq¨|¨|¨|r„ rm re Ú_autocast_dtypeÚACCELERATE_MIXED_PRECISIONÚfp16ÚUNSLOTH_FORCE_FLOAT32Ú0Ú1Ú UNSLOTH_RETURN_HIDDEN_STATES)Ú device_typeÚdtypeF)Úkeep_fp32_wrapper)r« Úattention_maskÚlogits_to_keep)!rp Úranger— ÚnpÚ searchsortedÚlenÚhasattrrC Úenvironrœ rS Úfloat16Úbfloat16r ÚmodelÚget_output_embeddingsÚweightÚampÚautocastrÒ Útyper¦ Ú acceleratorÚ unwrap_modelÚdisable_adapterry rÄ Úapplyr rø rÇ rÈ rÁ )Útrainerr« r r Úcompletion_maskr® rÊ rù r¯ ÚqlenÚfactorsÚcompletion_input_idsrÏ rË rÉ r½ r¾ rÀ r© rª r s @r‚ Úgrpo_accumulated_lossr3 8 sN ø€ ð ”�I€Cˆð =Ð<Ð<Ð<�% 3¨¡7Ñ+Ô+Ð<Ñ<Ô<€GØ�2‚~€~ #�xØ•s�2œ?¨7°HÑ=Ô=½sÀ7¹|¼|ÈA¹~ÑNÔNÔO€Hå �7Ð -Ñ .Ô .ð hÝ35´:·>²>ÐB^Ð`fÑ3gÔ3gÐkqÒ3qÐ3q¥%¤- -Õw|ô xFˆÔ Ý
|
|||
|
|
Œ:�>Š>Ð1°3Ñ 7Ô 7¸3Ò >Ð >ÕZ_ÔZgÀÔ@WØØ14…B„JÐ-Ñ.à$ Q Q Q¨ ¨Ð(8Ð(8Ð%8Ô9ÐØŒm×1Ò1Ñ3Ô3Ô:€Gå Œ× Ò ¨'¬-Ô*>Ô*CÈWÔMdÐ Ñ eÔ eð
|
|||
|
|
ð
|
|||
|
|
Ý
|
|||
|
|
Ô
|
|||
|
|
!Ñ
|
|||
|
|
#Ô
|
|||
|
|
#ð ð WÔ%8×%EÒ%EÀgÄmÐinÐ%EÑ%oÔ%o×%Ò%ñ &Bô &Bð ð Ø '§
|
|||
|
|
¢
|
|||
|
|
Ø %Ø!/Ø!/°!Ñ!3ð !.ñ !ô !ô ð
|
|||
|
|
ð ð ð ñ ô ð ð ð ð ð ð øøøð ð ð ð ð ð ð ñ ô ð ð ð ð ð ð øøøð ð ð ð ð
|
|||
|
|
Ø #ŸMšMØ!Ø +Ø +¨aÑ /ð *ñ
|
|||
|
|
ô
|
|||
|
|
ô ð õ ,@×+EÒ+EØ Ø Ø Ø Ø Ø Ø Ø ŒLØ Ô Ô &Ø Ø ñ ,
|
|||
|
|
ô ,
|
|||
|
|
Ñ(ˆÐ ð
|
|||
|
|
ð
|
|||
|
|
ð
|
|||
|
|
ñ
|
|||
|
|
ô
|
|||
|
|
ð
|
|||
|
|
ð
|
|||
|
|
ð
|
|||
|
|
ð
|
|||
|
|
ð
|
|||
|
|
ð
|
|||
|
|
øøøð
|
|||
|
|
ð
|
|||
|
|
ð
|
|||
|
|
ð
|
|||
|
|
ð6 à14…B„JÐ-Ñ.Ø Ð" GÐ +Ð+s[ Å$I Å84G0Æ,!GÇ
|
|||
|
|
G0ÇG Ç G0Ç G Ç!G0Ç$ I Ç0G4 Ç4I Ç7G4 Ç8AI É I#É&I#c óL — | dd¦ « }| dd¦ « } | dd¦ « }
|
|||
|
|
| dd¦ « } | dd ¦ « } | d d
|
|||
|
|
¦ « }
|
|||
|
|
| d d ¦ « }| d
|
|||
|
|
d ¦ « }| dd ¦ « }| d¦ « }|dk r||z }|dk r||z }|dk r|t j ||z ¦ « z }| t j ¦ « }|
|
|||
|
|
d
|
|||
|
|
k r||
|
|||
|
|
z }t j |d|¬¦ « d¦ « }|t j |d¬¦ « z
|
|||
|
|
}t j ¦ « 5 |d k r¯| €
|
|||
|
|
J d¦ « ‚|dk r| |z } |dk r| |z } |dk r| t j | |z ¦ « z } | t j ¦ « } |
|
|||
|
|
d
|
|||
|
|
k r| |
|
|||
|
|
z } t j | d|¬¦ « d¦ « }|t j | d¬¦ « z
|
|||
|
|
} |�£|dk r||z }|dk r||z }|dk r|t j ||z ¦ « z }| t j ¦ « }|
|
|||
|
|
d
|
|||
|
|
k r||
|
|||
|
|
z }t j |d|¬¦ « d¦ « }|t j |d¬¦ « z
|
|||
|
|
} d d d ¦ « n # 1 swxY w Y |d k r!t j
|
|||
|
|
||z
|
|||
|
|
¦ « ||z
|
|||
|
|
z
|
|||
|
|
d
|
|||
|
|
z
|
|||
|
|
}nd }|�t j
|
|||
|
|
||z
|
|||
|
|
¦ « }n)t j
|
|||
|
|
|| ¦ « z
|
|||
|
|
¦ « }t j |d| z
|
|||
|
|
d|
|
|||
|
|
z ¦ « }| �-t j || ¬¦ « | d¦ « z }n|| d¦ « z } || d¦ « z }t j
|
|||
|
|
||¦ « } |d k r| ||z z } | t j ¦ « }| d¦ « } |dk rU| |z d¦ « | d¦ « d
|
|||
|
|
¬¦ « z ¦ « } n�|dk rA| |z ¦ « | ¦ « d
|
|||
|
|
¬¦ « z } nI|dk r1| |z ¦ « | d¦ « | z z } nt# d|› �¦ « ‚t j ¦ « 5 | ¦ « } ||z d¦ « | z } | ¦ « }!d d d ¦ « n # 1 swxY w Y | | |!fS r† r› r§ s" r‚ Úgrpo_compute_loss_slowr5 „ sN € ð —
|
|||
|
|
’
|
|||
|
|
˜;¨Ñ/Ô/€IØ—*’*˜]¨CÑ0Ô0€KØ—:’:˜n¨cÑ2Ô2€LØ "ŸJšJÐ'>ÀÑ EÔ EÐØ �JŠJ�w Ñ %Ô %€EØ—*’*˜]¨CÑ0Ô0€KØ!Ÿ:š:Ð&<¸cÑBÔBÐØ!Ÿ:š:Ð&:¸CÑ@Ô@ÐØ!Ÿ:š:Ð&9¸3Ñ?Ô?Ðà×#Ò# BÑ'Ô'€Ið ˜qÒ Ð ¨zÐ<PÑ/P *ؘqÒ Ð ¨zÐ<NÑ/N *ؘqÒ Ð ¨z½E¼JÀzÐTeÑGeÑ<fÔ<fÑ/f *à— ’ �uœ}Ñ-Ô-€Jà�cÒШ
|
|||
|
|
°[Ñ(@˜:Ý ŒL˜¨2°yÐ AÑ AÔ A× IÒ IÈ"Ñ MÔ M€EØ
|
|||
|
|
•%”/ *°BÐ7Ñ7Ô7Ñ
|
|||
|
|
7€Cõ
|
|||
|
|
Œ ‰ Œ ð
|
|||
|
|
ð
|
|||
|
|
Ø �3Š;ˆ;Ø Ð)Ð)Ð+[Ñ)Ô)Ð)ð $ qÒ(Ð(°zÐDXÑ7X¨*Ø! qÒ(Ð(°zÐDVÑ7V¨*Ø qÒ(Ð(°zÅEÄJÈzÐ\mÑOmÑDnÔDnÑ7n¨*à#Ÿ š ¥u¤}Ñ5Ô5ˆJà˜cÒ!Ð!°
|
|||
|
|
¸[Ñ0H :Ý”L °2¸yÐIÑIÔI×QÒQÐRTÑUÔUˆEØ�%œ/¨*¸BÐ?Ñ?Ô?Ñ?ˆCØ à Ð !à# qÒ(Ð(°zÐDXÑ7X¨*Ø! qÒ(Ð(°zÐDVÑ7V¨*Ø qÒ(Ð(°zÅEÄJÈzÐ\mÑOmÑDnÔDnÑ7n¨*à#Ÿ š ¥u¤}Ñ5Ô5ˆJà˜cÒ!Ð!°
|
|||
|
|
¸[Ñ0H :Ý”L °2¸yÐIÑIÔI×QÒQÐRTÑUÔUˆEØ�%œ/¨*¸BÐ?Ñ?Ô?Ñ?ˆCØ ð7
|
|||
|
|
ð
|
|||
|
|
ð
|
|||
|
|
ñ
|
|||
|
|
ô
|
|||
|
|
ð
|
|||
|
|
ð
|
|||
|
|
ð
|
|||
|
|
ð
|
|||
|
|
ð
|
|||
|
|
ð
|
|||
|
|
øøøð
|
|||
|
|
ð
|
|||
|
|
ð
|
|||
|
|
ð
|
|||
|
|
ð8 ð ˆs‚{€{ÝŒy˜˜s™Ñ#Ô# s¨S¡yÑ1°CÑ7ˆˆð ˆð Ð Ý”˜3 ™9Ñ%Ô%ˆˆå”˜3 §¢¡ ¤ Ñ-Ñ.Ô.ˆÝ
|
|||
|
|
Œ[˜ [¡ °!°lÑ2BÑ
|
|||
|
|
CÔ
|
|||
|
|
C€Fà ÐÝ”˜V¨Ð/Ñ/Ô/°*×2FÒ2FÀqÑ2IÔ2IÑIˆˆà˜*×.Ò.¨qÑ1Ô1Ñ1ˆØð �j×*Ò*¨1Ñ-Ô-Ñ
|
|||
|
|
-€FÝŒi˜ Ñ'Ô'Ð
|
|||
|
|
'€FØ ˆs‚{€{ؘ$ ™+Ñ%ˆà �7Š7•5”=Ñ !Ô !€DØ Ÿš ™ œ Ðð �FÒÐؘ$‘ ×#Ò# BÑ'Ô'¨$¯(ª(°2©,¬,×*<Ò*<ÀÐ*<Ñ*EÔ*EÑE×KÒKÑMÔMˆˆØ �fÒ Ð Ø˜‘
|
|||
|
|
×"Ò"Ñ$Ô$ t§x¢x¡z¤z×'7Ò'7¸CÐ'7Ñ'@Ô'@Ñ@ˆˆØ �iÒ Ð Ø˜‘
|
|||
|
|
×"Ò"Ñ$Ô$¨¯ ª °A©¬Ð9NÑ(NÑOˆˆåÐ:¨yÐ:Ð:Ñ;Ô;Ð;õ
|
|||
|
|
|
|||
|
|
Ô Ñ Ô ð ,ð ,Ø -× 2Ò 2Ñ 4Ô 4ÐØ " T™k× .Ò .¨qÑ 1Ô 1Ð4EÑ EÐØ$×)Ò)Ñ+Ô+ˆð,ð ,ð ,ñ ,ô ,ð ,ð ,ð ,ð ,ð ,ð ,øøøð ,ð ,ð ,ð ,ð à Ð" GÐ +Ð+r c ó0 — ddl m} |di | ¤Ž}| |_ |S )Nr )ÚSamplingParamsr )Úvllmr7 Ú _set_kwargs)r¯ r7 Úsampling_paramss r‚ ÚvLLMSamplingParamsr; s7 € Ø#Ð#Ð#Ð#Ð#Ð#Ø$�nÐ.Ð. vÐ.Ð.€OØ"(€OÔ Ø Ðr„ c ¯ óì ‡ — e Zd ZU dZ edddi¬¦ « Zee ed< edddi¬¦ « Z ee
|
|||
|
|
ed < ddd
|
|||
|
|
d
|
|||
|
|
d
|
|||
|
|
d d
|
|||
|
|
d d ddd
|
|||
|
|
d
|
|||
|
|
dddddddddddddddddd d
|
|||
|
|
d d
|
|||
|
|
d d ddd
|
|||
|
|
d
|
|||
|
|
d
|
|||
|
|
d
|
|||
|
|
d
|
|||
|
|
d
|
|||
|
|
d d d
|
|||
|
|
d
|
|||
|
|
d
|
|||
|
|
d
|
|||
|
|
d d!d
|
|||
|
|
d
|
|||
|
|
ddddd
|
|||
|
|
d"d
|
|||
|
|
ddddddd
|
|||
|
|
dd
|
|||
|
|
ddd
|
|||
|
|
d"dddddd#d$dd
|
|||
|
|
d
|
|||
|
|
d%dddddd
|
|||
|
|
dd
|
|||
|
|
d
|
|||
|
|
ddd&ddd
|
|||
|
|
dd
|
|||
|
|
dd
|
|||
|
|
dd!dddd"dd
|
|||
|
|
dd'd(d
|
|||
|
|
ddd
|
|||
|
|
d
|
|||
|
|
ddd
|
|||
|
|
d
|
|||
|
|
d
|
|||
|
|
dd
|
|||
|
|
ddd
|
|||
|
|
d)d*d+ddddddddi dd
|
|||
|
|
dd
|
|||
|
|
dd,d-dd.d/d0d1d d2d d3ddd4ddd5d
|
|||
|
|
d
|
|||
|
|
d6d)dd
|
|||
|
|
d
|
|||
|
|
dd
|
|||
|
|
ddf¯ˆ fd7„ Z ˆ xZ S )8ÚUnslothGRPOConfiguâ<
|
|||
|
|
|
|||
|
|
Configuration class for the [`GRPOTrainer`].
|
|||
|
|
|
|||
|
|
This class includes only the parameters that are specific to GRPO training. For a full list of training arguments,
|
|||
|
|
please refer to the [`~transformers.TrainingArguments`] documentation. Note that default values in this class may
|
|||
|
|
differ from those in [`~transformers.TrainingArguments`].
|
|||
|
|
|
|||
|
|
Using [`~transformers.HfArgumentParser`] we can turn this class into
|
|||
|
|
[argparse](https://docs.python.org/3/library/argparse#module-argparse) arguments that can be specified on the
|
|||
|
|
command line.
|
|||
|
|
|
|||
|
|
Parameters:
|
|||
|
|
> Parameters that control the model and reference model
|
|||
|
|
|
|||
|
|
model_init_kwargs (`str`, `dict[str, Any]` or `None`, *optional*, defaults to `None`):
|
|||
|
|
Keyword arguments for [`~transformers.AutoModelForCausalLM.from_pretrained`], used when the `model`
|
|||
|
|
argument of the [`GRPOTrainer`] is provided as a string.
|
|||
|
|
disable_dropout (`bool`, *optional*, defaults to `False`):
|
|||
|
|
Whether to disable dropout in the model. This is useful for training with a reference model, as it prevents
|
|||
|
|
the model from generating different logprobs for the same input.
|
|||
|
|
|
|||
|
|
> Parameters that control the data preprocessing
|
|||
|
|
|
|||
|
|
remove_unused_columns (`bool`, *optional*, defaults to `False`):
|
|||
|
|
Whether to only keep the column `"prompt"` in the dataset. If you use a custom reward function that
|
|||
|
|
requires any column other than `"prompts"` and `"completions"`, you should keep this to `False`.
|
|||
|
|
max_prompt_length (`int` or `None`, *optional*, defaults to `512`):
|
|||
|
|
Maximum length of the prompt. If the prompt is longer than this value, it will be truncated left.
|
|||
|
|
num_generations (`int` or `None`, *optional*, defaults to `8`):
|
|||
|
|
Number of generations per prompt to sample. The effective batch size (num_processes * per_device_batch_size
|
|||
|
|
* gradient_accumulation_steps) must be evenly divisible by this value.
|
|||
|
|
max_completion_length (`int` or `None`, *optional*, defaults to `256`):
|
|||
|
|
Maximum length of the generated completion.
|
|||
|
|
ds3_gather_for_generation (`bool`, *optional*, defaults to `True`):
|
|||
|
|
This setting applies to DeepSpeed ZeRO-3. If enabled, the policy model weights are gathered for generation,
|
|||
|
|
improving generation speed. However, disabling this option allows training models that exceed the VRAM
|
|||
|
|
capacity of a single GPU, albeit at the cost of slower generation. Disabling this option is not compatible
|
|||
|
|
with vLLM generation.
|
|||
|
|
shuffle_dataset (`bool`, *optional*, defaults to `True`):
|
|||
|
|
Whether to shuffle the training dataset.
|
|||
|
|
|
|||
|
|
> Parameters that control generation
|
|||
|
|
|
|||
|
|
generation_batch_size: (`int` or `None`, *optional*, defaults to `None`):
|
|||
|
|
Batch size to use for generation. If `None`, it defaults to the effective training batch size:
|
|||
|
|
`per_device_train_batch_size * num_processes * steps_per_generation`. In other words, there is one
|
|||
|
|
generation batch processed per optimization step. Mutually exclusive with `steps_per_generation`.
|
|||
|
|
steps_per_generation: (`int` or `None`, *optional*, defaults to `None`):
|
|||
|
|
Number of steps per generation. If `None`, it defaults to `gradient_accumulation_steps`. Mutually exclusive
|
|||
|
|
with `generation_batch_size`.
|
|||
|
|
temperature (`float`, defaults to `1.0`):
|
|||
|
|
Temperature for sampling. The higher the temperature, the more random the completions.
|
|||
|
|
top_p (`float`, *optional*, defaults to `1.0`):
|
|||
|
|
Float that controls the cumulative probability of the top tokens to consider. Must be in (0, 1]. Set to
|
|||
|
|
`1.0` to consider all tokens.
|
|||
|
|
top_k (`int` or `None`, *optional*, defaults to `None`):
|
|||
|
|
Number of highest probability vocabulary tokens to keep for top-k-filtering. If `None`, top-k-filtering is
|
|||
|
|
disabled and all tokens are considered.
|
|||
|
|
min_p (`float` or `None`, *optional*, defaults to `None`):
|
|||
|
|
Minimum token probability, which will be scaled by the probability of the most likely token. It must be a
|
|||
|
|
value between `0.0` and `1.0`. Typical values are in the `0.01-0.2` range.
|
|||
|
|
repetition_penalty (`float`, *optional*, defaults to `1.0`):
|
|||
|
|
Float that penalizes new tokens based on whether they appear in the prompt and the generated text so far.
|
|||
|
|
Values > `1.0` encourage the model to use new tokens, while values < `1.0` encourage the model to repeat
|
|||
|
|
tokens.
|
|||
|
|
use_transformers_paged (`bool`, *optional*, defaults to `False`):
|
|||
|
|
Whether to use the `transformers` paged implementation for generation. If set to `True`, the `transformers`
|
|||
|
|
paged implementation will be used for generation instead of the default padded implementation. This
|
|||
|
|
parameter is only effective when `use_vllm` is set to `False`.
|
|||
|
|
cache_implementation (`str` or `None`, *optional*, defaults to `None`):
|
|||
|
|
Implementation of the cache method for faster generation when `use_vllm` is set to `False`.
|
|||
|
|
generation_kwargs (`dict[str, Any]` or `None`, *optional*, defaults to `None`):
|
|||
|
|
Additional keyword arguments to pass to `GenerationConfig` (if using transformers) or `SamplingParams` (if
|
|||
|
|
using vLLM) when sampling completions. This can be used to further customize the generation behavior, such
|
|||
|
|
as setting `supress_tokens`, `num_beams`, etc. If it contains keys that conflict with the other generation
|
|||
|
|
parameters (like `min_p`, `top_p`, etc.), they will override them.
|
|||
|
|
|
|||
|
|
> Parameters that control generation acceleration powered by vLLM
|
|||
|
|
|
|||
|
|
use_vllm (`bool`, *optional*, defaults to `False`):
|
|||
|
|
Whether to use vLLM for generating completions. If set to `True`, the trainer will use vLLM for generation
|
|||
|
|
instead of the default model.generate(). Requires `vllm` to be installed.
|
|||
|
|
vllm_mode (`str`, *optional*, defaults to `"server"`):
|
|||
|
|
Mode to use for vLLM integration when `use_vllm` is set to `True`. Must be one of `"server"` or
|
|||
|
|
`"colocate"`.
|
|||
|
|
|
|||
|
|
- `"server"`: The trainer will send generation requests to a separate vLLM server. Make sure a TRL vLLM
|
|||
|
|
server is running (start with `trl vllm-serve`).
|
|||
|
|
- `"colocate"`: vLLM will run in the same process and share the training GPUs. This avoids the need for a
|
|||
|
|
separate server but may cause resource contention with training.
|
|||
|
|
vllm_guided_decoding_regex (`str` or `None`, *optional*, defaults to `None`):
|
|||
|
|
Regex for vLLM guided decoding. If `None` (default), guided decoding is disabled.
|
|||
|
|
|
|||
|
|
> Parameters that control the vLLM server (only used when `vllm_mode` is `"server"`)
|
|||
|
|
|
|||
|
|
vllm_server_base_url (`str` or `None`, *optional*, defaults to `None`):
|
|||
|
|
Base URL for the vLLM server (e.g., `"http://localhost:8000"`). If provided, `vllm_server_host` and
|
|||
|
|
`vllm_server_port` are ignored.
|
|||
|
|
vllm_server_host (`str`, *optional*, defaults to `"0.0.0.0"`):
|
|||
|
|
Host of the vLLM server to connect to. Ignored if `vllm_server_base_url` is provided.
|
|||
|
|
vllm_server_port (`int`, *optional*, defaults to `8000`):
|
|||
|
|
Port of the vLLM server to connect to. Ignored if `vllm_server_base_url` is provided.
|
|||
|
|
vllm_server_timeout (`float`, *optional*, defaults to `240.0`):
|
|||
|
|
Total timeout duration in seconds to wait for the vLLM server to be up. If the server is not up after the
|
|||
|
|
timeout, a `ConnectionError` is raised.
|
|||
|
|
|
|||
|
|
> Parameters that control colocated vLLM execution (only used when `vllm_mode` is `"colocate"`)
|
|||
|
|
|
|||
|
|
vllm_gpu_memory_utilization (`float`, *optional*, defaults to `0.3`):
|
|||
|
|
Control the GPU memory utilization for vLLM. This setting only applies when `vllm_mode` is set to
|
|||
|
|
`"colocate"`. If you are using `vllm_mode="server"`, this parameter must be passed separately when
|
|||
|
|
launching the vLLM server via the `--vllm_gpu_memory_utilization` flag.
|
|||
|
|
vllm_tensor_parallel_size (`int`, *optional*, defaults to `1`):
|
|||
|
|
Control the tensor parallel size for vLLM. This setting only applies when `vllm_mode` is set to
|
|||
|
|
`"colocate"`. If you are using `vllm_mode="server"`, this parameter must be passed separately when
|
|||
|
|
launching the vLLM server via the `--vllm_tensor_parallel_size` flag.
|
|||
|
|
vllm_model_impl (`str`, *optional*, defaults to `"vllm"`):
|
|||
|
|
Model implementation to use for vLLM. Must be one of `"transformers"` or `"vllm"`. `"transformers"`: Use
|
|||
|
|
the `transformers` backend for model implementation. `"vllm"`: Use the `vllm` library for model
|
|||
|
|
implementation.
|
|||
|
|
|
|||
|
|
> Parameters that control the training
|
|||
|
|
|
|||
|
|
beta (`float`, *optional*, defaults to `0.0`):
|
|||
|
|
KL coefficient. If `0.0` (default), the reference model is not loaded, reducing memory usage and improving
|
|||
|
|
training speed.
|
|||
|
|
num_iterations (`int`, *optional*, defaults to `1`):
|
|||
|
|
Number of iterations per batch (denoted as μ in the algorithm).
|
|||
|
|
epsilon (`float`, *optional*, defaults to `0.2`):
|
|||
|
|
Epsilon value for clipping.
|
|||
|
|
delta: (`float` or `None`, *optional*, defaults to `None`):
|
|||
|
|
Enables the upper clipping bound in two-sided GRPO loss when set to a float. If `None` (default), standard
|
|||
|
|
GRPO clipping is used. Recommended to be greater than `1 + ε` when enabled. This method is introduced in
|
|||
|
|
the [INTELLECT-2 tech report](https://huggingface.co/papers/2505.07291).
|
|||
|
|
epsilon_high (`float` or `None`, *optional*, defaults to `None`):
|
|||
|
|
Upper-bound epsilon value for clipping. If not specified, it defaults to the same value as the lower-bound
|
|||
|
|
specified in argument `epsilon`. Paper [DAPO](https://huggingface.co/papers/2503.14476) recommends `0.28`.
|
|||
|
|
importance_sampling_level (`str`, *optional*, defaults to `"token"`):
|
|||
|
|
Controls whether importance sampling ratios are computed at the `"token"` or `"sequence"` level. `"token"`
|
|||
|
|
keeps the raw per-token log-probability ratios (one weight per token). `"sequence"` averages the
|
|||
|
|
log-probability ratios across valid tokens to produce a single ratio per sequence. The
|
|||
|
|
[GSPO paper](https://huggingface.co/papers/2507.18071) shows that sequence-level sampling often yields more
|
|||
|
|
stable training and better alignment with sequence-level rewards.
|
|||
|
|
reward_weights (`list[float]` or `None`, *optional*, defaults to `None`):
|
|||
|
|
Weights for each reward function. Must match the number of reward functions. If `None`, all rewards are
|
|||
|
|
weighted equally with weight `1.0`.
|
|||
|
|
scale_rewards (`bool`, *optional*, defaults to `True`):
|
|||
|
|
Whether to scale the rewards by dividing them by their standard deviation. If `True` (default), the rewards
|
|||
|
|
are normalized by the standard deviation, ensuring they have unit variance. If `False`, no scaling is
|
|||
|
|
applied. The [Dr. GRPO paper](https://huggingface.co/papers/2503.20783) recommends not scaling the rewards,
|
|||
|
|
as scaling by the standard deviation introduces a question-level difficulty bias.
|
|||
|
|
loss_type (`str`, *optional*, defaults to `"bnpo"`):
|
|||
|
|
Specifies the loss formulation to use. Supported values are:
|
|||
|
|
|
|||
|
|
- `"grpo"`: Aggregates token-level losses by normalizing over sequence length. Not recommended due to
|
|||
|
|
length bias—this approach tends to prefer shorter completions with positive advantages and longer ones
|
|||
|
|
with negative advantages.
|
|||
|
|
- `"bnpo"`: Aggregates token-level losses by normalizing number of active token in the local batch.
|
|||
|
|
Note that normalization is performed over the local batch only, so results may slightly vary depending
|
|||
|
|
on the local batch size, despite a constant effective batch size. When using
|
|||
|
|
`per_device_train_batch_size==1`, the loss is equivalent to the GRPO loss.
|
|||
|
|
- `"dr_grpo"`: Aggregates token-level losses by normalizing with a global constant. This method was
|
|||
|
|
introduced in the [Dr. GRPO paper](https://huggingface.co/papers/2503.20783) to eliminate length bias.
|
|||
|
|
The value of the constant corresponds to `max_completion_length`.
|
|||
|
|
mask_truncated_completions (`bool`, *optional*, defaults to `False`):
|
|||
|
|
When enabled, truncated completions are excluded from the loss calculation, preventing them from being
|
|||
|
|
incorrectly penalized and introducing noise during training. According to the
|
|||
|
|
[DAPO](https://huggingface.co/papers/2503.14476) paper, this is a good practice for training stability.
|
|||
|
|
sync_ref_model (`bool`, *optional*, defaults to `False`):
|
|||
|
|
Whether to synchronize the reference model with the active model every `ref_model_sync_steps` steps, using
|
|||
|
|
the `ref_model_mixup_alpha` parameter. This synchronization originates from the
|
|||
|
|
[TR-DPO](https://huggingface.co/papers/2404.09656) paper.
|
|||
|
|
ref_model_mixup_alpha (`float`, *optional*, defaults to `0.6`):
|
|||
|
|
α parameter from the [TR-DPO](https://huggingface.co/papers/2404.09656) paper, which controls the mix
|
|||
|
|
between the current policy and the previous reference policy during updates. The reference policy is
|
|||
|
|
updated according to the equation: `π_ref = α * π_θ + (1 - α) * π_ref_prev`. To use this parameter, you
|
|||
|
|
must set `sync_ref_model=True`.
|
|||
|
|
ref_model_sync_steps (`int`, *optional*, defaults to `512`):
|
|||
|
|
Ï„ parameter from the [TR-DPO](https://huggingface.co/papers/2404.09656) paper, which determines how
|
|||
|
|
frequently the current policy is synchronized with the reference policy. To use this parameter, you must
|
|||
|
|
set `sync_ref_model=True`.
|
|||
|
|
top_entropy_quantile (`float`, *optional*, defaults to `1.0`):
|
|||
|
|
� parameter from [Beyond the 80/20 Rule](https://huggingface.co/papers/2506.01939). Keeps in the policy
|
|||
|
|
loss term only the top-� quantile of tokens by entropy of the probability distribution at each sequence
|
|||
|
|
position, improving results. Range: `[0.0-1.0]`. A value of `0.0` masks all but the highest entropy token;
|
|||
|
|
`1.0` keeps all tokens. The paper recommends a value of `0.2`.
|
|||
|
|
If used with `mask_truncated_completions=True`, only tokens from non-truncated completions are considered.
|
|||
|
|
use_liger_loss (`bool`, *optional*, defaults to `False`):
|
|||
|
|
Whether to use the Liger GRPO loss.
|
|||
|
|
|
|||
|
|
> Parameters that control the logging
|
|||
|
|
|
|||
|
|
log_completions (`bool`, *optional*, defaults to `False`):
|
|||
|
|
Whether to log a sample of (prompt, completion) pairs every `logging_steps` steps. If `rich` is installed,
|
|||
|
|
it prints the sample. If `wandb` logging is enabled, it logs it to `wandb`.
|
|||
|
|
num_completions_to_print (`int` or `None`, *optional*, defaults to `None`):
|
|||
|
|
Number of completions to print with `rich`. If `None`, all completions are logged.
|
|||
|
|
wandb_log_unique_prompts (`bool`, *optional*, defaults to `False`):
|
|||
|
|
Whether to log unique prompts in wandb. If `True`, only unique prompts are logged. If `False`, all prompts
|
|||
|
|
are logged.
|
|||
|
|
|
|||
|
|
NÚhelpzvLLM SamplingParams)ÚdefaultÚmetadataÚvllm_sampling_paramsre z8Chunk size to reduce memory usage. -1 is most efficient.Úunsloth_num_chunksFÚnorf é r éú g-C ëâ6
|
|||
|
|
?g{®Gáz„?gÍÌÌÌÌÌì?g+‡ÙÎ÷ï?g:Œ0âŽyE>r� g @Úlinearçš™™™™™¹?ÚpassiveÚwarningTÚstepsrm éô iO
|
|||
|
|
ÚO1ÚautoÚ r‘ Ú
|
|||
|
|
adamw_8bitÚlengthÚ
|
|||
|
|
every_saveÚlasti é é é Úcolocater8 z0.0.0.0i@ g n@g333333Ó?gü©ñÒMbP?rŠ Útokenr˜ g333333ã?c° ó6 •— |dk rt d|› d�¦ « ‚|dk rt d|› d�¦ « ‚|€|#dk r
|
|||
|
|
|$dk rd}d }#|¤ ¦ « d
|
|||
|
|
k rd
|
|||
|
|
}¤n|¤ ¦ « d k rd }¤|¤ ¦ « d
|
|||
|
|
k r!|£d k rd }£ne|£d k rt d
|
|||
|
|
¦ « d}£nM|¤ ¦ « d k r5t d¦ « t d¦ « t d¦ « d }¥d} d}œd}¤||„z |„z |k r4t dt |¦ « z dz t |„¦ « z ¦ « |„}|Šdk rt d¦ « ‚|Šdk rt d¦ « ‚ t
|
|||
|
|
¦ « j dÈi d|“d |“d |“d |“d |“d |“d!|“d"|“d#| “d$|
|
|||
|
|
“d%| “d&| “d'|
|
|||
|
|
“d(|“d)|“d*|“d+|“d,|“d-|“d.|“d/|“d0|“d1|“d2|“d3|“d4|“d5|“d6| “d7| “d8| “d9| “d:| “d;|!“d<|"“d=|#“d>|$“d?|%“d@|&“dA|'“dB|(“dC|)“dD|*“dE|+“dF|,“dG|-“dH|.“dI|/“dJ|0“dK|1“dL|2“dM|3“dN|4“dO|5“dP|6“dQ|7“dR|8“dS|9“dT|:“dU|;“dV|<“dW|=“dX|>“dY|?“dZ|@“d[|A“d\|B“d]|C“d^|D“d_|E“d`|F“da|G“db|H“dc|I“dd|J“de|K“df|L“dg|M“dh|N“di|O“dj|P“dk|Q“dl|R“dm|S“dn|T“do|U“dp|V“dq|W“dr|X“ds|Y“dt|Z“du|[“dv|\“dw|]“dx|^“dy|_“dz|`“d{|a“d||b“d}|c“d~|d“d|e“d€|f“d�|g“d‚|h“dƒ|i“d„|j“d…|k“d†|l“d‡|m“dˆ|n“d‰|o“dŠ|p“d‹|q“dŒ|r“d�|s“dŽ|t“d�|u“d�|v“d‘|w“d’|x“d“|y“d”|z“d•|{“d–||“d—|}“d˜|~“d™|“dš|€“d›|�“dœ|‚“d�|ƒ“dž|„“dŸ|…“d |†“d¡|‡“d¢|ˆ“d£|‰“d¤|Š“d¥|‹“d¦|Œ“d§|�“d¨|Ž“d©|�“dª|�“d«|‘“d¬|’“d|““d®|”“d¯|•“d°|–“d±|—“d²|˜“d³|™“d´|š“dµ|›“d¶|œ“d·|�“d¸|ž“d¹|Ÿ“dº| “d»|¡“d¼|¢“d½|£“d¾|¤“d¿|¥“dÀ|¦“dÁ|§“dÂ|¨“dÃ|©“dÄ|ª“dÅ|«“dÆ|¬“dÇ|“|°¤Ž |®| _ |¯| _ d S )ÉNgH¯¼šò×z>z Unsloth: Your learning rate of `zi` is too small and less than 1e-7! Consider increasing it, otherwise gradient updates will be close to 0!rm za` is way too larger > 1! Consider decreasing it to 1e-1, otherwise gradient updates will explode!rJ rK Ú unsloth_training_checkpointsrC r™ ÚdapoTzwUnsloth: The Dr GRPO paper recommends setting `scale_rewards` to False! Will override. Set it to `None` to force False.FzFUnsloth: The DAPO paper recommends `mask_truncated_completions = True`z8Unsloth: The DAPO paper recommends `epsilon_high = 0.28`zMUnsloth: The DAPO paper recommends setting `beta = 0.0` to remove the KL termgìQ¸ …ëÑ?r‘ r˜ z}Unsloth: We now expect `per_device_train_batch_size` to be a multiple of `num_generations`.
|
|||
|
|
We will change the batch size of z to the `num_generations` of r zUUnsloth: Please set a positive non-zero temperature since your results will be wrong.é
|
|||
|
|
zgUnsloth: Please set a positive non-zero temperature less than 10, since sampling will be quite erratic.Ú
|
|||
|
|
output_dirÚoverwrite_output_dirÚdo_trainÚdo_evalÚ
|
|||
|
|
do_predictÚ
|
|||
|
|
eval_strategyÚprediction_loss_onlyÚper_device_train_batch_sizeÚper_device_eval_batch_sizeÚper_gpu_train_batch_sizeÚper_gpu_eval_batch_sizeÚgradient_accumulation_stepsÚeval_accumulation_stepsÚ
|
|||
|
|
eval_delayÚtorch_empty_cache_stepsÚ
|
|||
|
|
learning_rateÚ weight_decayÚ
|
|||
|
|
adam_beta1Ú
|
|||
|
|
adam_beta2Ú adam_epsilonÚ
|
|||
|
|
max_grad_normÚnum_train_epochsÚ max_stepsÚlr_scheduler_typeÚ warmup_ratioÚ warmup_stepsÚ log_levelÚlog_level_replicaÚlog_on_each_nodeÚ logging_dirÚlogging_strategyÚlogging_first_stepÚ
|
|||
|
|
logging_stepsÚlogging_nan_inf_filterÚ
|
|||
|
|
save_strategyÚ
|
|||
|
|
save_stepsÚsave_total_limitÚsave_safetensorsÚsave_on_each_nodeÚsave_only_modelÚ'restore_callback_states_from_checkpointÚno_cudaÚuse_cpuÚuse_mps_deviceÚseedÚ data_seedÚ
|
|||
|
|
jit_mode_evalÚuse_ipexÚbf16r Úfp16_opt_levelÚhalf_precision_backendÚbf16_full_evalÚfp16_full_evalÚtf32Ú
|
|||
|
|
local_rankÚ ddp_backendÚ
|
|||
|
|
tpu_num_coresÚtpu_metrics_debugÚdebugÚdataloader_drop_lastÚ
|
|||
|
|
eval_stepsÚdataloader_num_workersÚdataloader_prefetch_factorÚ
|
|||
|
|
past_indexÚrun_nameÚ disable_tqdmÚremove_unused_columnsÚ label_namesÚload_best_model_at_endÚmetric_for_best_modelÚgreater_is_betterÚignore_data_skipÚfsdpÚfsdp_min_num_paramsÚ fsdp_configÚ"fsdp_transformer_layer_cls_to_wrapÚaccelerator_configÚ deepspeedÚlabel_smoothing_factorÚoptimÚ
|
|||
|
|
optim_argsÚ adafactorÚgroup_by_lengthÚlength_column_nameÚ report_toÚddp_find_unused_parametersÚddp_bucket_cap_mbÚddp_broadcast_buffersÚdataloader_pin_memoryÚ dataloader_persistent_workersÚskip_memory_metricsÚuse_legacy_prediction_loopÚ push_to_hubÚresume_from_checkpointÚ hub_model_idÚ hub_strategyÚ hub_tokenÚhub_private_repoÚhub_always_pushÚ hub_revisionÚgradient_checkpointingÚ gradient_checkpointing_kwargsÚinclude_inputs_for_metricsÚeval_do_concat_batchesÚ fp16_backendÚpush_to_hub_model_idÚpush_to_hub_organizationÚpush_to_hub_tokenÚ
|
|||
|
|
mp_parametersÚauto_find_batch_sizeÚfull_determinismÚ torchdynamoÚ ray_scopeÚ ddp_timeoutÚ
|
|||
|
|
torch_compileÚtorch_compile_backendÚtorch_compile_modeÚinclude_tokens_per_secondÚ include_num_input_tokens_seenÚneftune_noise_alphaÚoptim_target_modulesÚbatch_eval_metricsÚ
|
|||
|
|
eval_on_startÚuse_liger_kernelÚliger_kernel_configÚeval_use_gather_objectÚ average_tokens_across_devicesÚmodel_init_kwargsÚdisable_dropoutÚmax_prompt_lengthÚnum_generationsrŒ Úds3_gather_for_generationÚshuffle_datasetÚgeneration_batch_sizeÚsteps_per_generationrŽ Útop_pÚtop_kÚmin_pÚgeneration_kwargsÚrepetition_penaltyÚuse_transformers_pagedÚcache_implementationÚuse_vllmÚvllm_server_base_urlÚ vllm_modeÚvllm_model_implÚvllm_guided_decoding_regexÚvllm_server_hostÚvllm_server_portÚvllm_server_timeoutÚvllm_gpu_memory_utilizationÚvllm_tensor_parallel_sizer Únum_iterationsÚepsilonr� r‹ Úimportance_sampling_levelÚreward_weightsÚ
|
|||
|
|
scale_rewardsr‡ Úmask_truncated_completionsÚsync_ref_modelÚref_model_mixup_alphaÚref_model_sync_stepsÚtop_entropy_quantileÚuse_liger_lossÚlog_completionsÚnum_completions_to_printÚwandb_log_unique_promptsr )
|
|||
|
|
ÚFloatingPointErrorÚ
|
|||
|
|
OverflowErrorÚlowerÚprintÚstrÚ MathErrorÚsuperÚ__init__rA rB )²Úselfr\ r] r^ r_ r` ra rb rc rd re rf rg rh ri rj rk rl rm rn ro rp rq rr rs rt ru rv rw rx ry rz r{ r| r} r~ r r€ r� r‚ rƒ r„ r… r† r‡ rˆ r‰ rŠ r‹ rŒ r r� rŽ r� r� r‘ r’ r“ r” r• r– r— r˜ r™ rš r› rœ r� rž rŸ r r¡ r¢ r£ r¤ r¥ r¦ r§ r¨ r© rª r« r¬ r r® r¯ r° r± r² r³ r´ rµ r¶ r· r¸ r¹ rº r» r¼ r½ r¾ r¿ rÀ rÁ r rà rÄ rÅ rÆ rÇ rÈ rÉ rÊ rË rÌ rÍ rÎ rÏ rÐ rÑ rÒ rÓ rÔ rÕ rÖ r× rØ rÙ rÚ rÛ rÜ rÝ rÞ rŒ rß rà rá râ rŽ rã rä rå ræ rç rè ré rê rë rì rí rî rï rð rñ rò ró r rô rõ r� r‹ rö r÷ rø r‡ rù rú rû rü rý rþ rÿ r r rA rB r¯ Ú __class__s² €r‚ r zUnslothGRPOConfig.__init__Ú s‘
|
|||
|
|
ø€ ðf ˜4Ò Ð Õ'9ð ;VÐ]jð ;Vð ;Vð ;Vñ (Wô (Wð "WØ ˜1Ò Ð ¥Mð 3FÐUbð 3Fð 3Fð 3Fñ %Gô %Gð GØ Ð -°7Ò":Ð":¸zÈSÒ?PÐ?PØ7ˆJØ ˆMØ �?Š?Ñ Ô Ò )Ð )Ø!ˆIˆIØ
|
|||
|
|
�_Š_Ñ
|
|||
|
|
Ô
|
|||
|
|
&Ò
|
|||
|
|
(Ð
|
|||
|
|
(Ø ˆIØ �?Š?Ñ Ô Ò )Ð )Ø Ò$Ð$Ø $�
|
|||
|
|
�
|
|||
|
|
Ø $Ò&Ð&Ýð Pñ Qô Qð QØ %�
|
|||
|
|
øØ
|
|||
|
|
�_Š_Ñ
|
|||
|
|
Ô
|
|||
|
|
&Ò
|
|||
|
|
(Ð
|
|||
|
|
(Ý ÐZÑ [Ô [Ð [Ý ÐLÑ MÔ MÐ MÝ ÐaÑ bÔ bÐ bØ)-Ð &Ø ˆL؈DØ ˆIà '¨?Ñ :¸oÑ MÐQlÒ lÐ lÝ ð Sõ VYð Zuñ Vvô Vvñ vð yXñ Xõ [^ð _nñ [oô [oñ oñ
|
|||
|
|
pô
|
|||
|
|
pð
|
|||
|
|
pØ*9Ð 'à ˜!Ò Ð ÝÐ sÑtÔtÐ tØ
|
|||
|
|
˜BÒ
|
|||
|
|
Ð
|
|||
|
|
Ýð Fñ Gô Gð
|
|||
|
|
Gð �‰ŒÔð m Jð m Jð m JØ#˜ðm Jà#7Ð#7ðm Jð �xðm Jð �gð m Jð
|
|||
|
|
$˜ð m Jð *˜Mð
|
|||
|
|
m Jð $8Ð#7ðm Jð +FÐ*Eðm Jð *DÐ)Cðm Jð (@Ð'?ðm Jð '>Ð&=ðm Jð +FÐ*Eðm Jð '>Ð&=ðm Jð $˜ð m Jð '>Ð&=ð m Jð *˜Mð!m Jð" (˜<ð#m Jð$ $˜ð%m Jð& $˜ð'm Jð( (˜<ð)m Jð* *˜Mð+m Jð, 0Ð /ð-m Jð. "˜ ð/m Jð0 !2Ð 1ð1m Jð2 (˜<ð3m Jð4 (˜<ð5m Jð6 "˜ ð7m Jð8 !2Ð 1ð9m Jð: 0Ð /ð;m Jð< &˜+ð=m Jð> 0Ð /ð?m Jð@ "4Ð!3ðAm JðB *˜MðCm JðD &<Ð%;ðEm JðF *˜MðGm JðH $˜ðIm JðJ 0Ð /ðKm JðL 0Ð /ðMm JðN !2Ð 1ðOm JðP .˜oðQm JðR 7^Ð6]ðSm JðT �gðUm JðV �gðWm JðX ,˜^ðYm JðZ �4ð[m Jð\ "˜ ð]m Jð^ *˜Mð_m Jð` �xðam Jðb �4ðcm Jðd �4ðem Jðf ,˜^ðgm Jðh &<Ð%;ðim Jðj ,˜^ðkm Jðl ,˜^ðmm Jðn �4ðom Jðp $˜ðqm Jðr &˜+ðsm Jðt *˜Mðum Jðv !2Ð 1ðwm Jðx �Eðym Jðz $8Ð#7ð{m Jð| $˜ð}m Jð~ &<Ð%;ðm Jð@ *DÐ)CðAm JðB $˜ðCm JðD �xðEm JðF (˜<ðGm JðH %:Ð$9ðIm JðJ &˜+ðKm JðL &<Ð%;ðMm JðN %:Ð$9ðOm JðP !2Ð 1ðQm JðR 0Ð /ðSm JðT �4ðUm JðV #6Ð"5ðWm JðX &˜+ðYm JðZ 2TÐ1Sð[m Jð\ "4Ð!3ð]m Jð^ "˜ ð_m Jð` &<Ð%;ðam Jðb �Eðcm Jðd $˜ðem Jðf "˜ ðgm Jðh .˜oðim Jðj "4Ð!3ðkm Jðl "˜ ðmm Jðn *DÐ)Cðom Jðp !2Ð 1ðqm Jðr %:Ð$9ðsm Jðt %:Ð$9ðum Jðv -JÐ,Iðwm Jðx #6Ð"5ðym Jðz *DÐ)Cð{m Jð| &˜+ð}m Jð~ &<Ð%;ðm Jð@ (˜<ðAm JðB (˜<ðCm JðD "˜ ðEm JðF 0Ð /ðGm JðH .˜oðIm JðJ (˜<ðKm JðL &<Ð%;ðMm JðN -JÐ,IðOm JðP *DÐ)CðQm JðR &<Ð%;ðSm JðT (˜<ðUm JðV $8Ð#7ðWm JðX (@Ð'?ðYm JðZ !2Ð 1ð[m Jð\ *˜Mð]m Jð^ $8Ð#7ð_m Jð` 0Ð /ðam Jðb &˜+ðcm Jðd "˜ ðem Jðf &˜+ðgm Jðh *˜Mðim Jðj %:Ð$9ðkm Jðl "4Ð!3ðmm Jðn )BÐ(Aðom Jðp -JÐ,Iðqm Jðr #6Ð"5ðsm Jðt $8Ð#7ðum Jðv "4Ð!3ðwm Jðx *˜Mðym Jðz 0Ð /ð{m Jð| #6Ð"5ð}m Jð~ &<Ð%;ðm Jð@ -JÐ,IðAm JðB !2Ð 1ðCm JðD .˜oðEm JðF !2Ð 1ðGm JðH .˜oðIm JðJ %:Ð$9ðKm JðL )BÐ(AðMm JðN .˜oðOm JðP %:Ð$9ðQm JðR $8Ð#7ðSm JðT &˜+ðUm JðV �EðWm JðX �EðYm JðZ �Eð[m Jð\ !2Ð 1ð]m Jð^ "4Ð!3ð_m Jð` &<Ð%;ðam Jðb $8Ð#7ðcm Jðd �xðem Jðf $8Ð#7ðgm Jðh "˜ ðim Jðj .˜oðkm Jðl *DÐ)Cðmm Jðn 0Ð /ðom Jðp 0Ð /ðqm Jðr #6Ð"5ðsm Jðt +FÐ*Eðum Jðv )BÐ(Aðwm Jðx �4ðym Jðz ,˜^ð{m Jð| �gð}m Jð~ �Eðm Jð@ (˜<ðAm JðB )BÐ(AðCm JðD ,˜^ðEm JðF *˜MðGm JðH "˜ ðIm JðJ *DÐ)CðKm JðL ,˜^ðMm JðN %:Ð$9ðOm JðP $8Ð#7ðQm JðR $8Ð#7ðSm JðT ,˜^ðUm JðV .˜oðWm JðX (@Ð'?ðYm JðZ (@Ð'?À&ð[m Jð m Jð m Jð\ %9ˆÔ!Ø"4ˆÔ Ð Ð r„ )
|
|||
|
|
r r r Ú__doc__rZ rA r r Ú__annotations__rB Úintr Ú
|
|||
|
|
__classcell__©r s @r‚ r= r= sÚ ø€ € € € € € ðJð JðV +0¨%ØØÐ 1Ð2ð+ñ +ô +И( 3œ-ð ð ñ ð */¨ØØÐ VÐWð*ñ *ô *И #œ ð ð ñ ð Ø #ØØØØ Ø $Ø&'Ø%&Ø#'Ø"&Ø&'Ø"#ØØ"%Ø ØØØØ ØØ ØØ $ØØØ Ø %Ø ØØ"Ø "ØØ!&Ø ØØ Ø Ø !Ø Ø27ØØØ ØØØ ØØØØ Ø!'Ø Ø ØØØØ Ø !ØØ $ØØ!"Ø%)ØØØØ %ØØ!&Ø $Ø Ø ØØ ØØ-1Ø !ØØ!$Ø ØØØ Ø %ØØ%)Ø Ø $Ø $Ø(-Ø "Ø%*ØØ!%ØØ#ØØ Ø ØØ!&Ø(,Ø%*Ø!%Ø Ø #Ø#'Ø ØØ #Ø ØØØØ Ø $Ø !Ø$)Ø(-Ø "Ø #Ø "Ø Ø Ø "Ø!&Ø(,Ø Ø Ø ØØ #Ø$(Ø Ø $Ø #ØØØØØ Ø Ø!&Ø #ØØ #Ø Ø Ø%)Ø$Ø Ø #Ø&)Ø$%ØØØØØØ$+Ø Ø ØØ%*Ø Ø #Ø "Ø "Ø Ø Ø#'Ø#(Ø #Ø ðaD5ð D5ð D5ð D5ð D5ð D5ð D5ð D5ð D5ð D5r„ r= c ó’ ‡ — e Zd ZdZddgZ d9deeef deee e f de
|
|||
|
|
e d e
|
|||
|
|
ee e
|
|||
|
|
f d
|
|||
|
|
e
|
|||
|
|
ee e
|
|||
|
|
eeee e
|
|||
|
|
f f f d e
|
|||
|
|
eeef d e
|
|||
|
|
eee e f d
|
|||
|
|
e
|
|||
|
|
e e dee
|
|||
|
|
ej j e
|
|||
|
|
ej j j f de
|
|||
|
|
d fˆ fd„
|
|||
|
|
Zd„ Zd„ Zd:de
|
|||
|
|
e defd„Z defd„Z dede defd„Z e d;d„¦ « Z d:dej! dej! d e"dej! fd „Z#d<d „Z$d:d e
|
|||
|
|
e e fd!„Z%d=d"e&j' d#efd$„Z(d"e&j' fd%„Z)d&„ Z*e d'eeeej! e+f f deeeej! e+f f fd(„¦ « Z,e ˆ fd)„¦ « Z-d*e eeeej! e+f f deeeej! e+f f fˆ fd+„ Z.d,„ Z/d>d-„Z0d.„ Z1d:d/e
|
|||
|
|
e e fd0„Z2d:d1eee"f d2e
|
|||
|
|
e" ddfˆ fd3„
|
|||
|
|
Z3ˆ fd4„Z4 d?d5e
|
|||
|
|
e d6e
|
|||
|
|
e d7eee e df fd8„Z5ˆ xZ6S )@Ú_UnslothGRPOTrainerrN Útrlrˆ N©NNr$ Ú reward_funcsÚargsÚ
|
|||
|
|
train_datasetÚ eval_datasetÚprocessing_classÚreward_processing_classesÚ callbacksÚ
|
|||
|
|
optimizersÚ peft_configr c ó¬ •‡ ‡— t |d¦ « r3t ‰d¦ « r#t ‰dd¦ « dk rd‰_ d‰_ ‰€Pt |t
|
|||
|
|
¦ « r|n |j j } | d¦ « d } t | › d�¦ « Š‰j
|
|||
|
|
pi } t |t
|
|||
|
|
¦ « rÀ|}
|
|||
|
|
| d ¦ « }t |t j
|
|||
|
|
¦ « s|d
|
|||
|
|
k s|€nCt |t
|
|||
|
|
¦ « rt t |¦ « }|| d < nt d |› d �¦ « ‚t j |
|
|||
|
|
¦ « }t t" |j d
|
|||
|
|
¦ « } |j |
|
|||
|
|
fi | ¤Ž}n"|j j }
|
|||
|
|
‰j
|
|||
|
|
�t d¦ « ‚t |d¦ « s0t' j |j ¦ « j ¦ « nAt' j | ¦ « j ¦ « j ¦ « ‰ _ ‰j r‰ |‰¦ « }|€ t= j |j j ¦ « }t |t> ¦ « r|j }n't |tB ¦ « r|}ntE d¦ « ‚|j# € |j$ |_# |j# ‰ _# |j% ‰ _% |j& ‰ _& t |dd ¦ « ‰ _' t |dd ¦ « ‰ _( t |j dd ¦ « ‰ _) t |j dd ¦ « ‰ _* t |tV ¦ « s|g}g ‰ _, t[ |¦ « D ]º\ }}t |t
|
|||
|
|
¦ « rt] j |fddi| ¤Ž||< t || t^ j0 ¦ « rD‰ j, 1 || j j d¦ « d ¦ « Œ•‰ j, 1 || j2 ¦ « Œ»|‰ _3 ‰j4 �…tk ‰j4 ¦ « tk |¦ « k r5t dtk ‰j4 ¦ « › dtk |¦ « › d�¦ « ‚t j6 ‰j4 t j7 ¬¦ « ‰ _4 n2t j8 tk |¦ « t j7 ¬¦ « ‰ _4 |€d gtk |¦ « z }nHt |tV ¦ « s|g}n/tk |¦ « tk |¦ « k rt d ¦ « ‚t[ ts ||¦ « ¦ « D ]f\ }\ }}t |tt ¦ « rI|€ tw j |j j ¦ « }|j% € |j$ |_# |j% |j _% |||< Œg|‰ _< ‰j= ‰ _= ‰j> ‰ _> ‰j? ‰ _? ‰j@ ‰ _@ ‰jA ‰ _A ‰jB ‰ _B ‰jC ‰ _C ‰jD ‰ _D ‰jE ‰ _E ‰j ‰ _ ‰j ‰ _ ‰jF ‰ _F ‰jG ‰ _G ‰jH ‰ _H ‰jI ‰ _I ‰jJ ‰ _J ‰jK ‰ _K ‰jL ‰ _L ‰jM ‰ _M ‰ jH r‰ jM d k rt� d ¦ « ‚‰ jH r‰ jK d k st� d ¦ « ‚‰jO ‰ _O t |t ¦ « sUt |t ¦ « s@t |t¢ ¦ « r:t¥ d!„ | S ¦ « D ¦ « ¦ « rt� d"¦ « ‚‰jT ‰ _T ‰jU ‰ _V ‰jW �‰jW n‰jU ‰ _W d
|
|||
|
|
‰ _X d ‰ _Y d|jZ d#< t· ¦ « \ |‰tº ||||| ¬$¦ « ‰j^ ‰ _^ ‰ j^ d%k rd ‰ __ n^tÁ |¦ « rd ‰ __ nGt j |
|
|||
|
|
¦ « }t t" |j d
|
|||
|
|
¦ « } |j |
|
|||
|
|
fi | ¤Ž‰ __ ‰ja r*tÅ |¦ « ‰ j_ �tÅ ‰ j_ ¦ « ‰ jH rrtÇ ¦ « st7 d&¦ « ‚tÉ ¦ « ‰ _e tÍ ‰ j^ ‰ jV ‰ jW ‰ j@ ‰ j^ d%k ‰ jI ‰ j> ¬'¦ « ‰ _g tÑ tV ¦ « tÑ tV ¦ « d(œ‰ _i d
|
|||
|
|
‰ _j ‰jk ‰ _k ‰jl ‰ _l ‰jm ‰ _m tÝ ‰jo ¬)¦ « tÝ ‰jo ¬)¦ « tÝ ‰jo ¬)¦ « tÑ ˆfd*„¦ « tÝ ‰jo ¬)¦ « d+œ‰ _p tã ‰jr d¬,¦ « ‰ j �r|tç ¦ « st7 d-¦ « ‚‰ j d.k r�‰ jt ju ru‰jv �‰jv }nd/‰jw › d0‰jx › �}tó |‰jz ¬1¦ « ‰ _{ ‰ j{ | t j} ~ ¦ « ¬2¦ « �n¦‰ j dk �rš‰ jt j ‰ jG z d
|
|||
|
|
k s%t d3‰ jG › d4‰ jt j › d5�¦ « ‚‰ jG dk rSt j€ � ˆ fd6„�t ‰ jt j ‰ jG z ¦ « D ¦ « ¦ « \ ‰ _ƒ }t ‰ jt j„ ¦ « �t
|
|||
|
|
j† d7< t ‰ jt j‡ ¦ « �t
|
|||
|
|
j† d8< t ‰ jt j ¦ « �t
|
|||
|
|
j† d9< �t
|
|||
|
|
j† d:d;¦ « �t
|
|||
|
|
j† d:< �t
|
|||
|
|
j† d<d=¦ « �t
|
|||
|
|
j† d<< ‰ j= �‰ j> �‰ j= ‰ j> z }nd }|jˆ ‰ _‰ ‰jŠ ‰ _‹ d‰ _Œ ‰ jt � ¦ « n‰‰ j> d|j% |jŽ |j& ‰ j@ ‰ jA ‰ jB ‰ jC ‰ jD ‰j� d>œ }‰jE rd?|d@< dA|dB< dC|dD< ‰j� �| ‘ ‰j� ¦ « �t% dHi |¤Ž‰ _“ d‰ _” ‰ j• – ‰ j— ¦ « ‰ j_ �v‰ j˜ r!�t3 ‰ j_ ‰ jt ¦ « ‰ __ nN‰ jš r!�t7 ‰ j_ ‰ jt ¦ « ‰ __ n&‰ jt œ ‰ j_ d¬E¦ « ‰ __ ‰j� r/‰ ž �t? ‰ j_ ‰ jt ¬F¦ « ¦ « t[ ‰ j3 ¦ « D ]e\ }}t |tt ¦ « rK‰ j˜ r �t3 |‰ jt ¦ « ‰ j3 |< Œ@‰ jt œ |dd¬G¦ « ‰ j3 |< Œfd S )INÚ vllm_enginerê FTrV ú/re z-GRPOÚ torch_dtyperM z‰Invalid `torch_dtype` passed to `GRPOConfig`. Expected either 'auto' or a string representing a `torch.dtype` (e.g., 'float32'), but got ú.r z¡You passed `model_init_kwargs` to the `GRPOConfig`, but your model is already instantiated. This argument can only be used when the `model` argument is a string.Úget_base_modelz>PEFT is required to use `peft_config`. Run `pip install peft`.zWThe `processing_class` must be either a `PreTrainedTokenizerBase` or a `ProcessorMixin`Ú image_tokenÚimage_token_idÚvision_start_token_idÚvision_end_token_idÚ
|
|||
|
|
num_labelsrm zNumber of reward weights (z)) must match number of reward functions (ú)©r zRThe number of reward processing classes must match the number of reward functions.r� zOLiger Kernels don't currently support masking token positions based on entropy.rW zwLiger Kernels currently only support token-level importance sampling. Please set`importance_sampling_level` to 'token'.c 3 ó@ K — | ]}t |t ¦ « V — Œd S rþ )Ú
|
|||
|
|
isinstancer )r Údss r‚ ú <genexpr>z/_UnslothGRPOTrainer.__init__.<locals>.<genexpr> s- è è € Ð6wÐ6wÐ[]µzÀ"ÅoÑ7VÔ7VÐ6wÐ6wÐ6wÐ6wÐ6wÐ6wr„ z^Iterable datasets are not yet supported in GRPOTrainer. Please use a standard dataset instead.Úestimate_tokens)r$ r Ú
|
|||
|
|
data_collatorr r r r r r‘ zWLiger is required to use `liger_loss` as the GRPO loss. Run `pip install liger-kernel`.)r r‰ r‹ rŽ Ú
|
|||
|
|
use_ref_modelr‡ rŒ )ÚtrainÚeval©Úmaxlenc ó. •— t ‰ j ¬¦ « S )Nr4 )r+ rá )r s €r‚ ú<lambda>z._UnslothGRPOTrainer.__init__.<locals>.<lambda>n s ø€ 5¸Ô8RÐ+SÑ+SÔ+S€ r„ )ÚimageÚpromptÚ
|
|||
|
|
completionÚrewardsr® )Údevice_specificzkvLLM is not available and `use_vllm` is set to True. Please install vLLM with `pip install vllm` to use it.Úserverzhttp://ú:)Úbase_urlÚconnection_timeoutrÑ zvllm_tensor_parallel_size (z) must divide world size (z ) evenly.c óp •— g | ]2}t t |‰j z |d z ‰j z ¦ « ¦ « ‘Œ3S )rm )Úlistr ró )r r
|
|||
|
|
r
|
|||
|
|
s €r‚ r z0_UnslothGRPOTrainer.__init__.<locals>.<listcomp>� sQ ø€ ð ð ð à !õ !¥ q¨4Ô+IÑ'IÈAÐPQÉEÐUYÔUsÑKsÑ!tÔ!tÑ uÔ uðð ð r„ ÚRANKÚ
|
|||
|
|
LOCAL_RANKÚ
|
|||
|
|
WORLD_SIZEÚ MASTER_ADDRÚ localhostÚ MASTER_PORTÚ12345) Úmax_new_tokensÚ do_sampleÚ pad_token_idÚ bos_token_idÚ eos_token_idrŽ rã rä rå rç ré rS Úmax_batch_tokensi Ú
|
|||
|
|
num_blocksé€ Ú
|
|||
|
|
block_size)Úevaluation_mode)Ú ref_modelr* )rS Údevice_placementr ) r Úgetattrrê rì r, r ÚconfigÚ
|
|||
|
|
_name_or_pathÚsplitr rÛ rœ rS r r¥ r
|
|||
|
|
Úfrom_pretrainedrT Ú
|
|||
|
|
architecturesr3 Ú signaturerü Ú
|
|||
|
|
parametersÚkeysr# Úmodel_kwarg_keysr8 Ú ImportErrorrÀ Ú _enable_gradient_checkpointingr r Ú tokenizerr Ú TypeErrorÚ pad_tokenÚ eos_tokenrL rN r$ r% r& r' rB Úreward_func_namesÚ enumerater rA ÚModulerw r r r÷ r Útensorrs Úonesrq r r r rÝ rŒ rÞ rŽ rã rä rå rç rè rò ró rþ r‡ rø rö rù rý ÚNotImplementedErrorrà r ÚdictÚanyÚvaluesrô rõ r‰ r‹ Ú_stepÚ_buffered_inputsÚwarnings_issuedr r r2 r rT r9 rÜ r, r7 r% Ú_forward_redirectionÚLigerFusedLinearGRPOLossÚliger_grpo_lossr* Ú_metricsÚ_total_train_tokensrÿ r r r+ rá Ú_logsrN rˆ r; r* Úis_main_processrë rï rð r$ rñ Ú vllm_clientÚinit_communicatorÚcudaÚcurrent_deviceÚ
|
|||
|
|
num_processesÚ distributedÚ new_subgroups_by_enumerationr Útp_groupÚ
|
|||
|
|
process_indexrC r! Úlocal_process_indexr Úllmrî Úguided_decoding_regexÚ_last_loaded_stepÚwait_for_everyonerM ré ræ Úupdater Úgeneration_configÚmodel_accepts_loss_kwargsr$ Úadd_model_tagsÚ
|
|||
|
|
_tag_namesÚis_deepspeed_enabledrF Úis_fsdp_enabledrG Ú
|
|||
|
|
prepare_modelrú Ú add_callbackr! )r
|
|||
|
|
r$ r r r r r r r r r Ú
|
|||
|
|
model_namerÛ Úmodel_idr! rW Ú architecturerb r
|
|||
|
|
Ú reward_funcÚreward_processing_classr? Ú_Ú
|
|||
|
|
max_model_lenræ r s ` ` €r‚ r z _UnslothGRPOTrainer.__init__f sÑ øøø€ õ �5˜-Ñ (Ô (ð &W°T¸:Ñ-FÔ-Fð &ݘ˜j¨%Ñ0Ô0°EÒ9Ð9Ø $�”
|
|||
|
|
Ø%ˆDŒNà ˆ<Ý",¨UµCÑ"8Ô"8ÐX˜˜¸e¼lÔ>XˆJØ#×)Ò)¨#Ñ.Ô.¨rÔ2ˆJÝ Ð 2Ð 2Ð 2Ñ3Ô3ˆDð !Ô 2Ð 8°bÐÝ �e�SÑ !Ô !ð Ø ˆHØ+×/Ò/°
|
|||
|
|
Ñ>Ô>ˆKݘ+¥u¤{Ñ3Ô3ð
|
|||
|
|
°{ÀfÒ7LÐ7LÐP[ÐPcØݘKÑ-Ô-ð
|
|||
|
|
Ý %¥e¨[Ñ 9Ô 9� Ø3>Ð! -Ñ0Ð0å ðQØBMðQð Qð Qñô ð õ
|
|||
|
|
Ô/°Ñ9Ô9ˆFÝ"¥<°Ô1EÀaÔ1HÑIÔIˆLØ0�LÔ0°ÐOÐOÐ=NÐOÐOˆEˆEà ”|Ô1ˆHØÔ%Ð1Ý ð\ñô ð õ ˜5Ð"2Ñ3Ô3ð
|
|||
|
|
U�GÔ ˜eœmÑ ,Ô ,Ô 7× <Ò <Ñ >Ô >Ð >åÔ" 5×#7Ò#7Ñ#9Ô#9Ô#AÑBÔBÔM×RÒRÑTÔTð
|
|||
|
|
Ô ð ð Ô &ð EØ×7Ò7¸¸tÑDÔDˆEð Ð #Ý ,Ô <¸U¼\Ô=WÑ XÔ XÐ õ Ð&Ñ 7Ô 7ð wØ(Ô2ˆIˆIÝ
|
|||
|
|
Ð(Õ*AÑ
|
|||
|
|
BÔ
|
|||
|
|
Bð wØ(ˆIˆIåÐ uÑvÔvÐ và Ô Ð &Ø"+Ô"5ˆIÔ à"Ô,ˆŒØ %Ô 2ˆÔØ %Ô 2ˆÔÝ"Ð#3°]ÀDÑIÔIˆÔÝ %Ð&6Ð8HÈ$Ñ OÔ OˆÔÝ%,¨U¬\Ð;RÐTXÑ%YÔ%YˆÔ"Ý#*¨5¬<Ð9NÐPTÑ#UÔ#UˆÔ õ ˜,Ñ-Ô-ð *Ø (˜>ˆLØ!#ˆÔ Ý '¨ Ñ 5Ô 5ð Hð H‰NˆAˆ{ݘ+¥sÑ+Ô+ð
|
|||
|
|
Ý"DÔ"TØ ð#ð #Ø,-ð#Ø1Bð#ð #� ˜Q‘õ ˜, qœ/2¬9Ñ5Ô5ð
|
|||
|
|
HØÔ&×-Ò-¨l¸1¬oÔ.DÔ.R×.XÒ.XÐY\Ñ.]Ô.]Ð^`Ô.aÑbÔbÐbÐbàÔ&×-Ò-¨l¸1¬oÔ.FÑGÔGÐGÐGØ (ˆÔð Ô Ð *Ý�4Ô&Ñ'Ô'3¨|Ñ+<Ô+<Ò<Ð<Ý ð7µ°TÔ5HÑ1IÔ1Ið 7ð 7Ý"% lÑ"3Ô"3ð7ð 7ð 7ñô ð õ #(¤,¨tÔ/BÍ%Ì-Ð"XÑ"XÔ"XˆDÔ Ð å"'¤*S° Ñ->Ô->ÅeÄmÐ"TÑ"TÔ"TˆDÔ ð %Ð ,Ø)-¨µ°\Ñ1BÔ1BÑ(BÐ %Ð %ÝÐ 5µtÑ<Ô<ð wØ)BÐ(CÐ %Ð %åÐ,Ñ-Ô-µ°\Ñ1BÔ1BÒBÐBÝ Ð!uÑvÔvÐvå9BÅ3ÐG`ÐbnÑCoÔCoÑ9pÔ9pð Gð GÑ 5ˆAÑ5Ð'¨Ý˜+¥Ñ7Ô7ð
|
|||
|
|
GØ*Ð2Ý.;Ô.KÈKÔL^ÔLlÑ.mÔ.mÐ+Ø*Ô7Ð?Ø8OÔ8YÐ+Ô5ð 3JÔ2V� Ô"Ô/Ø/FÐ)¨!Ñ,øØ)BˆÔ&ð "&Ô!7ˆÔ Ø%)Ô%?ˆÔ"Ø #Ô 3ˆÔ Ø Ô+ˆÔØ”ZˆŒ
|
|||
|
|
Ø”ZˆŒ
|
|||
|
|
Ø”ZˆŒ
|
|||
|
|
Ø"&Ô"9ˆÔ Ø&*Ô&AˆÔ#Ø œ
|
|||
|
|
ˆŒ
|
|||
|
|
Ø œ ˆŒØ+/Ô+KˆÔ(Ø)-Ô)GˆÔ&Ø "Ô 1ˆÔØ œ ˆŒØ !Ô /ˆÔØ)-Ô)GˆÔ&Ø*.Ô*IˆÔ'Ø$(Ô$=ˆÔ!Ø Ô ð 4Ô#<¸sÒ#BÐ#BÝ%Øañô ð
|
|||
|
|
ð Ô ð tÔ'EÈÒ'PÐ'PÝ%ð:ñô ð
|
|||
|
|
ð $Ô 3ˆÔ õ
|
|||
|
|
�}¥oÑ 6Ô 6ð
|
|||
|
|
å˜,Ñ8Ô8ð
|
|||
|
|
õ ˜<Ñ.Ô.ð
|
|||
|
|
õ 47Ð6wÐ6wÐam×atÒatÑavÔavÐ6wÑ6wÔ6wÑ3wÔ3wð
|
|||
|
|
õ &Øpñô ð
|
|||
|
|
ð
|
|||
|
|
#Ô 1ˆÔØ œ<ˆÔØ15Ô1BÐ1N˜DÔ -Ð -ÐTXÔT`ˆÔàˆŒ
|
|||
|
|
ð !%ˆÔ ð 48ˆÔ Ð /Ñ0å
|
|||
|
|
‰Œ×ÒØØÝ"Ø'Ø%Ø -Ø Ø!ð ñ
|
|||
|
|
ô
|
|||
|
|
ð
|
|||
|
|
ð ”IˆŒ Ø Œ9˜Ò Ð à !ˆDŒNˆNÝ
|
|||
|
|
˜5Ñ
|
|||
|
|
!Ô
|
|||
|
|
!ð Yð "ˆDŒNˆNõ Ô/°Ñ9Ô9ˆFÝ"¥<°Ô1EÀaÔ1HÑIÔIˆLØ 9˜\Ô 9¸(Ð XÐ XÐFWÐ XÐ XˆDŒNð Ô ð 9Ý $ UÑ +Ô +Ð +ØŒ~Ð)Ý(¨¬ Ñ8Ô8Ð8ð Ô ð Ý,Ñ.Ô.ð
|
|||
|
|
Ý!Ømñô ð õ )<Ñ(=Ô(=ˆDÔ %å#;Ø”YØ Ô ,Ø !Ô .Ø Ô ,Ø "œi¨3Ò .Ø œ.Ø&*Ô&@ð$ñ $ô $ˆDÔ õ #.dÑ"3Ô"3½[ÍÑ=NÔ=NÐOÐOˆŒ
|
|||
|
|
Ø#$ˆÔ Ø #Ô 3ˆÔ Ø(,Ô(EˆÔ%Ø(,Ô(EˆÔ%õ $Ô"<Ð=Ñ=Ô=Ý 4Ô#=Ð>Ñ>Ô>Ý tÔ'AÐBÑBÔBÝ"Ð#SÐ#SÐ#SÐ#SÑTÔTÝ tÔ'AÐBÑBÔBð
|
|||
|
|
ð
|
|||
|
|
ˆŒ
|
|||
|
|
õ �”¨DÐ1Ñ1Ô1Ð1à Œ=ñ @ KÝ$Ñ&Ô&ð
|
|||
|
|
Ý!ð4ñô ð ð
|
|||
|
|
Œ~ Ò)Ð)¨dÔ.>Ô.NÐ)ØÔ,Ð8Ø #Ô 8�H�Hà X¨Ô)>Ð XÐ XÀÔAVÐ XÐ X�HÝ#-°xÐTXÔTlÐ#mÑ#mÔ#m�Ô ØÔ ×2Ò2½%¼*×:SÒ:SÑ:UÔ:UÐ2ÑVÔVÐVÑVà” :Ò-Ñ-ØÔ'Ô5¸Ô8VÑVÐZ[Ò[Ð[Ý$ðF°dÔ6Tð Fð FØ Ô ,Ô :ðFð Fð Fñô ð ð
|
|||
|
|
Ô1°AÒ5Ð5Ý',Ô'8×'UÒ'Uðð ð ð æ%*¨4Ô+;Ô+IÈTÔMkÑ+kÑ%lÔ%lðñ ô ñ(ô (Ñ$�D”M 1õ &)¨Ô)9Ô)GÑ%HÔ%H–”
|
|||
|
|
˜6Ñ"Ý+.¨tÔ/?Ô/SÑ+TÔ+T–”
|
|||
|
|
˜<Ñ(Ý+.¨tÔ/?Ô/MÑ+NÔ+N–”
|
|||
|
|
˜<Ñ(Þ,.¬J¯NªN¸=È+Ñ,VÔ,V–”
|
|||
|
|
˜=Ñ)Þ,.¬J¯NªN¸=È'Ñ,RÔ,R–”
|
|||
|
|
˜=Ñ)àÔ)Ð5¸$Ô:TÐ:`Ø$(Ô$:¸TÔ=WÑ$W�M�Mà$(�MØ Ô,�”Ø)-Ô)HˆDÔ &à%'ˆDÔ "Ø Ô × .Ò .Ñ 0Ô 0Ð 0Ð 0ð #'Ô"<Ø !Ø )Ô 6Ø )Ô 6Ø )Ô 6Ø #Ô /Ø œØ œØ œØ&*Ô&=Ø(,Ô(Að !ð !Ð ð Ô*ð
|
|||
|
|
6Ø8;Ð!Ð"4Ñ5Ø26Ð! ,Ñ/Ø25Ð! ,Ñ/ØÔ%Ð1Ø!×(Ò(¨Ô)?Ñ@Ô@Ð@Þ%5Ð%JÐ%JÐ8IÐ%JÐ%JˆDÔ "ð
|
|||
|
|
*/ˆÔ&ð
|
|||
|
|
Œ
|
|||
|
|
×!Ò! $¤/Ñ2Ô2Ð2à Œ>Ð %ØÔ(ð
|
|||
|
|
fÞ!2°4´>À4ÔCSÑ!TÔ!T�”�ØÔ%ð
|
|||
|
|
fÞ!-¨d¬n¸dÔ>NÑ!OÔ!O�”�à!%Ô!1×!?Ò!?ÀÄÐ`dÐ!?Ñ!eÔ!e�”à Ô ð lØ × Ò Ö 2¸T¼^ÐY]ÔYiÐ jÑ jÔ jÑ kÔ kÐ kå '¨Ô(9Ñ :Ô :ð ð ‰NˆAˆ{ݘ+¥Ñ7Ô7ð
|
|||
|
|
ØÔ,ð Þ+<¸[È$ÔJZÑ+[Ô+[�DÔ% aÑ(Ð(ð ,0Ô+;×+IÒ+IØ#°TÈDð ,Jñ ,ô ,�DÔ% aÑ(øð
|
|||
|
|
ð r„ c ó* — | j € ddg| _ d S d S )Nr9 r8 )Ú_signature_columns)r
|
|||
|
|
s r‚ Ú _set_signature_columns_if_neededz4_UnslothGRPOTrainer._set_signature_columns_if_neededÖ s( € ð
|
|||
|
|
Ô "Ð *Ø'/°Ð&9ˆDÔ #Ð #Ð #ð +Ð *r„ c óÀ — | j €t d¦ « ‚| j }| j }t ¦ « r2t |t
|
|||
|
|
j ¦ « r| |d¬¦ « }n| |d¬¦ « }| j | j
|
|||
|
|
j z || j
|
|||
|
|
j | j
|
|||
|
|
j
|
|||
|
|
| j
|
|||
|
|
j dœ}t |t j j j ¦ « sc| ¦ « |d< | j
|
|||
|
|
j |d< t+ t, | j
|
|||
|
|
j | j
|
|||
|
|
j ¬¦ « |d< | j
|
|||
|
|
j |d < | j t7 |fi |¤Ž¦ « S )
|
|||
|
|
Nz+Trainer: training requires a train_dataset.Útraining)Ú description)Ú
|
|||
|
|
batch_sizeÚ
|
|||
|
|
collate_fnÚ num_workersÚ
|
|||
|
|
pin_memoryÚpersistent_workersÚsamplerÚ drop_last)rŸ ÚrankÚworker_init_fnÚprefetch_factor) r r¥ r0 r5 r, r) r Ú_remove_unused_columnsÚ"_get_collator_with_removed_columnsÚ_train_batch_sizer râ r™ r´ rµ rS ÚutilsÚdatar Ú_get_train_samplerr— rE rL r� rš r* Úpreparer )r
|
|||
|
|
r r0 Údataloader_paramss r‚ Úget_train_dataloaderz(_UnslothGRPOTrainer.get_train_dataloaderè si € Ø Ô Ð %Ý Ð JÑKÔKÐ Kà Ô*ˆ
|
|||
|
|
Ø Ô*ˆ
|
|||
|
|
Ý Ñ "Ô "ð k¥z°-ÅÔAQÑ'RÔ'Rð kØ × 7Ò 7¸
|
|||
|
|
ÐS]Ð 7Ñ ^Ô ^ˆMˆMà × CÒ CÀMÐ_iÐ CÑ jÔ jˆMð Ô0°4´9Ô3QÑQØ'Ø œ9Ô;Ø œ)Ô9Ø"&¤)Ô"Ið
|
|||
|
|
ð
|
|||
|
|
Ðõ ˜-¬Ô)9Ô)IÑJÔJð XØ+/×+BÒ+BÑ+DÔ+DÐ ˜iÑ (Ø-1¬YÔ-KÐ ˜kÑ *Ý29ݨ¬Ô)IÐPTÔPYÔPgð3ñ 3ô 3Ð Ð .Ñ /ð 48´9Ô3WÐ Ð /Ñ 0àÔ ×'Ò'
|
|||
|
|
°=Ð(VÐ(VÐDUÐ(VÐ(VÑWÔWÐWr„ ÚdatasetÚreturnc ó® — |€| j }t || j | j j | j z | j | j j z | j | j j ¬¦ « S )N)Ú data_sourceÚmini_repeat_countr� Ú repeat_countÚshufflerˆ ) r r rÞ r rá rô râ rà rˆ )r
|
|||
|
|
r° s r‚ r¬ z&_UnslothGRPOTrainer._get_train_sampler s^ € ð2 ˆ?ØÔ(ˆGÝ Ø Ø "Ô 2Ø”yÔ6¸$Ô:NÑNØ Ô,¨t¬yÔ/MÑMØÔ(Ø”” ð
|
|||
|
|
|
|||
|
|
ñ
|
|||
|
|
ô
|
|||
|
|
ð
|
|||
|
|
r„ c óD — t || j | j j ¬¦ « S )N)r³ r´ rˆ )r rÞ r rˆ )r
|
|||
|
|
r s r‚ Ú_get_eval_samplerz%_UnslothGRPOTrainer._get_eval_sampler* s) € å Ø$Ø "Ô 2Ø”” ð
|
|||
|
|
ñ
|
|||
|
|
ô
|
|||
|
|
ð
|
|||
|
|
r„ c óî — d|j _ t |¦ « r|j ¦ « n| ¦ « |j pi }d|vp|d }|r| ¦ « |S )z-Enables gradient checkpointing for the model.FÚ
|
|||
|
|
use_reentrant)rW Ú use_cacher9 Ú
|
|||
|
|
base_modelÚ gradient_checkpointing_enablerÁ Úenable_input_require_grads)r
|
|||
|
|
r$ r rÁ rº s r‚ ra z2_UnslothGRPOTrainer._enable_gradient_checkpointing2 s• € ð "'ˆŒ Ô õ ˜Ñ Ô ð 2Ø Ô × :Ò :Ñ <Ô <Ð <Ð <ð
|
|||
|
|
× /Ò /Ñ 1Ô 1Ð 1à(,Ô(JÐ(PÈbÐ%à Ð#@Ð @Ð rÐDaÐbqÔDrð ð ð /Ø × ,Ò ,Ñ .Ô .Ð .àˆ r„ c ó — t |¦ « r |j j }||dœ} |�|�|| d< |�|| d< |�|| d< |�|| d< d| j v r|dz | d< |j d i | ¤Žj }
|
|||
|
|
|
|
|||
|
|
d d …d d…d d …f }
|
|||
|
|
|
|
|||
|
|
d d …| d …d d …f }
|
|||
|
|
|
|
|||
|
|
S )
|
|||
|
|
N)r« r Úimage_grid_thwÚ pixel_valuesÚpixel_attention_maskÚ image_sizesr rm re r )r9 r¼ r$ r_ Úlast_hidden_state) r
|
|||
|
|
Úunwrapped_modelr« r r rÁ rÀ r rÃ Ú model_inputsrÄ s r‚ Ú_get_last_hidden_statez*_UnslothGRPOTrainer._get_last_hidden_stateH sø € õ ˜ Ñ )Ô )ð ?Ø -Ô 8Ô >ˆOð &/À.ÐQÐQˆ ð Ð %¨,Ð*BØ-;ˆLÐ)Ñ *à Ð #Ø+7ˆL˜ Ñ (à Ð +Ø3GˆLÐ/Ñ 0à Ð "Ø*5ˆL˜ Ñ 'ð ˜tÔ 4Ð 4Ð 4à-;¸aÑ-?ˆLÐ)Ñ *à 1˜OÔ 1Ð AÐ A°LÐ AÐ AÔ SÐà -¨a¨a¨a°°"°°a°a°a¨iÔ 8Ðà -¨a¨a¨a°.° Ð1AÐ1AÀ1À1À1Ð.DÔ EÐØ Ð r„ Ú entropiesr¬ Ú thresholdc ó� — || ¦ « ¦ « }| ¦ « dk r t j |t j ¬¦ « S | j |¦ « }t j ||¦ « }|| ¦ « z }||k } | | ¦ « z S )aÐ
|
|||
|
|
Returns a binary mask identifying tokens whose entropy exceeds a given quantile threshold.
|
|||
|
|
|
|||
|
|
Args:
|
|||
|
|
entropies (`torch.Tensor`):
|
|||
|
|
Tensor of shape (batch_size, seq_len) with per-token entropy values.
|
|||
|
|
mask (`torch.Tensor`):
|
|||
|
|
Binary mask of the same shape as `entropies`, where `1` indicates valid tokens and `0` padding.
|
|||
|
|
threshold (`float`):
|
|||
|
|
Quantile threshold between `0.0` and `1.0` to select high-entropy tokens.
|
|||
|
|
|
|||
|
|
Returns:
|
|||
|
|
`torch.Tensor`:
|
|||
|
|
Boolean mask of shape (batch_size, seq_len), where `True` indicates tokens with entropy >= threshold and
|
|||
|
|
`False` otherwise.
|
|||
|
|
r r* )ÚboolÚfloatÚnumelrS Ú
|
|||
|
|
zeros_liker* r. Úquantile)
|
|||
|
|
r
|
|||
|
|
rÈ r¬ rÉ r* Únon_pad_entropiesÚall_non_pad_entropiesÚentropy_thresholdÚmasked_entropiesÚ entropy_masks
|
|||
|
|
r‚ Úget_high_entropy_maskz)_UnslothGRPOTrainer.get_high_entropy_masks s® € ð& & d§i¢i¡k¤kÔ 2× 8Ò 8Ñ :Ô :ÐØ × "Ò "Ñ $Ô $¨Ò )Ð )ÝÔ# IµU´ZÐ@Ñ@Ô@Ð @Ø $Ô 0× 7Ò 7Ð8IÑ JÔ JÐ å !œNÐ+@À)Ñ LÔ LÐØ$ t§z¢z¡|¤|Ñ3ÐØ'Ð+<Ò<ˆ ؘdŸiši™kœkÑ)Ð)r„ Fc ó — dS )NTr )
|
|||
|
|
r rC r! rœ rS r" r# r r' r( ry Útrl.trainer.utilsr- ) r
|
|||
|
|
r$ r« r r r� Úcompute_entropyr r¯ ry rÈ r- s r‚ Ú"_get_per_token_logps_and_entropiesz6_UnslothGRPOTrainer._get_per_token_logps_and_entropies� s € ð Ø �:r„ Úextra_prefixesc óP — |pg }dg|z }|D ]}| |d¦ « }Œ|S )Nz_checkpoint_wrapped_module.rN )Úreplace)r
|
|||
|
|
ÚnamerÚ ÚprefixesÚprefixs r‚ Ú_fix_param_name_to_vllmz+_UnslothGRPOTrainer._fix_param_name_to_vllmº sB € Ø'Ð-¨2ˆØ1Ð2°^ÑCˆØ ð ,ð ,ˆFØ—<’< ¨Ñ+Ô+ˆDˆD؈ r„ Úmodulerß c óh — |€t ¦ « }| ¦ « D ](\ }}|r|› d|› �n|}| |||¬¦ « Œ)t |t ¦ « rÏt j |dd¬¦ « 5 | ¦ « D ]‡\ }}|r|› d|› �n|} | | dg¬¦ « } | |v rŒ-| | ¦ « | j dk r-| j
|
|||
|
|
j r!| j
|
|||
|
|
| |j ¦ « Œz| j d k r Œˆ ddd¦ « dS # 1 swxY w Y dS dS )
|
|||
|
|
zdMemory-efficient post-order traversal of FSDP modules to extract full parameters and sync with vLLM.Nr" )rß ÚvisitedF)ÚrecurseÚ writebackz_fsdp_wrapped_module.)rÚ r= rV )ÚsetÚnamed_childrenÚ_sync_fsdp1_params_to_vllmr, r Úsummon_full_paramsÚnamed_parametersrà Úaddrì r* rx ry Úupdate_named_paramr« )
|
|||
|
|
r
|
|||
|
|
rá rß rã Ú
|
|||
|
|
child_nameÚ child_moduleÚ child_prefixÚ
|
|||
|
|
param_nameÚparamÚ full_names
|
|||
|
|
r‚ rè z._UnslothGRPOTrainer._sync_fsdp1_params_to_vllmÁ sÚ € ð ˆ?Ý‘e”eˆGØ(.×(=Ò(=Ñ(?Ô(?ð ð Ñ $ˆJ˜ Ø7=ÐM˜fÐ3Ð3 zÐ3Ð3Ð3À:ˆLØ × +Ò +Ø \¸7ð
|
|||
|
|
,ñ
|
|||
|
|
ô
|
|||
|
|
ð
|
|||
|
|
ð
|
|||
|
|
õ �f�dÑ #Ô #ð ÝÔ(¨¸È%ÐPÑPÔPð
|
|||
|
|
ð
|
|||
|
|
Ø)/×)@Ò)@Ñ)BÔ)Bð ð Ñ%�J Ø<BÐ R 6Ð 8Ð 8¨JÐ 8Ð 8Ð 8È
|
|||
|
|
�IØ $× <Ò <¸YÐXoÐWpÐ <Ñ qÔ q�Ià GÐ+Ð+Ø Ø—K’K Ñ*Ô*Ð*à”~¨Ò1Ð1°dÔ6FÔ6VÐ1Ø Ô(×;Ò;¸IÀuÄzÑRÔRÐRÐRØ œ ¨:Ò5Ð5à à øð ð
|
|||
|
|
ð
|
|||
|
|
ð
|
|||
|
|
ñ
|
|||
|
|
ô
|
|||
|
|
ð
|
|||
|
|
ð
|
|||
|
|
ð
|
|||
|
|
ð
|
|||
|
|
ð
|
|||
|
|
ð
|
|||
|
|
ð
|
|||
|
|
øøøð
|
|||
|
|
ð
|
|||
|
|
ð
|
|||
|
|
ð
|
|||
|
|
ð
|
|||
|
|
ð
|
|||
|
|
ð ð s Á:B D%Ä%D)Ä,D)c ó> — | ¦ « D ]‡\ }}|j r'| t j d¦ « ¦ « }| ¦ « }| j dk r(| j j r | j
|
|||
|
|
||¦ « Œz| j dk r Œˆd S )Nr{ r= rV ) ÚitemsÚis_cpurr rS rÒ Ú full_tensorrì r* rx ry rì )r
|
|||
|
|
rá rÝ rñ s r‚ Ú_sync_fsdp2_params_to_vllmz._UnslothGRPOTrainer._sync_fsdp2_params_to_vllmÞ s¥ € à!Ÿ<š<™>œ>ð ð ‰KˆD�%ØŒ|ð
|
|||
|
|
7Ø Ÿš¥¤ ¨fÑ!5Ô!5Ñ6Ô6�Ø×%Ò%Ñ'Ô'ˆEàŒ~ Ò)Ð)¨dÔ.>Ô.NÐ)ØÔ ×3Ò3°D¸%Ñ@Ô@Ð@Ð@Ø” :Ò-Ð-ààøð ð r„ c ó — d S rþ r )r
|
|||
|
|
r r¯ s r‚ Ú_move_model_to_vllmz'_UnslothGRPOTrainer._move_model_to_vllmí s € ¸4¸4r„ Úgeneration_batchc ó@ — t | d¦ « r>t | j j j j dd¦ « r| j ¦ « | j j rdnd}|dk r·| j j
|
|||
|
|
| j z }| j |z dk s| j
|
|||
|
|
€^| |¦ « }t |¦ « }t! |¦ « }t# || j j
|
|||
|
|
¦ « }d„ |D ¦ « | _
|
|||
|
|
| j
|
|||
|
|
| j | j j
|
|||
|
|
z }| xj dz
|
|||
|
|
c_ n| |¦ « }t | d¦ « r]t | j j j j dd¦ « r8| j t& j d d¦ « ¦ « |S )
|
|||
|
|
Nrƒ Úenable_sleep_modeFr2 r3 r c ó, — g | ]}t |¦ « ‘ŒS r )rU )r Úbatchs r‚ r z7_UnslothGRPOTrainer._prepare_inputs.<locals>.<listcomp> s" € Ð(mÐ(mÐ(mÐQVÕ)EÀeÑ)LÔ)LÐ(mÐ(mÐ(mr„ rm ÚVLLM_SLEEP_MODE)r rV rƒ Ú
|
|||
|
|
llm_engineÚ vllm_configÚ model_configÚwake_upr$ r› r râ rô ro rp Ú _generate_and_score_completionsrP rO rQ ÚsleeprC r! rœ )r
|
|||
|
|
rú ÚmodeÚgenerate_everyÚgeneration_batchesÚinputss r‚ Ú_prepare_inputsz#_UnslothGRPOTrainer._prepare_inputsï s’ € õ �4˜Ñ Ô ð "Ý�d”hÔ)Ô5ÔBÐDWÐY^Ñ_Ô_ð "ØŒx× Ò Ñ!Ô!Ð!à œ*Ô-Ð9ˆwˆw°6ˆØ �7Š?ˆ?Ø !œYÔ ;¸dÔ>QÑ QˆNØŒz˜NÑ*¨aÒ/Ð/°4Ô3HÐ3Pà#'×#GÒ#GÐHXÑ#YÔ#YÐ Ý#=Ð>NÑ#OÔ#OÐ Ý#8Ð9IÑ#JÔ#JÐ Ý%6Ð7GÈÌÔIgÑ%hÔ%hÐ"Ø(mÐ(mÐZlÐ(mÑ(mÔ(m�Ô%ØÔ*¨4¬:¸¼ Ô8VÑ+VÔWˆFØ ˆJŒJ˜!‰OˆJŒJˆJð ×9Ò9Ð:JÑKÔKˆFÝ �4˜Ñ Ô ð EÝ�t”xÔ*Ô6ÔCÐEXÐZ_Ñ`Ô`ð
|
|||
|
|
EØ”—’�rœzŸ~š~Ð.?ÀÑ CÔ CÑDÔDÐD؈
|
|||
|
|
r„ c óÀ •‡‡‡— | j j }t j t |¦ « t | j ¦ « |¬¦ « }d„ ‰d D ¦ « }ˆfd„|D ¦ « }| j |d< t t | j | j | j
|
|||
|
|
¦ « ¦ « D �]_\ } \ }
|
|||
|
|
Š} t | | ¦ « 5 t |
|
|||
|
|
t j ¦ « rÓt ‰d ¦ « r)d„ t ||¦ « D ¦ « } ˆfd„| D ¦ « }
|
|||
|
|
nd„ t ||¦ « D ¦ « }
|
|||
|
|
‰|
|
|||
|
|
d d
|
|||
|
|
d d ¬
|
|||
|
|
¦ « }t! ¦ « |¦ « }t j ¦ « 5 |
|
|||
|
|
di |¤Žj d d …df |d d …| f< d d d ¦ « n # 1 swxY w Y n@ |
|
|||
|
|
d|||dœ|¤Ž}d„ |D ¦ « }t j |t j |¬¦ « |d d …| f< d d d ¦ « n # 1 swxY w Y �Œat j |¦ « d¬¦ « ¦ « r–t j |¦ « d¬¦ « d
|
|||
|
|
¬¦ « d d Šˆfd„| ¦ « D ¦ « }|‰ |d< |‰ |d< t7 j d|› d�¦ « t; |¦ « }|S )NrÑ c ó — g | ]}|d v¯|‘Œ S ))r9 r: Úcompletion_idsr )r Úkeys r‚ r z:_UnslothGRPOTrainer._calculate_rewards.<locals>.<listcomp> s# € ÐbÐbÐb˜¨CÐ7aÐ,aÐ,a�Ð,aÐ,aÐ,ar„ r c ó0 •‡— i | ]Š‰ˆfd „‰D ¦ « “ŒS )c ó •— g | ]
|
|||
|
|
}|‰ ‘Œ S r r )r Úexampler s €r‚ r zE_UnslothGRPOTrainer._calculate_rewards.<locals>.<dictcomp>.<listcomp> s ø€ Ð BÐ BÐ B°˜w sœ|Ð BÐ BÐ Br„ r )r r r s @€r‚ ú
|
|||
|
|
<dictcomp>z:_UnslothGRPOTrainer._calculate_rewards.<locals>.<dictcomp> s1 øø€ ÐSÐSÐSÀs˜Ð BÐ BÐ BÐ B¸6Ð BÑ BÔ BÐSÐSÐSr„ Ú
|
|||
|
|
trainer_statec ó" — g | ] \ }}d ||z i‘Œ
|
|||
|
|
S )Úmessagesr ©r ÚpÚcs r‚ r z:_UnslothGRPOTrainer._calculate_rewards.<locals>.<listcomp>* s% € Ð#^Ð#^Ð#^¹D¸A¸q Z°°Q±Ð$7Ð#^Ð#^Ð#^r„ c ó< •— g | ]}t |‰¦ « d ‘ŒS )Útext)r& )r Úxr” s €r‚ r z:_UnslothGRPOTrainer._calculate_rewards.<locals>.<listcomp>+ s, ø€ Ð kÐ kÐ kÐ]^Õ!4°QÐ8OÑ!PÔ!PÐQWÔ!XÐ kÐ kÐ kr„ c ó — g | ]
|
|||
|
|
\ }}||z ‘Œ S r r r s r‚ r z:_UnslothGRPOTrainer._calculate_rewards.<locals>.<listcomp>- s € Ð MÐ MÐ M©4¨1¨a Q¡Ð MÐ MÐ Mr„ ÚptTÚrightF©r Úreturn_tensorsÚpaddingÚ padding_sideÚadd_special_tokens)ÚpromptsÚ completionsr
|
|||
|
|
c ó. — g | ]}|�|n t j ‘ŒS rþ )rS Únan)r Úrewards r‚ r z:_UnslothGRPOTrainer._calculate_rewards.<locals>.<listcomp>9 s$ € Ð)wÐ)wÐ)wÐZ`°FÐ4F¨&¨&ÍEÌIÐ)wÐ)wÐ)wr„ ©r rÒ rm rl )Úas_tuplec ó( •— i | ]\ }}||‰ “ŒS r r )r r ÚvalueÚ nan_row_idxs €r‚ r z:_UnslothGRPOTrainer._calculate_rewards.<locals>.<dictcomp>@ s$ ø€ Ð aÐ aÐ a¹Z¸SÀ% e¨KÔ&8Ð aÐ aÐ ar„ r9 r: z=All reward functions returned None for the following kwargs: zI. Please ensure that at least one reward function returns a valid reward.r ) r* rÒ rS rë r r Ústaterg rq r rf rI r, rA rh r4 r r
|
|||
|
|
r¦ ry ri rs ÚisnanÚallrm Únonzerorô rW Úwarnr. )r
|
|||
|
|
r r$ r% Úcompletion_ids_listrÒ Úrewards_per_funcr^ Ú
|
|||
|
|
reward_kwargsr
|
|||
|
|
r“ Úreward_func_namer ÚtextsÚ
|
|||
|
|
reward_inputsÚoutput_reward_funcÚrow_reward_kwargsr- r” r s ` @@€r‚ Ú_calculate_rewardsz&_UnslothGRPOTrainer._calculate_rewards s øøøø€ àÔ!Ô(ˆÝ œ;¥s¨7¡|¤|µS¸Ô9JÑ5KÔ5KÐTZÐ[Ñ[Ô[Ðð cÐb˜v aœyÐbÑbÔbˆØSÐSÐSÐSÈdÐSÑSÔSˆ
|
|||
|
|
ð *.¬ˆ
|
|||
|
|
�oÑ&åKTÝ �Ô! 4Ô#AÀ4ÔCYÑ ZÔ ZñL
|
|||
|
|
ô L
|
|||
|
|
ð rñ rÑ GˆAÑG� Ð 4Ð6Fõ # 4Ð)9Ñ:Ô:ð
|
|||
|
|
rð
|
|||
|
|
rÝ ˜k2¬9Ñ5Ô5ð rÝ(¨°¬Ñ3Ô3ð NØ#^Ð#^ÅCÈÐQ\ÑD]ÔD]Ð#^Ñ#^Ô#^˜Ø kÐ kÐ kÐ kÐbjÐ kÑ kÔ k˜˜à MÐ Mµ3°wÀ Ñ3LÔ3LÐ MÑ MÔ M˜Ø$;Ð$;Ø "°4ÀÐT[Ðpuð%ñ %ô %�Mõ %*¡G¤G×$;Ò$;¸MÑ$JÔ$J�MÝ Ô-Ñ/Ô/ð [ð [Ø1<°Ð1MÐ1M¸}Ð1MÐ1MÔ1TÐUVÐUVÐUVÐXYÐUYÔ1ZÐ(¨¨¨¨A¨Ñ.ð[ð [ð [ñ [ô [ð [ð [ð [ð [ð [ð [øøøð [ð [ð [ð [øð *5¨ð *Ø '°[ÐQdð*ð *Øhuð*ð *Ð&ð *xÐ)wÐdvÐ)wÑ)wÔ)wÐ&å-2¬\Ð:LÕTYÔTaÐjpÐ-qÑ-qÔ-qÐ$ Q Q Q¨ TÑ*ð)
|
|||
|
|
rð
|
|||
|
|
rð
|
|||
|
|
rñ
|
|||
|
|
rô
|
|||
|
|
rð
|
|||
|
|
rð
|
|||
|
|
rð
|
|||
|
|
rð
|
|||
|
|
rð
|
|||
|
|
rð
|
|||
|
|
røøøð
|