Files
DS-LLM-TEMPLATE-FINETUNING/unsloth_compiled_cache/__pycache__/UnslothGRPOTrainer.cpython-311.pyc
T

918 lines
146 KiB
Plaintext
Raw Normal View History

2025-08-13 23:50:20 +00:00
§
4$h¼]ãódZddlmZddlZddlmZddlmZddlmZm Z m
Z
m Z m Z m
Z
mZmZddlmZmZmZmZmZmZmZmZmZmZmZmZm
Z
mZmZmZmZm Z m!Z!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z'm Z m(Z(m)Z)m*Z*m+Z+m,Z,m-Z-m.Z.m/Z/m0Z0m1Z1m2Z2m3Z3m4Z4m5Z5m6Z6m7Z7m8Z8m9Z9m:Z:m;Z;m<Z<m=Z=m>Z>m?Z?m@Z@mAZAmBZBmCZCmDZDmZmEZEmFZFmGZGmHZHmIZImJZJmKZKmLZLmMZMmNZNmOZOmPZPmQZQmRZRmSZSmTZTmUZUmZmVZVmWZWmXZXmYZYmZmZm Z m*Z*m+Z+m,Z,m2Z2m3Z3m8Z8m:Z:mAZAmDZDmEZEmFZFmGZGmLZLmNZNmZmVZVmXZXm1Z1mFZFmNZNmPZPmZmVZVmNZNmZm Z mFZFmMZMmNZNmRZRmSZSmTZTmZmWZWm
Z
mZm&Z&m<Z<mFZFmNZNmZmZmZmFZFmNZNmZmZmNZNmZmZm&Z&m2Z2mFZFmNZNmZddlFZFddlTddlZm[Z[m\Z\dd l]m^Z^ddlZddl_Z`dd
lamEZEddlmZdd lVmbZbmcZdd d
d d
d
dœZeejfd d ee¬¦«d¦«ZgdZhGddejijj¦«Zk ddZlejfd d ee¬¦«d¦«ZmdZne[Gdde¦«¦«Zo Gdde&¦«ZpGddep¦«ZqdS)z8
2025.8.4
2025.8.5
4.55.1
0.21.0
__UNSLOTH_VERSIONING__
é)ÚTensorN)Ú
functional)ÚAnyÚListÚOptionalÚTupleÚUnionÚDictÚSetÚCallable)ˆrÚ
AutoConfigÚ"AutoModelForSequenceClassificationÚ
AutoProcessorÚ
AutoTokenizerÚ
DataLoaderÚDatasetÚFSDPÚ
GRPOConfigÚ GRPOTrainerÚGenerationConfigÚIterableDatasetrÚPathÚ
PeftConfigÚPreTrainedModelÚPreTrainedTokenizerBaseÚProcessorMixinÚ
RepeatSamplerÚ
RewardFuncÚSamplerÚSequenceÚSyncRefModelCallbackÚTrainerÚTrainerCallbackr Ú
VLLMClientÚ_ForwardRedirectionÚapply_chat_templateÚbroadcast_object_listÚcopyÚdatasetsÚ defaultdictÚdequeÚdisable_dropout_in_modelÚentropy_from_logitsÚgatherÚ
gather_objectÚgenerate_model_cardÚget_comet_experiment_urlÚidentityÚinspectÚis_conversationalÚis_datasets_availableÚis_flash_attn_2_availableÚis_liger_kernel_availableÚis_peft_availableÚ
is_peft_modelÚis_rich_availableÚis_vllm_availableÚis_wandb_availableÚmaybe_apply_chat_templateÚnanmaxÚnanminÚnanstdÚnnÚ nullcontextÚosÚpadÚpartialÚprepare_deepspeedÚ prepare_fsdpÚprint_prompt_completions_sampleÚprofiling_contextÚprofiling_decoratorÚreÚ seed_workerÚselective_log_softmaxÚset_seedÚshuffle_sequence_dictÚsplit_pixel_values_by_gridÚsplit_tensor_dictÚtextwrapÚtorchÚ transformersÚunsplit_pixel_values_by_gridÚunwrap_model_for_generationÚwarningsrrr r&r'r(r.r/r4r6r=r@rBrCrDrIrKrSrTrVr-rCrKrMrSrTrKrr rCrJrKrOrPrQrSrUrrr"r8rCrKrSrrArCrKrrArKrSrr"r.rCrKrS)Ú*)Ú dataclassÚfield)ÚVersion)rB)ÚDataCollatorForSeq2SeqÚDataCollatorForLanguageModelingTF)Úepilogue_fusionÚ max_autotuneÚ
shape_paddingz
trace.enabledztriton.cudagraphs)ÚdynamicÚ fullgraphÚoptionscó’tj| d|jd¦«dd¬¦«}tj| d¦«dd¬¦«}g}t ||¦«D]\}}| tj¦«}tj|d| d¦«¬¦«  d¦«}tj
|d¬¦«}||z
} |  | ¦«Œ’ tj |¦«}| |jd|jdf¦«}|S)Néÿÿÿÿér©ÚchunksÚdim©riÚindex©rié)
rSÚchunkÚreshapeÚshapeÚzipÚtoÚfloat32r.Ú unsqueezeÚsqueezeÚ logsumexpÚappendÚconcat)
ÚlogitsrkÚchunked_logitsÚ
chunked_indexÚall_per_token_logpsÚ chunk_logitsÚ chunk_indexÚselected_logitsÚlogsumexp_valuesÚper_token_logpss
ú^/workspace/Fine-tuning/DS-LLM-TEMPLATE-FINETUNING/unsloth_compiled_cache/UnslothGRPOTrainer.pyÚchunked_selective_log_softmaxrƒ"s5õ”[ §¢°°F´LÀÔ4DÑ!EÔ!EÐPQÐYZÐ[€NÝ”[ §¢¨rÑ!2Ô!2¸QÀaÐH€MØÐå%(¨¸Ñ%GÔ%Gð #—¥u¤}Ñ Ýœ, |¸2À{×G\ÒG\Ð]_ÑG`ÔG`Ða×iÐjlÑmˆÝ œ?¨<¸Ø)Ð,<Ñ<ˆØ×" Ýœ,Ð':ÑØ-×5°v´|ÀA´ÈÌ ÐUVÌÐ6XÑØ Ðóc óL | dd¦«}| dd¦«} | dd¦«}
| dd¦«} | dd¦«} | d d
¦«}
| d d ¦«}| d
d ¦«}| dd ¦«}| d¦«}|dkr||z}|dkr||z }|dkr|tj||z ¦«z}| tj¦«}|
d
kr||
z }tj|d|¬¦« d¦«}|tj|d¬¦«z
}tj ¦«5|d kr¯|
Jd¦«|dkr||z}|dkr||z }|dkr|tj||z ¦«z}| tj¦«}|
d
kr||
z }tj|d|¬¦« d¦«}|tj|d¬¦«z
} |£|dkr||z}|dkr||z }|dkr|tj||z ¦«z}| tj¦«}|
d
kr||
z }tj|d|¬¦« d¦«}|tj|d¬¦«z
} ddd¦«n #1swxYwY |d kr!tj
||z
¦«||z
z
d
z
}nd }|tj
||z
¦«}n)tj
||  ¦«z
¦«}tj |d| z
d|
z¦«}| -tj || ¬¦«| d¦«z}n|| d¦«z} || d¦«z}tj
||¦« }|d kr|||zz}| tj¦«}| d¦«}|dkrU||z d¦«| d¦«  d
¬¦«z  ¦«}n|dkrA||z ¦«| ¦«  d
¬¦«z }nI|dkr1||z ¦«| d¦«| zz }nt#d|¦«tj¦«5| ¦«}||z d¦«|z } |  ¦«}!ddd¦«n #1swxYwY |||!fS©NÚ loss_typeÚgrpoÚ epsilon_lowçš™™™™™É?Ú epsilon_highÚmax_completion_lengthi ÚdeltaÚ temperatureçð?Úlogit_scale_multiplyçÚlogit_scale_divideÚlogit_softcappingrerrjrlz.ref_logits should not be None when beta != 0.0rm©Úmax©ÚminÚbnpoÚdr_grpoúUnknown loss type: ©ÚgetrtrSÚtanhrrrsr.rurvÚno_gradÚexpÚdetachÚclampr—ÚsumÚmeanÚsizeÚ
ValueErrorÚinference_mode©"Ú
ref_logitsÚ
new_logitsÚ
old_logitsÚ input_idsÚmaskÚbetaÚ
advantagesÚkwargsr‡r‰rrrrr“Únew_xÚnewÚref_xÚrefÚold_xÚoldÚkl_iÚcoef_1Úcoef_2Úloss_1Úloss_2Úloss_iÚn_mask_per_rewardÚlossÚcompletion_lengthÚmean_kl_per_rewardÚmean_kls" rÚgrpo_compute_lossrÁ4sNð
˜;¨Ñ/€IØ—*’*˜]¨CÑ0€KØ—::˜n¨cÑ2€LØ"ŸJšJÐ'>ÀÑØ JŠJÑ %€EØ—*’*˜]¨CÑ0€KØ!Ÿ:š:Ð&<¸Ø!Ÿ:š:Ð&:¸Ø!Ÿ:š:Ð&9¸à×# '€Ið˜ Ð ¨zÐ<PÑ/P ˜ Ð ¨zÐ<NÑ/N *ؘ Ð ¨z½E¼JÀzÐTeÑGeÑ<fÔ<fÑ/f uœ}Ñ-€JàШ
°[Ñ(@˜:Ý ŒL˜¨2°yÐ A× IÈ"Ñ M€EØ
•%”/ *°BÐ
7€Cõ
ŒŒð
ð
Ø 3Š;ˆ;ØÐ)Ð+[Ñ$ (°zÐDXÑ7X¨*Ø! (°zÐDVÑ7V¨*Ø  (°zÅEÄJÈzÐ\mÑOmÑDnÔDnÑ7n¨*àš¥u¤}Ñ5ˆJà˜cÒ
¸[Ñ0H :Ý”L °2¸QÐRTÑUˆ%œ/¨*¸BÐ?ˆCØ à Ð # qÒ(°zÐDXÑ7X¨*Ø! (°zÐDVÑ7V¨*Ø  (°zÅEÄJÈzÐ\mÑOmÑDnÔDnÑ7n¨*àš¥u¤}Ñ5ˆJà˜cÒ
¸[Ñ0H :Ý”L °2¸QÐRTÑUˆ%œ/¨*¸BÐ?ˆ ð7
ð
ð
ñ
ô
ð
ð
ð
ð
ð
ð
øøøð
ð
ð
ð
ð8 ð ˆs‚{€{ÝŒy˜˜s™Ñ# s¨S¡yÑ1°Cшðˆð Ðݘ3 ™9шå˜3 §¢¡¤ÑÝ
Œ[˜  [¡°!°lÑ2BÑ
C€Fà ÐݘV¨Ð/°*×2FÒ2FÀqÑ2IÔ2IÑIˆˆà˜*×.¨qÑØð *¨1Ñ
-€FÝŒi˜ Ñ
'€FØ ˆs{€{ؘ$ ™+Ñà 7Š7•5”=Ñ !€DØŸš  œ ÐðÐØ˜$×# '¨$¯(ª(°2©,¬,×*<Ò*<ÀÐ*<Ñ*EÔ*EÑE×ˆØ Ð Ø˜
×$ t§x¢x¡z¤z×'7Ò'7¸CÐ'7Ñ'@Ô'@Ñ@ˆˆØ Рؘ
ׯ ª °A©¬Ð9NÑ(NшåÐ:¨yÐ
Ô Ñ Ô ð-ר" T™k×.¨qÑ1Ð4EÑØð,øøøð à Ð"  +ó&ÆEK;Ë;K?ÌK?ÕAVÖVÖVcó>eZdZedd¦«Z ed¦«ZdS)ÚUnslothEfficientGRPONrmc óô ‡ ‡!— iŠ ˆˆ ˆfdŠ! |j} tj|¦«}
tjd| ¬¦«Štjd| ¬¦«Štjd| ¬¦«Š ˆˆˆ ˆ!fd} tj|ddt
¬¦«}tj|
|
d¬¦«}tj||
d¬¦«}|tj||
d¬¦«}ndg|
z}tj||
d¬¦«}tj||
d¬¦«}tj||
d¬¦«}tj||
d¬¦«}| |  ¦«nd }t|||||||¦«D]\}}}}}}}|||||||||¦«Œ |
  |
¦«  |
¦«  |
¦«‰   |
¦«| 
|
¦«‰ fS)
Nc ó0tj| ¦«¦«}|ddddddf}tj¦«5dkr9tj| ¦«¦«}|ddddddf}nd}|9tj| ¦«¦«} | ddddddf} nd} ddd¦«n #1swxYwYt ||| |||fi¤Ž\}
} } |
|z}
|
|
 ¦«| | ffS)Nrer)rSÚmatmulÚtržr )Únew_hidden_statesÚold_hidden_statesÚref_hidden_statesr«Úscalingr©Ú scaled_lossr­Ú extra_kwargsÚlm_heads €€€rÚ compute_lossz2UnslothEfficientGRPO.forward.<locals>.compute_lossµs‡ø€ÝœÐ&7¸¿º¹¼ÑEˆ# A A A  s¨A¨A¨A .ˆJÝð
˜3;;Ý!&¤Ð.?ÀÇÂÁÄÑ!MÔ!MJØ!+¨A¨A¨A¨s°¨s°A°A°A¨IÔ!6JJà!%0Ý!&¤Ð.?ÀÇÂÁÄÑ!MÔ!MJØ!+¨A¨A¨A¨s°¨s°A°A°A¨IÔ!6JJà!%
&øøøð
&õ"0AØØØØØØØð 0ð 0ðð 0ð 0Ñ ,ˆ# Wð ™.ˆKà §¢¡¤Ð0AÀ7Ð LÐ LsÁ
A?CÃCÃCrm©Údevicec ó tj dd¬¦«|||||||¦«\\}\} \}
} }  |
¦«
 | ¦« | ¦«||dd<dS)rT)ÚargnumsÚhas_aux)rSÚfuncÚgrad_and_valueÚadd_)Únew_hidden_states_jÚold_hidden_states_jÚref_hidden_states_jÚ input_ids_jÚmask_jÚ advantages_jrÌÚ
grad_inputs_jÚchunk_grad_inputÚ
chunk_lossÚ
unscaled_lossÚchunk_completion_lengthÚ
chunk_mean_klÚaccumulated_completion_lengthÚaccumulated_lossÚaccumulated_mean_klrÐs €€€€rÚaccumulate_chunkz6UnslothEfficientGRPO.forward.<locals>.accumulate_chunkàø€õkpÔjt÷kDòkDØØØðkDñkôkð"Ð#6Ð8KÈ[ÐZ`ÐbnÐpwñ kyôkyÑ Ð
Ñ!g *Ñ.f¨}Ð>UÐWdð
× .¨}Ñ .Ð/FÑ × .¨}Ñ /ˆM˜!˜!˜!Ñ Ð Ð r„T)rbrarcrrgr) rSÚ
empty_likeÚzerosÚcompileÚtorch_compile_optionsrnÚ get_scalerqÚdiv_Úsave_for_backward)"ÚctxÚ_new_hidden_statesÚ_old_hidden_statesÚ_ref_hidden_statesrÏÚ
_input_idsÚ_maskÚ _advantagesr­ÚscalerÚn_chunksrÎÚ grad_inputsréÚgrad_inputs_chunksrÉs" ` ` ` @@@@rÚforwardzUnslothEfficientGRPO.forward±øøøøøøø€à Р؈Lð" Mð" Mð" Mð" Mð" Mð" Mð" MðF
à)ˆÝÔ&Ð'9Ñ:ˆ Ý(-¬ °AÀÐ(GÑ(GÔ(GÐÝ(-¬ °AÀÐ(GÑ(GÔ(GÐ%Ý(-¬ °AÀÐ(GÑ(GÔ(GÐð 0ð&
å œ=Ø ØàÝ 
ñ
ô
Ðõ#œ[¨ÀhÐVWÐÝ"œ[Ð);ÀhÐVWÐØ Ð )Ý!&¤Ð-?È(ÐZ[Ð!\Ñ!\Ô!\Ð Ð à!% ¨Ñ 1Ð Ý"œ[Ð);ÀhÐVWÐÝ"œ[¨ÀhÐVWÐXˆ Ý"œ[¨ÀhÐVWÐXˆÝ"œ[¨ÀhÐVWÐXˆ
ð)/Ð(:&׈õ
Ð"Ð$5Ð7HÐJ[Ð]fÐhlÐnxÑ  ð Ñ ˆ/Ð1DÐFYÐ\gÐioÐq}ð
Ð ØØØØØñ
ô
ð
ð
ð
à×*¨8Ñ×*¨8Ñ*¨8Ñ×*¨8Ñ ×Ò˜kÑ Ø ð
ð
r„c ó.|j\}|ddddddddddf S©N)Ú
saved_tensors)Ú grad_outputÚdcompletion_lengthÚdmean_klÚ
grad_inputs rÚbackwardzUnslothEfficientGRPO.backward2s*àÔ)‰
ˆØ˜D $¨¨d°D¸$ÀÀdÈDÐRVÐWr„)NrmN)Ú__name__Ú
__module__Ú __qualname__Ú staticmethodrür©r„r¯sUàð}
ð}
ð}
ñ„\ð}
ð| àðXðXñ„\ðXð €Dr„rec óþ|j\Š} ˆfdtddz¦«D¦«}
|dkr}|
ttj|
|¦«t |
¦«dz
¦«}t
|d¦«svtj  dd¦«dkr tj n tj |_
tj  dd¦«d krtj |_
d tjd
<|dd| df} |j ¦«j} tj |jjj|j
¬ ¦«5tj¦«5|j |jd ¬
¦« ¦«5| |||dz¬¦«j}
ddd¦«n #1swxYwYddd¦«n #1swxYwY | |||dz¬¦«j}t4 |||
| | |||j|jj||¦ « \}}}ddd¦«n #1swxYwY dtjd
<|||fS)Ncó&g|]
}|zdk¯ |ŒSr )Ú.0Úbszs €rú
<listcomp>z)grpo_accumulated_loss.<locals>.<listcomp>Gs"ø€Ð<Q¨s°Q©w¸!ª|¨|ˆq¨|¨|¨|r„rmreÚ_autocast_dtypeÚACCELERATE_MIXED_PRECISIONÚfp16ÚUNSLOTH_FORCE_FLOAT32ÚUNSLOTH_RETURN_HIDDEN_STATES)Ú device_typeÚdtypeF)Úkeep_fp32_wrapper)Úattention_maskÚlogits_to_keep)!rpÚranger—ÚnpÚ searchsortedÚlenÚhasattrrCÚenvironrœrSÚfloat16Úbfloat16rÚmodelÚget_output_embeddingsÚweightÚampÚautocastrÒÚtyper¦Ú acceleratorÚ unwrap_modelÚdisable_adapterryÚapplyr­)Útrainerr«rrÚcompletion_maskr®ÚqlenÚfactorsÚcompletion_input_idsrÏrs @rÚgrpo_accumulated_lossr38sNø€ðI€Cˆð<%  ¡7Ñ<€GØ2~€~ #•s2œ?¨7°HÑ=½sÀ7¹|¼|ÈA¹~ÑO€Hå  hÝ35´:·>²>ÐB^Ð`fÑ3gÔ3gÐkqÒ3qÐ3q¥%¤- -Õw|ôxFˆÔÝ
Œ:>Š>Ð1°3Ñ 7¸ >ÕZ_ÔZgÀÔ@WØØ14…B„JÐ$ Q Q ¨Ð(8Ð(8Ð%8ÔØŒm×:€Gå Œ× Ò ¨'¬-Ô*>Ô*CÈWÔMdÐ Ñ eÔ eð
ð
Ý
Ô
 ð  WÔ%8×%EÒ%EÀgÄmÐinÐ%EÑ%oÔ%o×%Ò%ñ&Bô&Bð ð Ø '§
¢
Ø%Ø!/Ø!/°!Ñ!3ð!.ñ!ô!ôð
ð ð ð ñ ô ð ð ð ð ð ð øøøð ð ð ð ð ð ð ñ ô ð ð ð ð ð ð øøøð ð ð ð ð
Ø#ŸMšMØ+¨aÑ
ô
ô ð õ ,@×+EÒ+EØ Ø Ø Ø Ø Ø Ø Ø ŒLØ Ô Ô Ø ñ ,
ô ,
Ñ(ˆÐ ð
ð
ð
ñ
ô
ð
ð
ð
ð
ð
ð
øøøð
ð
ð
ð
ð6 à14…B„JÐ Ð"  +s[Å$IÅ84G0Æ,!GÇ
G0ÇG ÇG0Ç G Ç!G0Ç$ IÇ0G4 Ç4IÇ7G4 Ç8AIÉI#É&I#c óL | dd¦«}| dd¦«} | dd¦«}
| dd¦«} | dd¦«} | d d
¦«}
| d d ¦«}| d
d ¦«}| dd ¦«}| d¦«}|dkr||z}|dkr||z }|dkr|tj||z ¦«z}| tj¦«}|
d
kr||
z }tj|d|¬¦« d¦«}|tj|d¬¦«z
}tj ¦«5|d kr¯|
Jd¦«|dkr||z}|dkr||z }|dkr|tj||z ¦«z}| tj¦«}|
d
kr||
z }tj|d|¬¦« d¦«}|tj|d¬¦«z
} |£|dkr||z}|dkr||z }|dkr|tj||z ¦«z}| tj¦«}|
d
kr||
z }tj|d|¬¦« d¦«}|tj|d¬¦«z
} ddd¦«n #1swxYwY |d kr!tj
||z
¦«||z
z
d
z
}nd }|tj
||z
¦«}n)tj
||  ¦«z
¦«}tj |d| z
d|
z¦«}| -tj || ¬¦«| d¦«z}n|| d¦«z} || d¦«z}tj
||¦« }|d kr|||zz}| tj¦«}| d¦«}|dkrU||z d¦«| d¦«  d
¬¦«z  ¦«}n|dkrA||z ¦«| ¦«  d
¬¦«z }nI|dkr1||z ¦«| d¦«| zz }nt#d|¦«tj¦«5| ¦«}||z d¦«|z } |  ¦«}!ddd¦«n #1swxYwY |||!fSr†rs" rÚgrpo_compute_loss_slowr5sNð
˜;¨Ñ/€IØ—*’*˜]¨CÑ0€KØ—::˜n¨cÑ2€LØ"ŸJšJÐ'>ÀÑØ JŠJÑ %€EØ—*’*˜]¨CÑ0€KØ!Ÿ:š:Ð&<¸Ø!Ÿ:š:Ð&:¸Ø!Ÿ:š:Ð&9¸à×# '€Ið˜ Ð ¨zÐ<PÑ/P ˜ Ð ¨zÐ<NÑ/N *ؘ Ð ¨z½E¼JÀzÐTeÑGeÑ<fÔ<fÑ/f uœ}Ñ-€JàШ
°[Ñ(@˜:Ý ŒL˜¨2°yÐ A× IÈ"Ñ M€EØ
•%”/ *°BÐ
7€Cõ
ŒŒð
ð
Ø 3Š;ˆ;ØÐ)Ð+[Ñ$ (°zÐDXÑ7X¨*Ø! (°zÐDVÑ7V¨*Ø  (°zÅEÄJÈzÐ\mÑOmÑDnÔDnÑ7n¨*àš¥u¤}Ñ5ˆJà˜cÒ
¸[Ñ0H :Ý”L °2¸QÐRTÑUˆ%œ/¨*¸BÐ?ˆCØ à Ð # qÒ(°zÐDXÑ7X¨*Ø! (°zÐDVÑ7V¨*Ø  (°zÅEÄJÈzÐ\mÑOmÑDnÔDnÑ7n¨*àš¥u¤}Ñ5ˆJà˜cÒ
¸[Ñ0H :Ý”L °2¸QÐRTÑUˆ%œ/¨*¸BÐ?ˆ ð7
ð
ð
ñ
ô
ð
ð
ð
ð
ð
ð
øøøð
ð
ð
ð
ð8 ð ˆs‚{€{ÝŒy˜˜s™Ñ# s¨S¡yÑ1°Cшðˆð Ðݘ3 ™9шå˜3 §¢¡¤ÑÝ
Œ[˜  [¡°!°lÑ2BÑ
C€Fà ÐݘV¨Ð/°*×2FÒ2FÀqÑ2IÔ2IÑIˆˆà˜*×.¨qÑØð *¨1Ñ
-€FÝŒi˜ Ñ
'€FØ ˆs{€{ؘ$ ™+Ñà 7Š7•5”=Ñ !€DØŸš  œ ÐðÐØ˜$×# '¨$¯(ª(°2©,¬,×*<Ò*<ÀÐ*<Ñ*EÔ*EÑE×ˆØ Ð Ø˜
×$ t§x¢x¡z¤z×'7Ò'7¸CÐ'7Ñ'@Ô'@Ñ@ˆˆØ Рؘ
ׯ ª °A©¬Ð9NÑ(NшåÐ:¨yÐ
Ô Ñ Ô ð-ר" T™k×.¨qÑ1Ð4EÑØð,øøøð à Ð"  +rÂc ó0ddlm}|di|¤Ž}||_|S)Nr)ÚSamplingParamsr )Úvllmr7Ú _set_kwargs)r7Úsampling_paramss rÚvLLMSamplingParamsr;s7Ø$. .€OØ"(€OÔØ Ðr„c¯óìeZdZUdZedddi¬¦«Zeeed<edddi¬¦«Z ee
ed <ddd
d
d
d d
d d ddd
d
ddddddddddddddddddd
dd
ddddd
d
d
d
d
d
ddd
d
d
d
d d!d
d
ddddd
d"d
ddddddd
dd
ddd
d"dddddd#d$dd
d
d%dddddd
dd
d
ddd&ddd
dd
dd
dd!dddd"dd
dd'd(d
ddd
d
ddd
d
d
dd
ddd
d)d*d+ddddddddidd
dd
dd,d-dd.d/d0d1dd2dd3ddd4ddd5d
d
d6d)dd
d
dd
ddf¯ˆfd7„ Z ˆxZ S)8ÚUnslothGRPOConfiguâ<
Configuration class for the [`GRPOTrainer`].
This class includes only the parameters that are specific to GRPO training. For a full list of training arguments,
please refer to the [`~transformers.TrainingArguments`] documentation. Note that default values in this class may
differ from those in [`~transformers.TrainingArguments`].
Using [`~transformers.HfArgumentParser`] we can turn this class into
[argparse](https://docs.python.org/3/library/argparse#module-argparse) arguments that can be specified on the
command line.
Parameters:
> Parameters that control the model and reference model
model_init_kwargs (`str`, `dict[str, Any]` or `None`, *optional*, defaults to `None`):
Keyword arguments for [`~transformers.AutoModelForCausalLM.from_pretrained`], used when the `model`
argument of the [`GRPOTrainer`] is provided as a string.
disable_dropout (`bool`, *optional*, defaults to `False`):
Whether to disable dropout in the model. This is useful for training with a reference model, as it prevents
the model from generating different logprobs for the same input.
> Parameters that control the data preprocessing
remove_unused_columns (`bool`, *optional*, defaults to `False`):
Whether to only keep the column `"prompt"` in the dataset. If you use a custom reward function that
requires any column other than `"prompts"` and `"completions"`, you should keep this to `False`.
max_prompt_length (`int` or `None`, *optional*, defaults to `512`):
Maximum length of the prompt. If the prompt is longer than this value, it will be truncated left.
num_generations (`int` or `None`, *optional*, defaults to `8`):
Number of generations per prompt to sample. The effective batch size (num_processes * per_device_batch_size
* gradient_accumulation_steps) must be evenly divisible by this value.
max_completion_length (`int` or `None`, *optional*, defaults to `256`):
Maximum length of the generated completion.
ds3_gather_for_generation (`bool`, *optional*, defaults to `True`):
This setting applies to DeepSpeed ZeRO-3. If enabled, the policy model weights are gathered for generation,
improving generation speed. However, disabling this option allows training models that exceed the VRAM
capacity of a single GPU, albeit at the cost of slower generation. Disabling this option is not compatible
with vLLM generation.
shuffle_dataset (`bool`, *optional*, defaults to `True`):
Whether to shuffle the training dataset.
> Parameters that control generation
generation_batch_size: (`int` or `None`, *optional*, defaults to `None`):
Batch size to use for generation. If `None`, it defaults to the effective training batch size:
`per_device_train_batch_size * num_processes * steps_per_generation`. In other words, there is one
generation batch processed per optimization step. Mutually exclusive with `steps_per_generation`.
steps_per_generation: (`int` or `None`, *optional*, defaults to `None`):
Number of steps per generation. If `None`, it defaults to `gradient_accumulation_steps`. Mutually exclusive
with `generation_batch_size`.
temperature (`float`, defaults to `1.0`):
Temperature for sampling. The higher the temperature, the more random the completions.
top_p (`float`, *optional*, defaults to `1.0`):
Float that controls the cumulative probability of the top tokens to consider. Must be in (0, 1]. Set to
`1.0` to consider all tokens.
top_k (`int` or `None`, *optional*, defaults to `None`):
Number of highest probability vocabulary tokens to keep for top-k-filtering. If `None`, top-k-filtering is
disabled and all tokens are considered.
min_p (`float` or `None`, *optional*, defaults to `None`):
Minimum token probability, which will be scaled by the probability of the most likely token. It must be a
value between `0.0` and `1.0`. Typical values are in the `0.01-0.2` range.
repetition_penalty (`float`, *optional*, defaults to `1.0`):
Float that penalizes new tokens based on whether they appear in the prompt and the generated text so far.
Values > `1.0` encourage the model to use new tokens, while values < `1.0` encourage the model to repeat
tokens.
use_transformers_paged (`bool`, *optional*, defaults to `False`):
Whether to use the `transformers` paged implementation for generation. If set to `True`, the `transformers`
paged implementation will be used for generation instead of the default padded implementation. This
parameter is only effective when `use_vllm` is set to `False`.
cache_implementation (`str` or `None`, *optional*, defaults to `None`):
Implementation of the cache method for faster generation when `use_vllm` is set to `False`.
generation_kwargs (`dict[str, Any]` or `None`, *optional*, defaults to `None`):
Additional keyword arguments to pass to `GenerationConfig` (if using transformers) or `SamplingParams` (if
using vLLM) when sampling completions. This can be used to further customize the generation behavior, such
as setting `supress_tokens`, `num_beams`, etc. If it contains keys that conflict with the other generation
parameters (like `min_p`, `top_p`, etc.), they will override them.
> Parameters that control generation acceleration powered by vLLM
use_vllm (`bool`, *optional*, defaults to `False`):
Whether to use vLLM for generating completions. If set to `True`, the trainer will use vLLM for generation
instead of the default model.generate(). Requires `vllm` to be installed.
vllm_mode (`str`, *optional*, defaults to `"server"`):
Mode to use for vLLM integration when `use_vllm` is set to `True`. Must be one of `"server"` or
`"colocate"`.
- `"server"`: The trainer will send generation requests to a separate vLLM server. Make sure a TRL vLLM
server is running (start with `trl vllm-serve`).
- `"colocate"`: vLLM will run in the same process and share the training GPUs. This avoids the need for a
separate server but may cause resource contention with training.
vllm_guided_decoding_regex (`str` or `None`, *optional*, defaults to `None`):
Regex for vLLM guided decoding. If `None` (default), guided decoding is disabled.
> Parameters that control the vLLM server (only used when `vllm_mode` is `"server"`)
vllm_server_base_url (`str` or `None`, *optional*, defaults to `None`):
Base URL for the vLLM server (e.g., `"http://localhost:8000"`). If provided, `vllm_server_host` and
`vllm_server_port` are ignored.
vllm_server_host (`str`, *optional*, defaults to `"0.0.0.0"`):
Host of the vLLM server to connect to. Ignored if `vllm_server_base_url` is provided.
vllm_server_port (`int`, *optional*, defaults to `8000`):
Port of the vLLM server to connect to. Ignored if `vllm_server_base_url` is provided.
vllm_server_timeout (`float`, *optional*, defaults to `240.0`):
Total timeout duration in seconds to wait for the vLLM server to be up. If the server is not up after the
timeout, a `ConnectionError` is raised.
> Parameters that control colocated vLLM execution (only used when `vllm_mode` is `"colocate"`)
vllm_gpu_memory_utilization (`float`, *optional*, defaults to `0.3`):
Control the GPU memory utilization for vLLM. This setting only applies when `vllm_mode` is set to
`"colocate"`. If you are using `vllm_mode="server"`, this parameter must be passed separately when
launching the vLLM server via the `--vllm_gpu_memory_utilization` flag.
vllm_tensor_parallel_size (`int`, *optional*, defaults to `1`):
Control the tensor parallel size for vLLM. This setting only applies when `vllm_mode` is set to
`"colocate"`. If you are using `vllm_mode="server"`, this parameter must be passed separately when
launching the vLLM server via the `--vllm_tensor_parallel_size` flag.
vllm_model_impl (`str`, *optional*, defaults to `"vllm"`):
Model implementation to use for vLLM. Must be one of `"transformers"` or `"vllm"`. `"transformers"`: Use
the `transformers` backend for model implementation. `"vllm"`: Use the `vllm` library for model
implementation.
> Parameters that control the training
beta (`float`, *optional*, defaults to `0.0`):
KL coefficient. If `0.0` (default), the reference model is not loaded, reducing memory usage and improving
training speed.
num_iterations (`int`, *optional*, defaults to `1`):
Number of iterations per batch (denoted as μ in the algorithm).
epsilon (`float`, *optional*, defaults to `0.2`):
Epsilon value for clipping.
delta: (`float` or `None`, *optional*, defaults to `None`):
Enables the upper clipping bound in two-sided GRPO loss when set to a float. If `None` (default), standard
GRPO clipping is used. Recommended to be greater than `1 + ε` when enabled. This method is introduced in
the [INTELLECT-2 tech report](https://huggingface.co/papers/2505.07291).
epsilon_high (`float` or `None`, *optional*, defaults to `None`):
Upper-bound epsilon value for clipping. If not specified, it defaults to the same value as the lower-bound
specified in argument `epsilon`. Paper [DAPO](https://huggingface.co/papers/2503.14476) recommends `0.28`.
importance_sampling_level (`str`, *optional*, defaults to `"token"`):
Controls whether importance sampling ratios are computed at the `"token"` or `"sequence"` level. `"token"`
keeps the raw per-token log-probability ratios (one weight per token). `"sequence"` averages the
log-probability ratios across valid tokens to produce a single ratio per sequence. The
[GSPO paper](https://huggingface.co/papers/2507.18071) shows that sequence-level sampling often yields more
stable training and better alignment with sequence-level rewards.
reward_weights (`list[float]` or `None`, *optional*, defaults to `None`):
Weights for each reward function. Must match the number of reward functions. If `None`, all rewards are
weighted equally with weight `1.0`.
scale_rewards (`bool`, *optional*, defaults to `True`):
Whether to scale the rewards by dividing them by their standard deviation. If `True` (default), the rewards
are normalized by the standard deviation, ensuring they have unit variance. If `False`, no scaling is
applied. The [Dr. GRPO paper](https://huggingface.co/papers/2503.20783) recommends not scaling the rewards,
as scaling by the standard deviation introduces a question-level difficulty bias.
loss_type (`str`, *optional*, defaults to `"bnpo"`):
Specifies the loss formulation to use. Supported values are:
- `"grpo"`: Aggregates token-level losses by normalizing over sequence length. Not recommended due to
length bias—this approach tends to prefer shorter completions with positive advantages and longer ones
with negative advantages.
- `"bnpo"`: Aggregates token-level losses by normalizing number of active token in the local batch.
Note that normalization is performed over the local batch only, so results may slightly vary depending
on the local batch size, despite a constant effective batch size. When using
`per_device_train_batch_size==1`, the loss is equivalent to the GRPO loss.
- `"dr_grpo"`: Aggregates token-level losses by normalizing with a global constant. This method was
introduced in the [Dr. GRPO paper](https://huggingface.co/papers/2503.20783) to eliminate length bias.
The value of the constant corresponds to `max_completion_length`.
mask_truncated_completions (`bool`, *optional*, defaults to `False`):
When enabled, truncated completions are excluded from the loss calculation, preventing them from being
incorrectly penalized and introducing noise during training. According to the
[DAPO](https://huggingface.co/papers/2503.14476) paper, this is a good practice for training stability.
sync_ref_model (`bool`, *optional*, defaults to `False`):
Whether to synchronize the reference model with the active model every `ref_model_sync_steps` steps, using
the `ref_model_mixup_alpha` parameter. This synchronization originates from the
[TR-DPO](https://huggingface.co/papers/2404.09656) paper.
ref_model_mixup_alpha (`float`, *optional*, defaults to `0.6`):
α parameter from the [TR-DPO](https://huggingface.co/papers/2404.09656) paper, which controls the mix
between the current policy and the previous reference policy during updates. The reference policy is
updated according to the equation: `π_ref = α * π_θ + (1 - α) * π_ref_prev`. To use this parameter, you
must set `sync_ref_model=True`.
ref_model_sync_steps (`int`, *optional*, defaults to `512`):
Ï„ parameter from the [TR-DPO](https://huggingface.co/papers/2404.09656) paper, which determines how
frequently the current policy is synchronized with the reference policy. To use this parameter, you must
set `sync_ref_model=True`.
top_entropy_quantile (`float`, *optional*, defaults to `1.0`):
Ï parameter from [Beyond the 80/20 Rule](https://huggingface.co/papers/2506.01939). Keeps in the policy
loss term only the top-Ï quantile of tokens by entropy of the probability distribution at each sequence
position, improving results. Range: `[0.0-1.0]`. A value of `0.0` masks all but the highest entropy token;
`1.0` keeps all tokens. The paper recommends a value of `0.2`.
If used with `mask_truncated_completions=True`, only tokens from non-truncated completions are considered.
use_liger_loss (`bool`, *optional*, defaults to `False`):
Whether to use the Liger GRPO loss.
> Parameters that control the logging
log_completions (`bool`, *optional*, defaults to `False`):
Whether to log a sample of (prompt, completion) pairs every `logging_steps` steps. If `rich` is installed,
it prints the sample. If `wandb` logging is enabled, it logs it to `wandb`.
num_completions_to_print (`int` or `None`, *optional*, defaults to `None`):
Number of completions to print with `rich`. If `None`, all completions are logged.
wandb_log_unique_prompts (`bool`, *optional*, defaults to `False`):
Whether to log unique prompts in wandb. If `True`, only unique prompts are logged. If `False`, all prompts
are logged.
helpzvLLM SamplingParams)ÚdefaultÚmetadataÚvllm_sampling_paramsrez8Chunk size to reduce memory usage. -1 is most efficient.Úunsloth_num_chunksFÚnorféréúg-Cëâ6
?g{®Gáz„?gÍÌÌÌÌÌì?g+‡ÙÎ÷ï?g:Œ0âŽyE>rglinearçš™™™™™¹?ÚpassiveÚwarningTÚstepsrméôiO
ÚO1ÚautoÚrÚ
adamw_8bitÚlengthÚ
every_saveÚlastiéééÚcolocater8z0.0.0.0i@gn@g333333Ó?gü©ñÒMbP?rŠÚtokenr˜g333333ã?c° ó6|dkrtd|d¦«|dkrtd|d¦«||#dkr
|$dkrd}d }#|¤ ¦«d
krd
}¤n|¤ ¦«d krd }¤|¤ ¦«d
kr!|£dkrd }£ne|£d krtd
¦«d}£nM|¤ ¦«d kr5td¦«td¦«td¦«d }¥d} d}œd}¤||„z|„z|kr4tdt |¦«zdzt |„¦«z¦«|„}|Šdkrt d¦«|Šdkrt d¦«t
¦«jdÈid|d|d|d|d|d |d!|d"|d#| “d$|
d%| d&| d'|
d(|d)|d*|d+|d,|d-|d.|d/|d0|d1|d2|d3|d4|d5|d6|d7|d8|d9|d:| “d;|!“d<|"“d=|#“d>|$“d?|%“d@|&“dA|'“dB|(“dC|)“dD|*“dE|+“dF|,“dG|-“dH|.“dI|/“dJ|0“dK|1“dL|2“dM|3“dN|4“dO|5“dP|6“dQ|7“dR|8“dS|9“dT|:“dU|;“dV|<“dW|=“dX|>“dY|?“dZ|@“d[|A“d\|B“d]|C“d^|D“d_|E“d`|F“da|G“db|H“dc|I“dd|J“de|K“df|L“dg|M“dh|N“di|O“dj|P“dk|Q“dl|R“dm|S“dn|T“do|U“dp|V“dq|W“dr|X“ds|Y“dt|Z“du|[“dv|\“dw|]“dx|^“dy|_“dz|`“d{|a“d||b“d}|c“d~|d“d|e“d€|f“d|g“d|h“dƒ|i“d„|j“d…|k“d†|l“d‡|m“dˆ|n“d‰|o“dŠ|p“d|q“dŒ|r“d|s“dŽ|t“d|u“d|v“d|w“d|x“d“|y“d”|z“d•|{“d||“d—|}“d˜|~“d™|dš|€“d›|dœ|‚“d|ƒ“dž|„“dŸ|…“d |†“d¡|‡“d¢|ˆ“d£|‰“d¤|Š“d¥|‹“d¦|Œ“d§|d¨|Ž“d©|dª|d«|‘“d¬|’“d­|““d®|”“d¯|•“d°|–“d±|—“d²|˜“d³|™“d´|š“dµ|›“d¶|œ“d·|d¸|ž“d¹|Ÿ“dº| “d»|¡“d¼|¢“d½|£“d¾|¤“d¿|¥“dÀ|¦“dÁ|§“dÂ|¨“dÃ|©“dÄ|ª“dÅ|«“dÆ|¬“dÇ|­“|°¤Ž|®|_|¯|_ dS)ÉNgH¯¼šò×z>z Unsloth: Your learning rate of `zi` is too small and less than 1e-7! Consider increasing it, otherwise gradient updates will be close to 0!rmza` is way too larger > 1! Consider decreasing it to 1e-1, otherwise gradient updates will explode!rJrKÚunsloth_training_checkpointsrCr™ÚdapoTzwUnsloth: The Dr GRPO paper recommends setting `scale_rewards` to False! Will override. Set it to `None` to force False.FzFUnsloth: The DAPO paper recommends `mask_truncated_completions = True`z8Unsloth: The DAPO paper recommends `epsilon_high = 0.28`zMUnsloth: The DAPO paper recommends setting `beta = 0.0` to remove the KL termgìQ¸…ëÑ?rr˜z}Unsloth: We now expect `per_device_train_batch_size` to be a multiple of `num_generations`.
We will change the batch size of z to the `num_generations` of rzUUnsloth: Please set a positive non-zero temperature since your results will be wrong.é
zgUnsloth: Please set a positive non-zero temperature less than 10, since sampling will be quite erratic.Ú
output_dirÚoverwrite_output_dirÚdo_trainÚdo_evalÚ
do_predictÚ
eval_strategyÚprediction_loss_onlyÚper_device_train_batch_sizeÚper_device_eval_batch_sizeÚper_gpu_train_batch_sizeÚper_gpu_eval_batch_sizeÚgradient_accumulation_stepsÚeval_accumulation_stepsÚ
eval_delayÚtorch_empty_cache_stepsÚ
learning_rateÚ weight_decayÚ
adam_beta1Ú
adam_beta2Ú adam_epsilonÚ
max_grad_normÚnum_train_epochsÚ max_stepsÚlr_scheduler_typeÚ warmup_ratioÚ warmup_stepsÚ log_levelÚlog_level_replicaÚlog_on_each_nodeÚ logging_dirÚlogging_strategyÚlogging_first_stepÚ
logging_stepsÚlogging_nan_inf_filterÚ
save_strategyÚ
save_stepsÚsave_total_limitÚsave_safetensorsÚsave_on_each_nodeÚsave_only_modelÚ'restore_callback_states_from_checkpointÚno_cudaÚuse_cpuÚuse_mps_deviceÚseedÚ data_seedÚ
jit_mode_evalÚuse_ipexÚbf16rÚfp16_opt_levelÚhalf_precision_backendÚbf16_full_evalÚfp16_full_evalÚtf32Ú
local_rankÚ ddp_backendÚ
tpu_num_coresÚtpu_metrics_debugÚdebugÚdataloader_drop_lastÚ
eval_stepsÚdataloader_num_workersÚdataloader_prefetch_factorÚ
past_indexÚrun_nameÚ disable_tqdmÚremove_unused_columnsÚ label_namesÚload_best_model_at_endÚmetric_for_best_modelÚgreater_is_betterÚignore_data_skipÚfsdpÚfsdp_min_num_paramsÚ fsdp_configÚ"fsdp_transformer_layer_cls_to_wrapÚaccelerator_configÚ deepspeedÚlabel_smoothing_factorÚoptimÚ
optim_argsÚ adafactorÚgroup_by_lengthÚlength_column_nameÚ report_toÚddp_find_unused_parametersÚddp_bucket_cap_mbÚddp_broadcast_buffersÚdataloader_pin_memoryÚdataloader_persistent_workersÚskip_memory_metricsÚuse_legacy_prediction_loopÚ push_to_hubÚresume_from_checkpointÚ hub_model_idÚ hub_strategyÚ hub_tokenÚhub_private_repoÚhub_always_pushÚ hub_revisionÚgradient_checkpointingÚgradient_checkpointing_kwargsÚinclude_inputs_for_metricsÚeval_do_concat_batchesÚ fp16_backendÚpush_to_hub_model_idÚpush_to_hub_organizationÚpush_to_hub_tokenÚ
mp_parametersÚauto_find_batch_sizeÚfull_determinismÚ torchdynamoÚ ray_scopeÚ ddp_timeoutÚ
torch_compileÚtorch_compile_backendÚtorch_compile_modeÚinclude_tokens_per_secondÚinclude_num_input_tokens_seenÚneftune_noise_alphaÚoptim_target_modulesÚbatch_eval_metricsÚ
eval_on_startÚuse_liger_kernelÚliger_kernel_configÚeval_use_gather_objectÚaverage_tokens_across_devicesÚmodel_init_kwargsÚdisable_dropoutÚmax_prompt_lengthÚnum_generationsrŒÚds3_gather_for_generationÚshuffle_datasetÚgeneration_batch_sizeÚsteps_per_generationrŽÚtop_pÚtop_kÚmin_pÚgeneration_kwargsÚrepetition_penaltyÚuse_transformers_pagedÚcache_implementationÚuse_vllmÚvllm_server_base_urlÚ vllm_modeÚvllm_model_implÚvllm_guided_decoding_regexÚvllm_server_hostÚvllm_server_portÚvllm_server_timeoutÚvllm_gpu_memory_utilizationÚvllm_tensor_parallel_sizer­Únum_iterationsÚepsilonrrÚimportance_sampling_levelÚreward_weightsÚ
scale_rewardsr‡Úmask_truncated_completionsÚsync_ref_modelÚref_model_mixup_alphaÚref_model_sync_stepsÚtop_entropy_quantileÚuse_liger_lossÚlog_completionsÚnum_completions_to_printÚwandb_log_unique_promptsr )
ÚFloatingPointErrorÚ
OverflowErrorÚlowerÚprintÚstrÚ MathErrorÚsuperÚ__init__rArB)²Úselfr\r]r^r_r`rarbrcrdrerfrgrhrirjrkrlrmrnrorprqrrrsrtrurvrwrxryrzr{r|r}r~rr€rrr„r…r†r‡r‰rrrrrrrr“r”r•r–r—r™rrr r­r¿r­rrr‡rÿrrrArBÚ __class__s² €rr zUnslothGRPOConfig.__init__Ús
ø€ðf ˜ Ð Õ'9ð;VÐ]jð;Vð;Vð;Vñ(Wô(Wð"WØ ˜ Ð ¥Mð3FÐUbð3Fð3Fð3Fñ%Gô%GðGØ Ð  -°7Ò":Ð":¸zÈSÒ?PÐ?PØ7ˆ ˆ ?Š?Ñ Ô   Ò !ˆIˆIØ
_Š_Ñ
Ô
 
ˆIØ ?Š?Ñ Ô   Ò  Ò$Ø $
Ø ðPñQôQðQØ %
øØ
_Š_Ñ
Ô
 
Ð Ð Ð bØ)-Ð ˆˆˆ '¨?Ñ :¸ MÐQlÒ ðSõVYðZuñVvôVvñvðyXñXõ[^ð_nñ[oô[oñoñ
pô
pð
pØ*9Ð ˜ Ð ÝÐ
˜
Ð
ÝðFñGôGð
Gð ŒÔðm Jðm Jðm JØ#˜ðm Jà#7Ð#7ðm Jð m Jðgð m Jð
$˜ð m Jð *˜
m Jð$8Ð#7ðm Jð+FÐ*Eðm Jð*DÐ)Cðm Jð(@Ð'?ðm Jð'>Ð&=ðm Jð+FÐ*Eðm Jð'>Ð&=ðm Jð$˜ðm Jð'>Ð&=ðm Jð *˜Mð!m Jð"(˜<ð#m Jð$$˜ð%m Jð&$˜ð'm Jð((˜<ð)m Jð**˜Mð+m Jð,/ð-m Jð."˜ ð/m Jð0!2Ð 1ð1m Jð2(˜<ð3m Jð4(˜<ð5m Jð6"˜ ð7m Jð8!2Ð 1ð9m Jð:/ð;m Jð<&˜+ð=m Jð>/ð?m Jð@"4Ð!3ðAm JðB*˜MðCm JðD&<Ð%;ðEm JðF*˜MðGm JðH$˜ðIm JðJ/ðKm JðL/ðMm JðN!2Ð 1ðOm JðP.˜oðQm JðR7^Ð6]ðSm JðTgðUm JðVgðWm JðX,˜^ðYm JðZ4ð[m Jð\"˜ ð]m Jð^*˜Mð_m Jð` xðam Jðb4ðcm Jðd4ðem Jðf,˜^ðgm Jðh&<Ð%;ðim Jðj,˜^ðkm Jðl,˜^ðmm Jðn4ðom Jðp$˜ðqm Jðr&˜+ðsm Jðt*˜Mðum Jðv!2Ð 1ðwm JðxEðym Jðz$8Ð#7ð{m Jð|$˜ð}m Jð~&<Ð%;ðm Jð@*DÐ)CðAm JðB$˜ðCm JðD xðEm JðF(˜<ðGm JðH%:Ð$9ðIm JðJ&˜+ðKm JðL&<Ð%;ðMm JðN%:Ð$9ðOm JðP!2Ð 1ðQm JðR/ðSm JðT4ðUm JðV#6Ð"5ðWm JðX&˜+ðYm JðZ2TÐ1Sð[m Jð\"4Ð!3ð]m Jð^"˜ ð_m Jð`&<Ð%;ðam JðbEðcm Jðd$˜ðem Jðf"˜ ðgm Jðh.˜oðim Jðj"4Ð!3ðkm Jðl"˜ ðmm Jðn*DÐ)Cðom Jðp!2Ð 1ðqm Jðr%:Ð$9ðsm Jðt%:Ð$9ðum Jðv-JÐ,Iðwm Jðx#6Ð"5ðym Jðz*DÐ)Cð{m Jð|&˜+ð}m Jð~&<Ð%;ðm Jð@(˜<ðAm JðB(˜<ðCm JðD"˜ ðEm JðF/ðGm JðH.˜oðIm JðJ(˜<ðKm JðL&<Ð%;ðMm JðN-JÐ,IðOm JðP*DÐ)CðQm JðR&<Ð%;ðSm JðT(˜<ðUm JðV$8Ð#7ðWm JðX(@Ð'?ðYm JðZ!2Ð 1ð[m Jð\*˜Mð]m Jð^$8Ð#7ð_m Jð`/ðam Jðb&˜+ðcm Jðd"˜ ðem Jðf&˜+ðgm Jðh*˜Mðim Jðj%:Ð$9ðkm Jðl"4Ð!3ðmm Jðn)BÐ(Aðom Jðp-JÐ,Iðqm Jðr#6Ð"5ðsm Jðt$8Ð#7ðum Jðv"4Ð!3ðwm Jðx*˜Mðym Jðz/ð{m Jð|#6Ð"5ð}m Jð~&<Ð%;ðm Jð@-JÐ,IðAm JðB!2Ð 1ðCm JðD.˜oðEm JðF!2Ð 1ðGm JðH.˜oðIm JðJ%:Ð$9ðKm JðL)BÐ(AðMm JðN.˜oðOm JðP%:Ð$9ðQm JðR$8Ð#7ðSm JðT&˜+ðUm JðVEðWm JðXEðYm JðZEð[m Jð\!2Ð 1ð]m Jð^"4Ð!3ð_m Jð`&<Ð%;ðam Jðb$8Ð#7ðcm Jðd xðem Jðf$8Ð#7ðgm Jðh"˜ ðim Jðj.˜oðkm Jðl*DÐ)Cðmm Jðn/ðom Jðp/ðqm Jðr#6Ð"5ðsm Jðt+FÐ*Eðum Jðv)BÐ(Aðwm Jðx4ðym Jðz,˜^ð{m Jð|gð}m Jð~m Jð@(˜<ðAm JðB)BÐ(AðCm JðD,˜^ðEm JðF*˜MðGm JðH"˜ ðIm JðJ*DÐ)CðKm JðL,˜^ðMm JðN%:Ð$9ðOm JðP$8Ð#7ðQm JðR$8Ð#7ðSm JðT,˜^ðUm JðV.˜oðWm JðX(@Ð'?ðYm JðZ(@Ð'?À&ð[m Jðm Jðm Jð\%9ˆÔ!Ø"4ˆÔÐÐr„)
rrrÚ__doc__rZrArrÚ__annotations__rBÚintr Ú
__classcell__©r s@rr=r=ø€ðJðJðV+0¨%ØØÐ+ñ+ô+И( 3œ-ððñð*/¨ØØÐ*ñ*ô*И #œððñð ØØØØØ$Ø&'Ø%&Ø#'Ø"&Ø&'Ø"#ØØ"%ØØØØØØØØØØØØØØØ!&ØØØØØØ27ØØØØØØØØØØØ!'ØØØØØØØØØ!"Ø%)ØØØØ %ØØ!&Ø $Ø Ø ØØØØ-1ØØ!$ØØØØØØ%)Ø Ø $Ø $Ø(-Ø"Ø%*ØØ!%ØØØØØØ!&Ø(,Ø%*Ø!%ØØ#Ø#'Ø ØØ ØØØØØ $Ø!Ø$)Ø(-ØØ Ø"Ø!&Ø(,Ø ØØØØ #Ø$(ØØ $ØØØØØØ Ø!&ØØØ Ø%)ØØ#Ø&)Ø$%ØØØØØØ$+ØØØØ%*ØØ #ØØØ#'Ø#(ØðaDDDDDDDDDD5r„r=có’eZdZdZddgZ d9deeefdeee efde
e d e
ee e
fd
e
ee e
eeee e
fffd e
eeefd e
eee efd
e
e edee
ejje
ejjjfde
dfˆfd
ZdZdZd:de
e defdZdefdZdede defdZe d;d¦«Z d:dej!dej!de"dej!fdZ#d<dZ$d:d e
e efd!„Z%d=d"e&j'd#efd$„Z(d"e&j'fd%„Z)d&„Z*ed'eeeej!e+ffdeeeej!e+fffd(„¦«Z,eˆfd)„¦«Z-d*e eeeej!e+ffdeeeej!e+fffˆfd+„ Z.d,„Z/d>d-„Z0d.„Z1d:d/e
e efd0„Z2d:d1eee"fd2e
e"ddfˆfd3„
Z3ˆfd4„Z4 d?d5e
ed6e
ed7eee edffd8„Z5ˆxZ6S)@Ú_UnslothGRPOTrainerrNÚtrlrˆNNr$Ú reward_funcsÚargsÚ
train_datasetÚ eval_datasetÚprocessing_classÚreward_processing_classesÚ callbacksÚ
optimizersÚ peft_configrc ó¬t|d¦«r3td¦«r#tdd¦«dkrd_d_€Pt |t
¦«r|n |jj} |  d¦«d} t| d¦«Šj
pi} t |t
¦«rÀ|}
|   d ¦«}t |tj
¦«s|d
ks|nCt |t
¦«rtt|¦«}|| d <ntd |d ¦«tj|
¦«}tt"|jd
¦«}|j|
fi| ¤Ž}n"|jj}
j
td¦«t|d¦«s0t'j|j¦«j ¦«nAt'j| ¦«j¦«j ¦«_ jr |¦«}|t=j|jj¦«}t |t>¦«r|j }n't |tB¦«r|}ntEd¦«|j# |j$|_#|j#_#|j%_%|j&_&t|dd¦«_'t|dd¦«_(t|jdd¦«_)t|jdd¦«_*t |tV¦«s|g}g_,t[|¦«D]º\}}t |t
¦«rt]j|fddi| ¤Ž||<t ||t^j0¦«rD‰j, 1||jj d¦«d¦«Œ•‰j, 1||j2¦«Œ»|_3j4…tkj4¦«tk|¦«kr5tdtkj4¦«dtk|¦«d¦«tj6j4tj7¬¦«_4n2tj8tk|¦«tj7¬¦«_4|dgtk|¦«z}nHt |tV¦«s|g}n/tk|¦«tk|¦«krtd¦«t[ts||¦«¦«D]f\}\}}t |tt¦«rI|twj|jj¦«}|j% |j$|_#|j%|j_%|||<Œg|_<j=_=j>_>j?_?j@_@jA_AjB_BjC_CjD_DjE_Ej_j_jF_FjG_GjH_HjI_IjJ_JjK_KjL_LjM_MjHrjMdkrtd¦«jHrjKdkstd ¦«jO_Ot |t ¦«sUt |t ¦«s@t |¦«r:t¥d!„| S¦«D¦«¦«rtd"¦«jT_TjU_VjWjWnjU_Wd
_Xd_Yd|jZd#<¦« \|||||| ¬$¦«j^_^j^d%krd__n^tÁ|¦«rd__nGtj|
¦«}tt"|jd
¦«}|j|
fi| ¤Ž__jar*tÅ|¦«j_j_¦«jHrrtǦ«st7d&¦«¦«_ej^jVjWj@j^d%kjIj>¬'¦«_gtV¦«tV¦«d(œ_id
_jjk_kjl_ljm_mjo¬)¦«jo¬)¦«jo¬)¦«ˆfd*„¦«jo¬)¦«d+œ_pjrd¬,¦«jr|t禫st7d-¦«jd.krjtjuru‰jvjv}nd/‰jwd0‰jx}|jz¬1¦«_{j{ |tj} ~¦«¬2¦«n¦‰jdkrš‰jtjjGzd
ks%td3‰jGd4‰jtjd5¦«jGdkrStj€ ˆfd6„tjtjjGz¦«D¦«¦«\}t jtj„¦«t
j†d7<t jtj‡¦«t
j†d8<t jtj¦«t
j†d9<t
j†  d:d;¦«t
j†d:<t
j†  d<d=¦«t
j†d<<j=j>j=j>z}nd}|jˆ_‰_‹djt ¦«n‰‰j>d|j%||j&j@jAjBjCjDjd>œ }jErd?|d@<dA|dB<dC|dD<j| ‘j¦«t%dHi|¤Ž_“d_”j• –j—¦«j_v‰j˜r!t3j_jt¦«__nN‰r!t7j_jt¦«__n&‰jt œj_d¬E¦«__jr/‰ žt?j_jt¬F¦«¦«t[j3¦«D]e\}}t |tt¦«rK‰j˜rt3|jt¦«j3|<Œ@‰jt œ|dd¬G¦«j3|<ŒfdS)INÚ vllm_enginerêFTrVú/rez-GRPOÚ torch_dtyperMz‰Invalid `torch_dtype` passed to `GRPOConfig`. Expected either 'auto' or a string representing a `torch.dtype` (e.g., 'float32'), but got ú.rz¡You passed `model_init_kwargs` to the `GRPOConfig`, but your model is already instantiated. This argument can only be used when the `model` argument is a string.Úget_base_modelz>PEFT is required to use `peft_config`. Run `pip install peft`.zWThe `processing_class` must be either a `PreTrainedTokenizerBase` or a `ProcessorMixin`Ú image_tokenÚimage_token_idÚvision_start_token_idÚvision_end_token_idÚ
num_labelsrmzNumber of reward weights (z)) must match number of reward functions (úrzRThe number of reward processing classes must match the number of reward functions.rzOLiger Kernels don't currently support masking token positions based on entropy.rWzwLiger Kernels currently only support token-level importance sampling. Please set`importance_sampling_level` to 'token'.c3ó@K|]}t|t¦«VŒdS)Ú
isinstancer)r Údss rú <genexpr>z/_UnslothGRPOTrainer.__init__.<locals>.<genexpr>s-èèÐ6wÐ6wÐ[]µzÀ"ÅoÑ7VÔ7VÐ6wÐ6wÐ6wÐ6wÐ6wÐ6wr„z^Iterable datasets are not yet supported in GRPOTrainer. Please use a standard dataset instead.Úestimate_tokens)r$rÚ
data_collatorrrrrrrzWLiger is required to use `liger_loss` as the GRPO loss. Run `pip install liger-kernel`.)r­r‰rÚ
use_ref_modelr‡)ÚtrainÚeval©Úmaxlencó.tj¬¦«S)Nr4)r+)rs€rú<lambda>z._UnslothGRPOTrainer.__init__.<locals>.<lambda>nsø€­5¸Ô8RÐ+SÑ+SÔ+S€r„)ÚimageÚpromptÚ
completionÚrewardsr®)Údevice_specificzkvLLM is not available and `use_vllm` is set to True. Please install vLLM with `pip install vllm` to use it.Úserverzhttp://ú:)Úbase_urlÚconnection_timeoutrÑzvllm_tensor_parallel_size (z) must divide world size (z ) evenly.c ópg|]2}tt|jz|dzjz¦«¦«Œ3S)rm)Úlistr)r r
r
s €rrz0_UnslothGRPOTrainer.__init__.<locals>.<listcomp>sQø€ðððà !õ q¨4Ô+IÑ'IÈAÐPQÉEÐUYÔUsÑKsÑ!tÔ!tÑððr„ÚRANKÚ
LOCAL_RANKÚ
WORLD_SIZEÚ MASTER_ADDRÚ localhostÚ MASTER_PORTÚ12345) Úmax_new_tokensÚ do_sampleÚ pad_token_idÚ bos_token_idÚ eos_token_idrŽrSÚmax_batch_tokensiÚ
num_blocksé€Ú
block_size)Úevaluation_mode)Ú ref_modelr*)rSÚdevice_placementr ) r Úgetattrrêr,rÚconfigÚ
_name_or_pathÚsplitrrSrr
Úfrom_pretrainedrTÚ
architecturesr3Ú signaturerüÚ
parametersÚkeysr#Úmodel_kwarg_keysr8Ú ImportErrorrÀÚ_enable_gradient_checkpointingrrÚ tokenizerrÚ TypeErrorÚ pad_tokenÚ eos_tokenrLrNr$r%r&r'rBÚreward_func_namesÚ enumeraterrAÚModulerwrrrÚtensorrsÚonesrqrrrr‡ÚNotImplementedErrorràrÚdictÚanyÚvaluesrôr‰rÚ_stepÚ_buffered_inputsÚwarnings_issuedrr r2r­rTr9r,r7r%Ú_forward_redirectionÚLigerFusedLinearGRPOLossÚliger_grpo_lossr*Ú_metricsÚ_total_train_tokensrÿrrr+Ú_logsrNrˆr;r*Úis_main_processrër$Ú vllm_clientÚinit_communicatorÚcudaÚcurrent_deviceÚ
num_processesÚ distributedÚnew_subgroups_by_enumerationrÚtp_groupÚ
process_indexrCr!Úlocal_process_indexrÚllmrîÚguided_decoding_regexÚ_last_loaded_stepÚwait_for_everyonerMÚupdaterÚgeneration_configÚmodel_accepts_loss_kwargsr$Úadd_model_tagsÚ
_tag_namesÚis_deepspeed_enabledrFÚis_fsdp_enabledrGÚ
prepare_modelrúÚ add_callbackr!)r
r$rrrrrrrrrÚ
model_namerÛÚmodel_idr!rWÚ architecturerbr
Ú reward_funcÚreward_processing_classr?Ú
max_model_lenrær s` ` €rr z_UnslothGRPOTrainer.__init__føøø€õ 5˜  &­W°T¸:Ñ-FÔ-Fð˜˜j¨%Ñ0°EÒ9Ø $
Ø%ˆDŒNà ˆ<Ý",¨UµCÑ"8Ô"8ÐX˜˜¸e¼lÔ>Xˆ#×)¨#Ñ.¨rÔ2ˆJÝ Ð3ˆDð8°bÐÝ e  ؈
Ñ>ˆ˜+¥u¤{Ñ
°{ÀfÒ7LÐ7LÐP[ÐPcØÝ˜K­Ñ
Ý%¥e¨[Ñ9 Ø3>Ð!  ðQØBMðQðQðQñôðõ
 ÔÑ9ˆ"¥<°Ô1EÀaÔ1HÑIˆ0ÐOÐ=NÐOˆEˆEà”|Ô1ˆHØÔ ð\ñôðõ˜5Ð"2Ñ
U ˜eœmÑ 7× Ô" 5×#7Ò#7Ñ#9Ô#9Ô#AÑM×
Ôð  ð Ô  EØ×7¸¸tÑDˆEð Ð <¸U¼\Ô=WÑ õ Ð&­Ñ  wØ2ˆIˆ
Ð(Õ*AÑ
 wØ(ˆIˆIåÐ Ô Ð &Ø"+Ô"5ˆ à,ˆŒØÔØÔÝ"Ð#3°]ÀDÑIˆÔÝ%Ð&6Ð8HÈ$ÑOˆÔÝ%,¨U¬\Ð;RÐTXÑ%YÔ%YˆÔ"Ý#*¨5¬<Ð9NÐPTÑ#UÔ#UˆÔ õ˜,­Ñ(˜>ˆLØ!#ˆÔÝ Ñ Hð H‰NˆAˆ˜+¥sÑ
Ý"DÔ"TØð#ð#Ø,-ð#Ø1Bð#ð# ˜Qõ˜, qœ/­2¬9Ñ
HØÔ&×-¨l¸1¬oÔ.DÔ.R×.XÒ.XÐY\Ñ.]Ô.]Ð^`Ô.aÑÔ&×-¨l¸1¬oÔ.FÑÔð Ô Ð '­3¨|Ñ+<Ô+<Ò ð°TÔ5HÑ1IÔ1Ið7Ý"% lÑ"3Ô"3ðôðõ#(¤,¨tÔ/BÍ%Ì-Ð"XÑ"XÔ"Xˆ Ð å"'¤*­Ñ->Ô->ÅeÄmÐ"TÑ"TÔ"Tˆ ð ,Ø)-¨µ°\Ñ1BÔ1BÑ(BÐ Ð5µtÑ wØ)BÐ(CРа\Ñ1BÔ1BÒ Ð!uÑvå9BÅ3ÐG`ÐbnÑCoÔCoÑ9pÔ9pð Gð GÑ 5ˆÝ˜Ñ
GØ2Ý.;Ô.KÈKÔL^ÔLlÑ.mÔ.mÐ?Ø8OÔ8YÐ3JÔ2V Ô/Ø/FÐ)¨!Ñ,øØ)BˆÔ"&Ô!7ˆÔØ%)Ô%?ˆÔÔØÔÔØ”ZˆŒ
Ø”ZˆŒ
Ø”ZˆŒ
Ø"&Ô"9ˆÔØ&*Ô&AˆÔœ
ˆŒ
ØœˆŒØ+/Ô+KˆÔ(Ø)-Ô)GˆÔ1ˆÔØœˆŒØÔØ)-Ô)GˆÔ&Ø*.Ô*IˆÔ'Ø$(Ô$=ˆÔ Ô ð  4Ô#<¸sÒ#BÐ#BÝôð
ð Ô ð  tÔ'EÈÒ'PÐ'PÝôð
ð 3ˆÔõ
}¥oÑ 
å˜,­Ñ
õ˜<­Ñ
õ47Ð6wÐ6wÐam×atÒatÑavÔavÐ6wÑ6wÔ6wÑ3wÔ3wð
õôð
ð
ÔØœ<ˆÔØ15Ô1BÐ1N˜-ÐTXÔT`ˆÔàˆŒ
ð!%ˆÔð48ˆÔÐ
Œ×ÒØØÝØ ñ
ô
ð
ð”IˆŒ Ø Œ9˜Ò Ð à!ˆDŒNˆNÝ
˜
 Yð"ˆDŒNˆ ÔÑ9ˆ"¥<°Ô1EÀaÔ1HÑIˆ9˜9¸XÐFWÐXˆDŒNð Ô ð $ UÑ Œ~Ð¬Ñ Ô ð Ý
Ýôðõ)<Ñ(=Ô(=ˆ %å#;Ø”YØ Ô Ô"œi¨3Òœ.Ø&*Ô&@ð$ñ$ô$ˆ õ#.­dÑ"3Ô"3½[ÍÑ=NÔ=NÐOˆŒ
Ø#$ˆÔ ØÔØ(,Ô(EˆÔ%Ø(,Ô(EˆÔ $Ô"<Р4Ô#=РtÔ'AÐ"Ð#SÐ#SÐ#SÐ#SÑ tÔ'AÐ 
ð
ˆŒ
õ ¨DÐ Œ=ñ@ KÝ
Ýôðð
Œ~ Ò)¨dÔ.>Ô.NÐÔ8HÔ)>ÐÔAVÐXHÝ#-°xÐTXÔTlÐ#mÑ#mÔ#mÔ ØÔ ×2½%¼*×:SÒ:SÑ:UÔ:UРÔ5¸Ô8VÑVÐZ[ÒF°dÔ6TðFðFØ ÔFðFðFñôðð
Ô1°AÒ5Ý',Ô'8×'UÒ'Uððððæ%*¨4Ô+;Ô+IÈTÔMkÑ+kÑ%lÔ%lðñôñ(ô(Ñ$D”M  &)¨Ô)9Ô)GÑ%HÔ%H
˜"Ý+.¨tÔ/?Ô/SÑ+TÔ+T
˜(Ý+.¨tÔ/?Ô/MÑ+NÔ+N
˜(Þ,.¬J¯NªN¸=È+Ñ,VÔ,V
˜)Þ,.¬J¯NªN¸=È'Ñ,RÔ,R
˜=ÑÔ5¸$Ô:TÐ:`Ø$(Ô$:¸TÔ=WÑ$WMMà$( Ô,Ø)-Ô)Hˆ &à%'ˆ Ô × #'Ô"<Ø!Ø )Ô 6Ø )Ô 6Ø )Ô 6؜؜؜Ø&*Ô&=Ø(,Ô(Að !ð !Ð ðÔ
6Ø8;Ð!Ð"4Ñ5Ø26Ð! /Ø25Ð! ÔÔ)?Ñ@Þ%5Ð%JÐ%JÐ8IÐ%JÐ%Jˆ
*/ˆÔ
Œ
×! $¤/Ñ Œ>Ð Ô
fÞ!2°4´>À4ÔCSÑ!TÔ!TØÔ
fÞ!-¨d¬n¸dÔ>NÑ!OÔ!Oà!%Ô!1×!?Ò!?ÀÄÐ`dÐ!?Ñ!eÔ!eà Ô ð lØ × Ò Ö2¸T¼^ÐY]ÔYiÐ Ô(9Ñ ð ‰NˆAˆ{ݘ+¥Ñ
ØÔÞ+<¸[È$ÔJZÑ+[Ô+[% ,0Ô+;×+IÒ+IØ#°TÈDð,Jñ,ô,% (øð
 ð r„có*|j ddg|_dSdS)Nr9r8)Ú_signature_columns)r
s rÚ _set_signature_columns_if_neededz4_UnslothGRPOTrainer._set_signature_columns_if_neededÖs(ð
Ô *Ø'/°Ð&9ˆ  *r„cóÀ|jtd¦«|j}|j}t¦«r2t |t
j¦«r| |d¬¦«}n| |d¬¦«}|j |j
j z||j
j |j
j
|j
jdœ}t |tjjj¦«sc| ¦«|d<|j
j|d<t+t,|j
j |j
j¬¦«|d<|j
j|d <|j t7|fi|¤Ž¦«S)
Nz+Trainer: training requires a train_dataset.Útraining)Ú description)Ú
batch_sizeÚ
collate_fnÚ num_workersÚ
pin_memoryÚpersistent_workersÚsamplerÚ drop_last)ÚrankÚworker_init_fnÚprefetch_factor)rr0r5r,r)rÚ_remove_unused_columnsÚ"_get_collator_with_removed_columnsÚ_train_batch_sizerr™r´rSÚutilsÚdatarÚ_get_train_samplerr—rErLrr*Úpreparer)r
rr0Údataloader_paramss rÚget_train_dataloaderz(_UnslothGRPOTrainer.get_train_dataloaderèsiØ Ô Ð Ð Ô
ØÔ*ˆ
Ý Ñ  k¥z°-ÅÔAQÑ'RÔ'Rð kØ ×7¸
ÐS]Ð^ˆMˆMà ×CÀMÐ_iÐjˆÔ0°4´9Ô3QÑœ9Ôœ)Ô9Ø"&¤)Ô"Ið 
ð
Ðõ˜-­¬Ô)9Ô)IÑ XØ+/×+BÒ+BÑ+DÔ+DÐ ˜ (Ø-1¬YÔ-KÐ ˜kÑ *Ý29ݨ¬Ô)IÐPTÔPYÔPgð3ñ3ô3Ð Ð 48´9Ô3WÐ Ð Ô×'­
°=Ð(VÐ(VÐDUÐ(VÐ(VÑWr„ÚdatasetÚreturncó®||j}t||j|jj|jz|j|jjz|j|jj¬¦«S)N)Ú data_sourceÚmini_repeat_countrÚ repeat_countÚshufflerˆ) rrrrˆ)r
s rz&_UnslothGRPOTrainer._get_train_samplers^ð2 ˆÔ(ˆØØ”yÔ6¸$Ô:NÑÔ,¨t¬yÔ/MÑÔð

ñ
ô
ð
r„cóDt||j|jj¬¦«S)N)r´rˆ)rrrˆ)r
rs rÚ_get_eval_samplerz%_UnslothGRPOTrainer._get_eval_sampler*s)åØð
ñ
ô
ð
r„cóîd|j_t|¦«r|j ¦«n| ¦«|jpi}d|vp|d}|r| ¦«|S)z-Enables gradient checkpointing for the model.FÚ
use_reentrant)rWÚ use_cacher9Ú
base_modelÚgradient_checkpointing_enablerÁÚenable_input_require_grads)r
r$rs rraz2_UnslothGRPOTrainer._enable_gradient_checkpointing2s•ð"'ˆŒ Ôõ ˜Ñ Ô ð Ô × 
× 1à(,Ô(JÐ(PÈbÐ Ð#@Ð rÐDaÐbqÔDrð ð ð × ˆ r„c ót|¦«r |jj}||dœ} |||| d<||| d<||| d<||| d<d|jvr|dz| d<|jd i| ¤Žj}
|
ddddddf}
|
dd| dddf}
|
S)
N)rÚimage_grid_thwÚ pixel_valuesÚpixel_attention_maskÚ image_sizesrrmrer )r9r$r_Úlast_hidden_state) r
Úunwrapped_modelr«rrÚ model_inputsrÄs rÚ_get_last_hidden_statez*_UnslothGRPOTrainer._get_last_hidden_stateHõ ˜Ñ >ˆ&/À.ÐQˆ ð Ð %¨,Ð*BØ-;ˆLÐ Ð #Ø+7ˆL˜Ñ Ð +Ø3Gˆ Ð "Ø*5ˆL˜Ñ  ˜ 4à-;¸aÑ-?ˆ 1˜A°LÐà-¨a¨a¨a°°"°°a°a°a¨iÔà-¨a¨a¨a°.°Ð1AÐ1AÀ1À1À1Ð.DÔØ Ð r„Ú entropiesr¬Ú thresholdcó|| ¦« ¦«}| ¦«dkr tj|tj¬¦«S|j |¦«}tj||¦«}|| ¦«z}||k} | | ¦«zS)
Returns a binary mask identifying tokens whose entropy exceeds a given quantile threshold.
Args:
entropies (`torch.Tensor`):
Tensor of shape (batch_size, seq_len) with per-token entropy values.
mask (`torch.Tensor`):
Binary mask of the same shape as `entropies`, where `1` indicates valid tokens and `0` padding.
threshold (`float`):
Quantile threshold between `0.0` and `1.0` to select high-entropy tokens.
Returns:
`torch.Tensor`:
Boolean mask of shape (batch_size, seq_len), where `True` indicates tokens with entropy >= threshold and
`False` otherwise.
rr*)ÚboolÚfloatÚnumelrSÚ
zeros_liker*r.Úquantile)
r
r*Únon_pad_entropiesÚall_non_pad_entropiesÚentropy_thresholdÚmasked_entropiesÚ entropy_masks
rÚget_high_entropy_maskz)_UnslothGRPOTrainer.get_high_entropy_masksð&& d§i¢i¡k¤kÔ2ר × Ò Ô# IµU´ZÐ @Ø $Ô 0× 7Ò 7Ð8IÑ JÔ JÐå!œNÐ+@À)ÑØ$ t§z¢z¡|¤|ÑØ'Ð+<Ò<ˆ ؘdŸiši™kœkÑ)r„Fcó dS)NTr)
r rCr!rSr"r#rr'r(ryÚtrl.trainer.utilsr-) r
r$rrrÚcompute_entropyrryr-s rÚ"_get_per_token_logps_and_entropiesz6_UnslothGRPOTrainer._get_per_token_logps_and_entropiess ð Ø:r„Úextra_prefixescóP|pg}dg|z}|D]}| |d¦«}Œ|S)Nz_checkpoint_wrapped_module.rN)Úreplace)r
ÚnamerÚÚprefixesÚprefixs rÚ_fix_param_name_to_vllmz+_UnslothGRPOTrainer._fix_param_name_to_vllmºsBØ-¨2ˆØ2°^ÑØð ,ˆ—<< ¨Ñ+ˆDˆˆ r„Úmodulerßcóh|t¦«}| ¦«D](\}}|r|d|n|}| |||¬¦«Œ)t|t¦«rÏt j|dd¬¦«5| ¦«D]‡\}}|r|d|n|} | | dg¬¦«} | |vrŒ-| | ¦«|j dkr-|j
j r!|j  
| |j¦«Œz|j d kr Œˆ ddd¦«dS#1swxYwYdSdS)
zdMemory-efficient post-order traversal of FSDP modules to extract full parameters and sync with vLLM.Nr")ÚvisitedF)ÚrecurseÚ writebackz_fsdp_wrapped_module.)r=rV)ÚsetÚnamed_childrenÚ_sync_fsdp1_params_to_vllmr,rÚsummon_full_paramsÚnamed_parametersràÚaddrìr*rxryÚupdate_named_paramr«)
r
Ú
child_nameÚ child_moduleÚ child_prefixÚ
param_nameÚparamÚ full_names
rz._UnslothGRPOTrainer._sync_fsdp1_params_to_vllmÁð ˆe”eˆGØ(.×(=Ò(=Ñ(?Ô(?ð ð Ñ $ˆJ˜ Ø7=ÐM˜3 zÐ3À:ˆLØ ×  \¸7ð

ô
ð
ð
õ f  ÝÔ¸È%Ð
ð
Ø)/×)@Ò)@Ñ)BÔ)BððÑ%J Ø<BÐ R 6Ð 8Ð 8¨JÐ 8Ð 8Ð 8È
IØ $× <Ò <¸YÐXoÐWpÐ <Ñ qÔ q   Ø—K’K  Ñ”~¨Ò1°dÔ6FÔ6VÐÔ;¸IÀuÄzÑœ¨:Òàøðð
ð
ð
ñ
ô
ð
ð
ð
ð
ð
ð
ð
øøøð
ð
ð
ð
ð
ð
ð ð sÁ:BD%Ä%D)Ä,D)có>| ¦«D]‡\}}|jr'| tjd¦«¦«}| ¦«}|jdkr(|jjr|j  
||¦«Œz|jdkr ŒˆdS)Nr{r=rV) ÚitemsÚis_cpurrrSÚ full_tensorrìr*rxry)r
s rÚ_sync_fsdp2_params_to_vllmz._UnslothGRPOTrainer._sync_fsdp2_params_to_vllmÞà!Ÿ<š<™>œ>ð ð ‰KˆDŒ|ð
Ÿš¥¤¨fÑ!5Ô!5Ñ6Ø×'ˆŒ~ Ò)¨dÔ.>Ô.NÐÔ ×3°D¸%Ñ :Òàøð ð r„códSr )r
rs rÚ_move_model_to_vllmz'_UnslothGRPOTrainer._move_model_to_vllmís¸4¸4r„Úgeneration_batchcó@t|d¦«r>t|jjjjdd¦«r|j ¦«|jjrdnd}|dkr·|j j
|j z}|j |zdks|j
€^| |¦«}t|¦«}t!|¦«}t#||j j
¦«}d|D¦«|_
|j
|j |j j
z}|xj dz
c_ n| |¦«}t|d¦«r]t|jjjjdd¦«r8|j t&j d d¦«¦«|S)
NrƒÚenable_sleep_modeFr2r3rcó,g|]}t|¦«ŒSr )rU)r Úbatchs rrz7_UnslothGRPOTrainer._prepare_inputs.<locals>.<listcomp> s"Ð(mÐ(mÐ(mÐQVÕ)EÀeÑ)LÔ)LÐ(mÐ(mÐ(mr„rmÚVLLM_SLEEP_MODE)r rVÚ
llm_engineÚ vllm_configÚ model_configÚwake_upr$rrrorpÚ_generate_and_score_completionsrPrOrQÚsleeprCr!)r
ÚmodeÚgenerate_everyÚgeneration_batchesÚinputss rÚ_prepare_inputsz#_UnslothGRPOTrainer._prepare_inputsïsõ 4˜Ñ Ô ðd”hÔBÐDWÐY^Ñ Œx×ÒÑœ*Ô9ˆwˆw°6ˆØ 7Š?ˆ!œYÔ;¸dÔ>QÑQˆŒz˜*¨aÒ/°4Ô3HÐ3Pà#'×#GÒ#GÐHXÑ#YÔ#YÐ Ý#=Ð>NÑ#OÔ#OÐ Ý#8Ð9IÑ#JÔ#JÐ Ý%6Ð7GÈÌÔIgÑ%hÔ%hÐ"Ø(mÐ(mÐZlÐ(mÑ(mÔ(mÔÔ*¨4¬:¸¼ Ô8VÑ+VÔWˆFØ ˆJŒJ˜!‰OˆJŒJˆJð×9Ð:JÑKˆFÝ Ñ Ô ð EÝt”xÔCÐEXÐZ_Ñ
EØrœzŸ~š~Ð.?Àш
r„c óÀ|jj}tjt |¦«t |j¦«|¬¦«}ddD¦«}ˆfd|D¦«}|j|d<tt|j|j |j
¦«¦«D]_\} \}
Š} t|| ¦«5t|
tj¦«rÓtd¦«r)dt||¦«D¦«} ˆfd| D¦«}
ndt||¦«D¦«}
|
d d
d d ¬
¦«}t!¦« |¦«}tj¦«5|
di|¤Žjdddf|dd| f<ddd¦«n #1swxYwYn@|
d|||dœ|¤Ž}d|D¦«}tj|tj|¬¦«|dd| f<ddd¦«n #1swxYwYŒatj|¦« d¬¦« ¦«r–tj|¦« d¬¦« d
¬¦«ddŠˆfd| ¦«D¦«}||d<||d<t7jd|d¦«t;|¦«}|S)NrÑcóg|]}|dv¯|Œ S))r9r:Úcompletion_idsr )r Úkeys rrz:_UnslothGRPOTrainer._calculate_rewards.<locals>.<listcomp>s#Ðb˜¨CÐ7aÐ,aÐ,aÐ,aÐ,aÐ,ar„rcó0i|]ŠˆfdD¦«ŒS)có g|]
}|Œ Sr r )r Úexamplers €rrzE_UnslothGRPOTrainer._calculate_rewards.<locals>.<dictcomp>.<listcomp>sø€Ð˜w sœ|ÐBr„r )r rr s @€rú
<dictcomp>z:_UnslothGRPOTrainer._calculate_rewards.<locals>.<dictcomp>s1øø€ÐSÀs˜ÐB¸6ÐSr„Ú
trainer_statecó"g|] \}}d||ziŒ
S)Úmessagesr ©r Úcs rrz:_UnslothGRPOTrainer._calculate_rewards.<locals>.<listcomp>*s%Ð#^Ð#^Ð#^¹D¸A¸q °Q±Ð$7Ð#^Ð#^Ð#^r„có<g|]}t|¦«dŒS)Útext)r&)r Úxr”s €rrz:_UnslothGRPOTrainer._calculate_rewards.<locals>.<listcomp>+s,ø€Ð kÐ kÐ kÐ]^Õ!4°QÐ8OÑ!PÔ!PÐQWÔ!XÐ kÐ kÐ kr„cóg|]
\}}||zŒ Sr r rs rrz:_UnslothGRPOTrainer._calculate_rewards.<locals>.<listcomp>-s Ð MÐ MÐ M©4¨1¨a  Q¡Ð MÐ MÐ Mr„ÚptTÚrightF©rÚreturn_tensorsÚpaddingÚ padding_sideÚadd_special_tokens)ÚpromptsÚ completionsr
có.g|]}||n tjŒS)rSÚnan)r Úrewards rrz:_UnslothGRPOTrainer._calculate_rewards.<locals>.<listcomp>9s$Ð)wÐ)wÐ)wÐZ`°FÐ4F¨&¨&ÍEÌIÐ)wÐ)wÐ)wr„©rrmrl)Úas_tuplecó(i|]\}}||ŒSr r )r rÚvalueÚ nan_row_idxs €rrz:_UnslothGRPOTrainer._calculate_rewards.<locals>.<dictcomp>@s$ø€Ð aÐ aÐ a¹Z¸SÀ%  e¨KÔ&8Ð aÐ aÐ ar„r9r:z=All reward functions returned None for the following kwargs: zI. Please ensure that at least one reward function returns a valid reward.r )r*rSrrÚstatergrqrrfrIr,rArhr4rr
ryrirsÚisnanÚallrmÚnonzerorôrWÚwarnr.)r
r r$r%Úcompletion_ids_listrÒÚrewards_per_funcr^Ú
reward_kwargsr
r“Úreward_func_namerÚtextsÚ
reward_inputsÚoutput_reward_funcÚrow_reward_kwargsr-r”r s ` @@€rÚ_calculate_rewardsz&_UnslothGRPOTrainer._calculate_rewardssøøøø€àÔÝ œ;¥s¨7¡|¤|µS¸Ô9JÑ5KÔ5KÐTZÐðcÐb˜v aœyÐØSÈdÐ
ð*.¬ˆ
&åKTÝ Ô! 4Ô#AÀ4ÔCYÑ L
ôL
ð rñ rÑ GˆG Ð4Ð6Fõ# 4Ð)9Ñ
rð
rݘk­2¬9Ñrݰ¬ÑNØ#^Ð#^ÅCÈÐQ\ÑD]ÔD]Ð#^Ñ#^Ô#^˜Ø kÐ kÐ kÐ kÐbjÐ kÑ kÔ k˜˜à MÐ Mµ3°wÀ Ñ3LÔ3LÐ MÑ MÔ M˜Ø$;Ð$;Ø"°4ÀÐT[Ðpuð%ñ%ô%%*¡G¤G×$;Ò$;¸MÑ$JÔ$JÔ[ð[Ø1<°Ð1MÐ1M¸}Ð1MÐ1MÔ1TÐUVÐUVÐUVÐXYÐUYÔ1ZШ¨¨A¨Ñ[ð[ð[ñ[ô[ð[ð[ð[ð[ð[ð[øøøð[ð[ð[ð[øð*5¨ð*Ø '°[ÐQdð*ð*Øhuð*ð*Ð*xÐ)wÐdvÐ)wÑ)wÔ)wÐ&å-2¬\Ð:LÕTYÔTaÐjpÐ-qÑ-qÔ-qÐ$ Q Q  *ð)
rð
rð
rñ
rô
rð
rð
rð
rð
rð
rð
røøøð