33 lines
805 B
JSON
33 lines
805 B
JSON
|
{
|
||
|
|
"model_name": "unsloth/Meta-Llama-3.1-8B-Instruct-bnb-4bit",
|
||
|
|
"max_seq_length": 2048,
|
||
|
|
"dtype": null,
|
||
|
|
"load_in_4bit": true,
|
||
|
|
"hf_token": null,
|
||
|
|
"num_epochs": 3,
|
||
|
|
"batch_size": 4,
|
||
|
|
"learning_rate": 0.0002,
|
||
|
|
"weight_decay": 0.01,
|
||
|
|
"warmup_steps": 5,
|
||
|
|
"max_steps": 60,
|
||
|
|
"gradient_accumulation_steps": 4,
|
||
|
|
"lr_scheduler_type": "linear",
|
||
|
|
"seed": 3407,
|
||
|
|
"model_output_dir": "./models/styling",
|
||
|
|
"data_output_dir": "./data/processed/styling/formal",
|
||
|
|
"dataset_path": "./data/processed/styling/formal",
|
||
|
|
"style_instruction": "Rewrite the following text in a formal style",
|
||
|
|
"lora_r": 16,
|
||
|
|
"lora_alpha": 16,
|
||
|
|
"lora_dropout": 0,
|
||
|
|
"target_modules": [
|
||
|
|
"q_proj",
|
||
|
|
"k_proj",
|
||
|
|
"v_proj",
|
||
|
|
"o_proj",
|
||
|
|
"gate_proj",
|
||
|
|
"up_proj",
|
||
|
|
"down_proj"
|
||
|
|
],
|
||
|
|
"output_dir": "./outputs"
|
||
|
|
}
|