48 lines
4.7 KiB
Plaintext
48 lines
4.7 KiB
Plaintext
|
o
|
|||
|
|
üz°hç ã @ sŽ d dl Z d dlZd dlmZ d dlmZmZmZ d dlZe e
|
|||
|
|
¡Z G dd„ dƒZ de
|
|||
|
|
dee
|
|||
|
|
ef fdd „Zd dee
|
|||
|
|
dee
|
|||
|
|
ef fd
|
|||
|
|
d „ZdS )
|
|||
|
|
é N)ÚPath)ÚDictÚAnyÚOptionalc @ sè e Zd ZdZdd„ Zdedeeef fdd„Zdeeef d eeef deeef fd
|
|||
|
|
d „Z d eeef d
|
|||
|
|
edefdd„Z
|
|||
|
|
dd eeef d
|
|||
|
|
ededefdd„Z d dede j
|
|||
|
|
fdd„Zd dee d eeeef deeef fdd„ZdS ) Ú
|
|||
|
|
ConfigManagerzHManages configuration loading from YAML files and command-line argumentsc C s
|
|||
|
|
i | _ d S ©N)Úconfig)Úself© r
|
|||
|
|
úD/workspace/DS-LLM-TEMPLATE-FINETUNING/utils/config/config_manager.pyÚ__init__ s
|
|||
|
|
zConfigManager.__init__Ú config_pathÚreturnc C sj t |ƒ}| ¡ std|› �ƒ‚t|ddd��
|
|||
|
|
}t |¡}W d ƒ n1 s&w Y t d|› �¡ |S )z!Load configuration from YAML filez Configuration file not found: Úrzutf-8)ÚencodingNzLoaded configuration from: )r ÚexistsÚFileNotFoundErrorÚopenÚyamlÚ safe_loadÚloggerÚinfo)r r
|
|||
|
|
Z config_fileÚfr r
|
|||
|
|
r
|
|||
|
|
r Úload_yaml_config s ÿz ConfigManager.load_yaml_configÚ yaml_configÚcli_argsc C s4 | ¡ }| ¡ D ]\}}|dur| |||¡ q|S )z4Merge YAML configuration with command-line argumentsN)ÚcopyÚitemsÚ_set_nested_value)r r r Ú
|
|||
|
|
merged_configÚkeyÚvaluer
|
|||
|
|
r
|
|||
|
|
r Ú
|
|||
|
|
merge_configs s €zConfigManager.merge_configsr r r! c C sH | d¡}|}|dd… D ]}||vri ||< || }q
|
|||
|
|
|||d < dS )zMSet a nested value in config using dot notation (e.g., 'training.batch_size')Ú.Néÿÿÿÿ)Úsplit)r r r r! ÚkeysÚcurrentÚkr
|
|||
|
|
r
|
|||
|
|
r r ' s
|
|||
|
|
|
|||
|
|
z ConfigManager._set_nested_valueNÚdefaultc C s> | d¡}|}|D ]}t|tƒr||v r|| }q | S |S )z1Get a nested value from config using dot notationr# )r% Ú
|
|||
|
|
isinstanceÚdict)r r r r) r& r' r( r
|
|||
|
|
r
|
|||
|
|
r Úget_config_value3 s
|
|||
|
|
|
|||
|
|
z ConfigManager.get_config_valueú Pipeline with YAML configurationÚ descriptionc C s0 t j|d�}|jdtdd� |jdddgdd � |jd
|
|||
|
|
td d� |jd td
|
|||
|
|
d� |jdg d¢dd � |jdtdd� |jdtdd� |jdtdd� |jdtdd� |jdtdd� |jdtd d� |jd td d� |jd td d� |jd!td"d� |jd#td$d� |jd%g d&¢d'd � |jd(g d)¢d*d � |S )+z6Create argument parser that supports YAML config files)r. z--configz Path to YAML configuration file)ÚtypeÚhelpz
|
|||
|
|
--data-sourceZ huggingfaceZcustomz Data source)Úchoicesr0 z--dataset-namezHuggingFace dataset namez --data-pathzPath to custom data filez
|
|||
|
|
--data-format)ZjsonlZcsvÚjsonz Data formatz
|
|||
|
|
--input-fieldzInput field namez
|
|||
|
|
--label-fieldzLabel field namez
|
|||
|
|
--max-sampleszMaximum samples to processz --output-dirzOutput directoryz --model-namez
|
|||
|
|
Model namez --max-lengthzMaximum sequence lengthz --num-epochszNumber of epochsz --batch-sizez
|
|||
|
|
Batch sizez--learning-ratez
|
|||
|
|
Learning ratez --model-pathzPath to saved modelz--device)ÚautoÚcudaÚcpuZDevicez --log-level)ÚDEBUGÚINFOZWARNINGZERRORz Log level)ÚargparseÚArgumentParserÚ add_argumentÚstrÚintÚfloat)r r. Úparserr
|
|||
|
|
r
|
|||
|
|
r Úcreate_argparser_with_yaml@ s& z(ConfigManager.create_argparser_with_yamlc C sF i }|r | |¡}|p i }dd„ | ¡ D ƒ}| ||¡}t d¡ |S )z-Load YAML config and merge with CLI argumentsc S s i | ] \}}|d ur||“qS r r
|