j-TddlmZmZmZmZmZmZmZddlm Z ddl Z ddl Z ddl m Z ddlmZddlmZmZmZddlmZmZedddddfd ed ed eed eed eedeedeedeeedefdZeddddfd ed ed eed eed eedeedeedefdZde fdZdddefde ddd ed ed eedeedeed eedeeeeffdZ d!deeeefdedeedeed eedeedeefdZdefdZdedeeeefdeegeffd Zy)")DictAnyTupleCallableListOptionalIO) ModuleTypeN)Language)SimpleFrozenList) dict_to_dot dot_to_dictmatcher_for_regex_patterns)setup_default_console_loggerLoggerT project_name task_nameremove_config_valuesmodel_log_intervallog_dataset_dir log_best_dirlog_latest_dirlog_custom_statsreturnc  t tjtjfdtdt dt dt ttttfgdftgdfff f d }|S)a0Creates a logger that interoperates with the ClearML framework. Args: project_name (str): The name of the project in the ClearML interface. The project will be created automatically if it doesn't exist yet. task_name (str): The name of the ClearML task. A task is an experiment that lives inside a project. Can be non-unique. remove_config_values (List[str]): A list of values to exclude from the config before it is uploaded to ClearML. Defaults to []. model_log_interval (Optional[int]): Steps to wait between logging model checkpoints to the ClearML dasboard (default: `None`). Will have no effect without also setting `log_best_dir` or `log_latest_dir`. Defaults to None. log_dataset_dir (Optional[str]): Directory containing the dataset to be logged and versioned as a ClearML Dataset. Defaults to None. log_best_dir (Optional[str]): Directory containing the best trained model as saved by spaCy, to be logged and versioned as a ClearML artifact. Defaults to None. log_latest_dir (Optional[str]): Directory containing the latest trained model as saved by spaCy, to be logged and versioned as a ClearML artifact. Defaults to None. log_custom_stats (Optional[List[str]]): A list of regular expressions that will be applied to the info dictionary passed to the logger. Statistics and metrics that match these regexps will be automatically logged. Defaults to None. Returns: LoggerT: Logger instance. nlpstdoutstderrrNc  t t | \dtttt ff  fd }fd}||fS)Ninfoc Bt|t|yN)_log_step_clearml_log_custom_stats) r! best_modelclearml last_modelrr match_statrtasks _/home/spazio/kingdom_calls/work5/transenv/lib/python3.12/site-packages/spacy_loggers/clearml.pylog_stepz9clearml_logger_v2..setup_logger..log_step?s. "  gtZ 8ctyr#_finalize_clearmlr*sr+finalizez9clearml_logger_v2..setup_logger..finalizeKs d #r-)r_setup_clearmlrrstrr)rrrr,r2r&r(r)r*r'rrrrrrrrs @@@@r+ setup_loggerz'clearml_logger_v2..setup_logger0si00@A '5         ( $j* 98DcN3 9 9 $!!r- _import_clearmlsysrrr r rrrr4r) rrrrrrrrr5r's ```````` @r+clearml_logger_v2r9 syBG%(JJSZZ" "!"8:" xc3h($./"d(1CC D""@ r-c ttjtjfdtdt dt dt ttttfgdftgdffffd }|S)a4Creates a logger that interoperates with the ClearML framework. Args: project_name (str): The name of the project in the ClearML interface. The project will be created automatically if it doesn't exist yet. task_name (str): The name of the ClearML task. A task is an experiment that lives inside a project. Can be non-unique. remove_config_values (List[str]): A list of values to exclude from the config before it is uploaded to ClearML. Defaults to []. model_log_interval (Optional[int]): Steps to wait between logging model checkpoints to the ClearML dasboard (default: `None`). Will have no effect without also setting `log_best_dir` or `log_latest_dir`. Defaults to None. log_dataset_dir (Optional[str]): Directory containing the dataset to be logged and versioned as a ClearML Dataset. Defaults to None. log_best_dir (Optional[str]): Directory containing the best trained model as saved by spaCy, to be logged and versioned as a ClearML artifact. Defaults to None. log_latest_dir (Optional[str]): Directory containing the latest trained model as saved by spaCy, to be logged and versioned as a ClearML artifact. Defaults to None. Returns: LoggerT: Logger instance. rrrrNc  t|||\t | \ dtttt ff  fd } fd}||fS)Nr!c 8|t|yr#)r$)r!r&console_log_stepr(rrrr*s r+r,z9clearml_logger_v1..setup_logger..log_steps( T " " r-c*tyr#r/)console_finalizer*sr+r2z9clearml_logger_v1..setup_logger..finalizes   d #r-)rr3rrr4r)rrrr,r2r&r?r=r(r*r'rrrrrrrs @@@@@r+r5z'clearml_logger_v1..setup_loggertsy.J . **(6         ( $j* 8DcN3   $!!r-r6) rrrrrrrr5r's ``````` @r+clearml_logger_v1r@Tsw<G%(JJSZZ!" !"!!"8:!" xc3h($./"d(1CC D!"!"F r-c^ ddl}ddlm}m}m}|S#t$r}t d|d}~wwxYw)Nr)TaskDataset OutputModelzThe 'clearml' library could not be found - did you install it? Alternatively, specify the 'ConsoleLogger' in the 'training.logger' config section, instead of the 'ClearMLLogger'.)r'rBrCrD ImportError)r'rBrCrDexcs r+r7r7sC  76 N  P   s , ',r'rr cp|jj}t|} |D]} | | =t| }|jj ||d} |j D]\} } | j| | |r{|jj|tjj|}|j||jd| jd|j d|r|j#| dd }nd}|r|j#| dd }nd}| ||fS) NT)rr output_uri)name)dataset_project dataset_name) auto_uploadzCreated Dataset ID)rIvaluespaCyz Best Model)r* frameworkrIz Last Model)config interpolaterrrBinititemsconnectrCcreateospathbasename add_filesr2set_user_propertiesidrD)r'rrrrrrrrP config_dotfieldr*config_sectionsubconfig_or_valuedatasetr&r(s r+r3r3s]ZZ # # %FV$J% u   $F <<  !  D /5lln>** 'n =>//((())/:)  /*T*   ,   ((|)  ((|)  Z ''r-r!r*r&r(c|y|jd}|jd}|jd} |r&|jjdd|d|| rD| jD]1\} } |jjd| d| |d| 3t |t r|jD]\} } t | t rJt | } | jD]+\} }|jj| | |d|-`t | ttfsw|jj| | |d| |ru|jd rc|d|zd k(rW|dd k7rN|r|J|j|d d |r1|dt|dd k(r|J|j|d d yyyyyyy)Nscore other_scoreslossesScorestep) iterationrMloss_titleseriesrgrM output_pathrFr() weights_pathauto_delete_filetarget_filename checkpointsr&) get get_logger report_scalarrS isinstancedictrfloatintupdate_weights_packagemax)r!r*r&r(rrrrbrcrdmetric metric_valuesub_metrics_dict sub_metricsub_metric_values r+r$r$s8 | HHW E88N+L XXh F  '' WV E ( $*LLN  FL OO  + +fX&vh'v," ,  ,%%1$6$6$8  FL,-#.|#< &++- $OO%33$)"&v,. 4 L5#,7!//"6l& 0! .dhh}5 <, , 1d6la6G!---11!/%*$02 W T-5H1I!1L L!---11!-%*$02!M|7H 16r-cH|jd|jy)NT)wait_for_uploads)flushcloser1s r+r0r0&sJJJ%JJLr-matcherc|N|jD]:\}}||s|jjd|d||d|<yy)Nrhrfri)rSrrrs)r*r!rkvs r+r%r%+se JJL DAqqz!//!!+"1#;"6l 0 r-)NNNNN) typingrrrrrrr typesr rVr8spacyr spacy.utilr utilrrrrrr4rwr9r@r7r3r$r0boolr%r-r+rsAAA 'FF7'7&8(,%)"&$(,0CCCs)C! C c] C 3- CSMCtCy)C CT'7&8(,%)"&$(CCCs)C! C c] C 3- CSMC CL  (&*"&$(&6&83( 3( 3(3( 3( c] 3( 3- 3(SM3(s)3( 3S=3(r!% $(,"&$(D 4S> "D D D D ! D 3- DSMDNC   d38n- 8@#8M r-