Tangledup-ai/lang-agent
6
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add tool use for default eval

Browse Source
This commit is contained in:
goulustis
2025-10-29 16:17:20 +08:00
parent 7031444e2e
commit 285f8975c6
1 changed files with 3 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
lang_agent/eval/validator.py
View File

@@ -65,10 +65,10 @@ class Validator:
return response.content.upper() == "CORRECT" return response.content.upper() == "CORRECT"
def val_tool_use(self, inputs:dict, outputs:dict, reference_outputs:dict)->bool: def val_tool_use(self, inputs:dict, outputs:dict, reference_outputs:dict)->float:
tool_uses:List[str] = reference_outputs.get("tool_use") tool_uses:List[str] = reference_outputs.get("tool_use")
if tool_uses is None: if tool_uses is None:
return True return 1.0
tool_msgs = [e for e in outputs["output"] if isinstance(e, ToolMessage)] tool_msgs = [e for e in outputs["output"] if isinstance(e, ToolMessage)]
@@ -102,7 +102,7 @@ class Validator:
def get_val_fnc(self, dataset_name:str)->List[Callable]: def get_val_fnc(self, dataset_name:str)->List[Callable]:
return self.dict_corr_map.get(dataset_name, [self.default_correct]) return self.dict_corr_map.get(dataset_name, [self.default_correct, self.val_tool_use])
def get_inp_fnc(self,dataset_name:str)->Callable: def get_inp_fnc(self,dataset_name:str)->Callable: