Skip to content

fix(gooddata-eval): drop dead 'skills' fallback spelling in set_skills parsing #1861

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
cobanfurkanx wants to merge 1 commit into
base: master
Choose a base branch
from
+11 −14
Open

fix(gooddata-eval): drop dead 'skills' fallback spelling in set_skills parsing #1861

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 2 additions & 5 deletions packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -298,18 +298,15 @@ def _replace(match: re.Match) -> str: # type: ignore[type-arg]
def _set_skills_declarations(tool_call_events: list[ToolCallEvent]) -> list[list[str]]:
"""Every set_skills declaration in these events, in call order.

`skill_names` is the key the tool declares; `skills` is a legacy spelling kept as a
fallback. A call carrying neither is treated as declaring an empty list, which is what
the platform would do with one.
`skill_names` is the key the tool declares. A call carrying no skill_names is treated as
declaring an empty list, which is what the platform would do with one.
"""
declarations: list[list[str]] = []
for tc in tool_call_events:
if tc.function_name != "set_skills":
continue
args = tc.parsed_arguments() or {}
names = args.get("skill_names")
if names is None:
names = args.get("skills")
declarations.append(list(names or []))
return declarations

Expand Down
18 changes: 9 additions & 9 deletions packages/gooddata-eval/tests/test_agentic_conversation.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -199,7 +199,7 @@ def test_run_agentic_conversation_single_turn():
tc.result_ts = None
tc.index = None
tc.function_name = "set_skills"
tc.parsed_arguments = lambda: {"skills": ["visualization"]}
tc.parsed_arguments = lambda: {"skill_names": ["visualization"]}
mock_chat_result = MagicMock()
mock_chat_result.text_response = "Here is your visualization"
mock_chat_result.created_visualizations = [MagicMock()]
Expand Down Expand Up @@ -243,7 +243,7 @@ def test_run_agentic_conversation_uses_initial_conversation_id():
tc.result_ts = None
tc.index = None
tc.function_name = "set_skills"
tc.parsed_arguments = lambda: {"skills": ["visualization"]}
tc.parsed_arguments = lambda: {"skill_names": ["visualization"]}
mock_chat_result.tool_call_events = [tc]
mock_chat_result.reasoning_step_events = []
mock_chat_result.turn_wall_clock_sec = None
Expand Down Expand Up @@ -285,7 +285,7 @@ def test_run_agentic_conversation_creates_and_deletes_conversation():
tc.result_ts = None
tc.index = None
tc.function_name = "set_skills"
tc.parsed_arguments = lambda: {"skills": ["visualization"]}
tc.parsed_arguments = lambda: {"skill_names": ["visualization"]}
mock_chat_result.tool_call_events = [tc]
mock_chat_result.reasoning_step_events = []
mock_chat_result.turn_wall_clock_sec = None
Expand Down Expand Up @@ -680,7 +680,7 @@ def test_run_agentic_conversation_treats_alert_proposal_as_a_clarification():
"text_response": None,
"alertProposals": [{"cta": "Should I create this alert?", "recipients": [{"email": "a@b.com"}]}],
"toolCallEvents": [
{"functionName": "set_skills", "functionArguments": '{"skills": ["alert"]}', "result": None},
{"functionName": "set_skills", "functionArguments": '{"skill_names": ["alert"]}', "result": None},
{"functionName": "prepare_metric_alert_proposal", "functionArguments": "{}", "result": None},
],
}
Expand Down Expand Up @@ -883,7 +883,7 @@ def test_run_agentic_conversation_accumulates_reasoning_steps_across_turns():
tc.result_ts = None
tc.index = None
tc.function_name = "set_skills"
tc.parsed_arguments = lambda: {"skills": ["visualization"]}
tc.parsed_arguments = lambda: {"skill_names": ["visualization"]}

turn1_result = MagicMock()
turn1_result.text_response = "Here is your visualization"
Expand Down Expand Up @@ -940,7 +940,7 @@ def test_evaluate_agentic_conversation_returns_reasoning_steps_on_pass():
tc.result_ts = None
tc.index = None
tc.function_name = "set_skills"
tc.parsed_arguments = lambda: {"skills": ["visualization"]}
tc.parsed_arguments = lambda: {"skill_names": ["visualization"]}
chat_result = MagicMock()
chat_result.text_response = "Here is your visualization"
chat_result.created_visualizations = [MagicMock()]
Expand Down Expand Up @@ -1016,7 +1016,7 @@ def test_evaluate_agentic_conversation_attaches_reasoning_steps_to_exception_on_
tc.result_ts = None
tc.index = None
tc.function_name = "set_skills"
tc.parsed_arguments = lambda: {"skills": ["other_skill"]}
tc.parsed_arguments = lambda: {"skill_names": ["other_skill"]}
chat_result = MagicMock()
chat_result.text_response = "Here is something else"
chat_result.created_visualizations = None
Expand Down Expand Up @@ -1221,7 +1221,7 @@ def test_run_agentic_conversation_sums_the_reasoning_steps_of_every_turn():
"alertProposals": [{"cta": "Should I create this alert?", "recipients": [{"email": "a@b.com"}]}],
"reasoningStepCount": 2,
"toolCallEvents": [
{"functionName": "set_skills", "functionArguments": '{"skills": ["alert"]}', "result": None},
{"functionName": "set_skills", "functionArguments": '{"skill_names": ["alert"]}', "result": None},
{"functionName": "prepare_metric_alert_proposal", "functionArguments": "{}", "result": None},
],
}
Expand Down Expand Up @@ -1267,7 +1267,7 @@ def test_conversation_writes_the_turn_step_and_clarification_counts_to_langfuse(
"alertProposals": [{"cta": "Should I create this alert?", "recipients": [{"email": "a@b.com"}]}],
"reasoningStepCount": 2,
"toolCallEvents": [
{"functionName": "set_skills", "functionArguments": '{"skills": ["alert"]}', "result": None},
{"functionName": "set_skills", "functionArguments": '{"skill_names": ["alert"]}', "result": None},
{"functionName": "prepare_metric_alert_proposal", "functionArguments": "{}", "result": None},
],
}
Expand Down