diff --git a/elementary/utils/json_utils.py b/elementary/utils/json_utils.py index d0e3ee3c1..35cdcd461 100644 --- a/elementary/utils/json_utils.py +++ b/elementary/utils/json_utils.py @@ -16,6 +16,11 @@ def try_load_json(value: Optional[Union[str, dict, list]]): return None +def normalize_scalar_token(raw: str, decoded: Any) -> str: + """Use decoded JSON strings; keep other scalar tokens as written.""" + return (decoded if isinstance(decoded, str) else raw).strip() + + def unpack_and_flatten_str_to_list(list_as_str: str) -> List[str]: """ if given a simple token like "marketing" -> return ["marketing"] @@ -33,7 +38,10 @@ def unpack_and_flatten_str_to_list(list_as_str: str) -> List[str]: if isinstance(list_unpacked, list): return list_unpacked - return [] # edge case of a string of an empty dict or IDK + if isinstance(list_unpacked, dict): + return [] # edge case of a string of an empty dict or IDK + # A JSON scalar such as 12345 or true is just a plain token. + return [normalize_scalar_token(list_as_str, list_unpacked)] def sum_lists(list_of_lists: List[List]) -> List: @@ -44,7 +52,7 @@ def sum_lists(list_of_lists: List[List]) -> List: def unpack_and_flatten_and_dedup_list_of_strings( - list_maybe_jsoned: Optional[Union[List[str], str]] + list_maybe_jsoned: Optional[Union[List[str], str]], ) -> List[str]: if not list_maybe_jsoned: return [] @@ -54,11 +62,16 @@ def unpack_and_flatten_and_dedup_list_of_strings( elif isinstance(list_maybe_jsoned, list): ret = sum_lists( [ - unpack_and_flatten_str_to_list(x) + unpack_and_flatten_str_to_list(x) if isinstance(x, str) else [x] for x in list_maybe_jsoned - if isinstance(x, str) ] ) + # Owners/tags need strings, but the general parser also handles sample rows. + ret = [ + item.strip() if isinstance(item, str) else json.dumps(item) + for item in ret + if isinstance(item, (str, int, float, bool)) + ] return list(set(ret)) diff --git a/elementary/utils/schema.py b/elementary/utils/schema.py index 068dd53f6..9e810d378 100644 --- a/elementary/utils/schema.py +++ b/elementary/utils/schema.py @@ -1,7 +1,7 @@ import json from typing import Union -from elementary.utils.json_utils import try_load_json +from elementary.utils.json_utils import normalize_scalar_token, try_load_json from elementary.utils.pydantic_shim import BaseModel @@ -25,6 +25,6 @@ def _load_var_to_list(var: Union[str, list]) -> list: loaded_var = try_load_json(var) if isinstance(loaded_var, dict): loaded_var = [json.dumps(loaded_var)] - if loaded_var is None: - loaded_var = [var] + if not isinstance(loaded_var, list): + loaded_var = [normalize_scalar_token(var, loaded_var)] return loaded_var diff --git a/tests/unit/utils/test_json_utils.py b/tests/unit/utils/test_json_utils.py new file mode 100644 index 000000000..3b51ce42c --- /dev/null +++ b/tests/unit/utils/test_json_utils.py @@ -0,0 +1,89 @@ +import pytest + +from elementary.utils.json_utils import ( + unpack_and_flatten_and_dedup_list_of_strings, + unpack_and_flatten_str_to_list, +) +from elementary.utils.schema import ExtendedBaseModel + + +@pytest.mark.parametrize("token", ["12345", "1.5", "true", "false"]) +def test_unpack_and_flatten_keeps_json_scalar_tokens(token): + assert unpack_and_flatten_str_to_list(token) == [token] + assert unpack_and_flatten_and_dedup_list_of_strings(token) == [token] + + +def test_unpack_and_flatten_still_handles_lists_and_commas(): + assert unpack_and_flatten_str_to_list('["a", "b"]') == ["a", "b"] + assert unpack_and_flatten_str_to_list("a, b") == ["a", "b"] + assert unpack_and_flatten_str_to_list("{}") == [] + + +@pytest.mark.parametrize("token", ["12345", "1.5", "true"]) +def test_load_var_to_list_keeps_json_scalar_tokens(token): + assert ExtendedBaseModel._load_var_to_list(token) == [token] + + +def test_load_var_to_list_still_handles_lists_and_strings(): + assert ExtendedBaseModel._load_var_to_list('["a"]') == ["a"] + assert ExtendedBaseModel._load_var_to_list("prod") == ["prod"] + + +@pytest.mark.parametrize( + "token, expected", + [ + ('"@alice"', ["@alice"]), + ('"prod"', ["prod"]), + ('" prod "', ["prod"]), + (" prod ", ["prod"]), + (" 12345 ", ["12345"]), + (" true ", ["true"]), + ], +) +def test_scalar_tokens_match_in_report_and_alerts(token, expected): + assert ExtendedBaseModel._load_var_to_list(token) == expected + assert unpack_and_flatten_and_dedup_list_of_strings(token) == expected + + +@pytest.mark.parametrize("value", ["[12345]", '[1, "a"]', [1, "a"]]) +def test_alert_numeric_list_items_are_strings(value): + from elementary.utils.strings import prettify_and_dedup_list + + result = unpack_and_flatten_and_dedup_list_of_strings(value) + expected = ["12345"] if value == "[12345]" else ["1", "a"] + assert sorted(result) == expected + assert prettify_and_dedup_list(result) == ", ".join(expected) + + +def test_general_list_parser_preserves_sample_rows(): + rows = [{"a": 1}, {"b": True}] + assert unpack_and_flatten_str_to_list('[{"a": 1}, {"b": true}]') == rows + assert ExtendedBaseModel._load_var_to_list(rows) is rows + + +def test_existing_dict_and_comma_policies_are_preserved(): + assert unpack_and_flatten_and_dedup_list_of_strings('{"a": 1}') == [] + assert ExtendedBaseModel._load_var_to_list('{"a": 1}') == ['{"a": 1}'] + assert sorted(unpack_and_flatten_and_dedup_list_of_strings("a, b")) == ["a", "b"] + assert ExtendedBaseModel._load_var_to_list("a, b") == ["a, b"] + assert ExtendedBaseModel._load_var_to_list(["a", "a", "b"]) == ["a", "a", "b"] + + +@pytest.mark.parametrize("value", ["[12345]", '[1, "a"]', [1, "a"]]) +def test_report_owner_lists_match_alert_strings(value): + from elementary.monitor.fetchers.models.schema import ArtifactSchema + + expected = ["12345"] if value == "[12345]" else ["1", "a"] + assert ArtifactSchema(owners=value).owners == expected + assert sorted(unpack_and_flatten_and_dedup_list_of_strings(value)) == expected + + +def test_dependency_nodes_keep_commas_order_and_duplicates(): + from elementary.monitor.fetchers.models.schema import ExposureSchema + + assert ExposureSchema(depends_on_nodes="a, b").depends_on_nodes == ["a, b"] + assert ExposureSchema(depends_on_nodes=["a", "a", "b"]).depends_on_nodes == [ + "a", + "a", + "b", + ]