diff --git a/cdisc_rules_engine/check_operators/dataframe_operators.py b/cdisc_rules_engine/check_operators/dataframe_operators.py index ddff41a59..19fb1e3aa 100644 --- a/cdisc_rules_engine/check_operators/dataframe_operators.py +++ b/cdisc_rules_engine/check_operators/dataframe_operators.py @@ -841,7 +841,9 @@ def starts_with(self, other_value): if self.value.is_series(comparison_data): # need to convert series to tuple to make startswith operator work correctly comparison_data: Tuple[str] = tuple(comparison_data) - results = self.value[target].str.startswith(comparison_data) + # na=False keeps the result a clean boolean Series when target values are + # missing, so negating operators (does_not_start_with) stay null-safe. + results = self.value[target].str.startswith(comparison_data, na=False) return results @log_operator_execution @@ -854,9 +856,21 @@ def ends_with(self, other_value): if self.value.is_series(comparison_data): # need to convert series to tuple to make endswith operator work correctly comparison_data: Tuple[str] = tuple(comparison_data) - results = self.value[target].str.endswith(comparison_data) + # na=False keeps the result a clean boolean Series when target values are + # missing, so negating operators (does_not_end_with) stay null-safe. + results = self.value[target].str.endswith(comparison_data, na=False) return results + @log_operator_execution + @type_operator(FIELD_DATAFRAME) + def does_not_start_with(self, other_value): + return ~self.starts_with(other_value) + + @log_operator_execution + @type_operator(FIELD_DATAFRAME) + def does_not_end_with(self, other_value): + return ~self.ends_with(other_value) + @log_operator_execution @type_operator(FIELD_DATAFRAME) def has_equal_length(self, other_value: dict): diff --git a/resources/schema/rule-merged/Operator.json b/resources/schema/rule-merged/Operator.json index 1c1f13a64..073da9d34 100644 --- a/resources/schema/rule-merged/Operator.json +++ b/resources/schema/rule-merged/Operator.json @@ -131,6 +131,16 @@ "required": ["operator", "value"], "type": "object" }, + { + "properties": { + "operator": { + "const": "does_not_end_with", + "markdownDescription": "\nComplement of `ends_with`. Returns True when the target does NOT end with the value.\n\n> DOMAIN not ending with 'FOOBAR'\n\n```yaml\n- name: \"DOMAIN\"\n operator: \"does_not_end_with\"\n value: \"FOOBAR\"\n```\n" + } + }, + "required": ["operator", "value"], + "type": "object" + }, { "properties": { "operator": { @@ -151,6 +161,16 @@ "required": ["operator", "ordering", "value", "within"], "type": "object" }, + { + "properties": { + "operator": { + "const": "does_not_start_with", + "markdownDescription": "\nComplement of `starts_with`. Returns True when the target does NOT start with the value.\n\n> DOMAIN not beginning with 'AP'\n\n```yaml\n- name: \"DOMAIN\"\n operator: \"does_not_start_with\"\n value: \"AP\"\n```\n" + } + }, + "required": ["operator", "value"], + "type": "object" + }, { "properties": { "operator": { diff --git a/resources/schema/rule/Operator.json b/resources/schema/rule/Operator.json index 61ed71f2f..19f08743b 100644 --- a/resources/schema/rule/Operator.json +++ b/resources/schema/rule/Operator.json @@ -72,6 +72,11 @@ "required": ["operator", "value"], "type": "object" }, + { + "properties": { "operator": { "const": "does_not_end_with" } }, + "required": ["operator", "value"], + "type": "object" + }, { "properties": { "operator": { "const": "does_not_equal_string_part" } }, "required": ["operator", "value", "regex"], @@ -84,6 +89,11 @@ "required": ["operator", "ordering", "value", "within"], "type": "object" }, + { + "properties": { "operator": { "const": "does_not_start_with" } }, + "required": ["operator", "value"], + "type": "object" + }, { "properties": { "operator": { "const": "empty" } }, "required": ["operator"], diff --git a/resources/schema/rule/Operator.md b/resources/schema/rule/Operator.md index 4685cac0f..d62ba944c 100644 --- a/resources/schema/rule/Operator.md +++ b/resources/schema/rule/Operator.md @@ -267,6 +267,30 @@ Substring matching value: "FOOBAR" ``` +### does_not_start_with + +Complement of `starts_with`. Returns True when the target does NOT start with the value. + +> DOMAIN not beginning with 'AP' + +```yaml +- name: "DOMAIN" + operator: "does_not_start_with" + value: "AP" +``` + +### does_not_end_with + +Complement of `ends_with`. Returns True when the target does NOT end with the value. + +> DOMAIN not ending with 'FOOBAR' + +```yaml +- name: "DOMAIN" + operator: "does_not_end_with" + value: "FOOBAR" +``` + ### prefix_equal_to True if the `prefix` number of characters beginning a string in `name` match the string in `value` diff --git a/tests/unit/test_check_operators/test_string_comparison.py b/tests/unit/test_check_operators/test_string_comparison.py index 1ec3277d3..324ce0ad4 100644 --- a/tests/unit/test_check_operators/test_string_comparison.py +++ b/tests/unit/test_check_operators/test_string_comparison.py @@ -134,6 +134,121 @@ def test_ends_with(data, comparator, dataset_type, expected_result): assert result.equals(df.convert_to_series(expected_result)) +@pytest.mark.parametrize( + "data,comparator,dataset_type,expected_result", + [ + ( + {"target": ["Att", "Btt", "Ctt"], "VAR2": ["A", "B", "D"]}, + "VAR2", + PandasDataset, + [False, False, True], + ), + ( + {"target": ["Att", "Btt", "Ctt"], "VAR2": ["A", "B", "D"]}, + "VAR2", + DaskDataset, + [False, False, True], + ), + ], +) +def test_does_not_start_with(data, comparator, dataset_type, expected_result): + df = dataset_type.from_dict(data) + dataframe_type = DataframeType({"value": df}) + result = dataframe_type.does_not_start_with( + {"target": "target", "comparator": comparator} + ) + assert result.equals(df.convert_to_series(expected_result)) + + +@pytest.mark.parametrize( + "data,comparator,dataset_type,expected_result", + [ + ( + {"target": ["Att", "Btt", "Ctt"], "VAR2": ["tt", "B", "D"]}, + "VAR2", + PandasDataset, + [False, False, False], + ), + ( + {"target": ["Att", "Btt", "Ctt"], "VAR2": ["tt", "B", "D"]}, + "VAR2", + DaskDataset, + [False, False, False], + ), + ], +) +def test_does_not_end_with(data, comparator, dataset_type, expected_result): + df = dataset_type.from_dict(data) + dataframe_type = DataframeType({"value": df}) + result = dataframe_type.does_not_end_with( + {"target": "target", "comparator": comparator} + ) + assert result.equals(df.convert_to_series(expected_result)) + + +# value_is_literal=True: comparator is a literal string, not a column name. +@pytest.mark.parametrize( + "data,comparator,dataset_type,expected", + [ + ({"target": ["Att", "Btt", "Axx"]}, "A", PandasDataset, [True, False, True]), + ({"target": ["Att", "Btt", "Axx"]}, "A", DaskDataset, [True, False, True]), + ], +) +def test_starts_with_literal(data, comparator, dataset_type, expected): + df = dataset_type.from_dict(data) + dataframe_type = DataframeType({"value": df}) + opts = {"target": "target", "comparator": comparator, "value_is_literal": True} + assert dataframe_type.starts_with(opts).equals(df.convert_to_series(expected)) + assert dataframe_type.does_not_start_with(opts).equals( + df.convert_to_series([not b for b in expected]) + ) + + +@pytest.mark.parametrize( + "data,comparator,dataset_type,expected", + [ + ({"target": ["Att", "Btt", "Cxx"]}, "tt", PandasDataset, [True, True, False]), + ({"target": ["Att", "Btt", "Cxx"]}, "tt", DaskDataset, [True, True, False]), + ], +) +def test_ends_with_literal(data, comparator, dataset_type, expected): + df = dataset_type.from_dict(data) + dataframe_type = DataframeType({"value": df}) + opts = {"target": "target", "comparator": comparator, "value_is_literal": True} + assert dataframe_type.ends_with(opts).equals(df.convert_to_series(expected)) + assert dataframe_type.does_not_end_with(opts).equals( + df.convert_to_series([not b for b in expected]) + ) + + +# Missing target values must stay null-safe: starts_with/ends_with return a clean +# boolean Series (missing -> False) so the negating operators don't crash on `~`. +@pytest.mark.parametrize("dataset_type", [PandasDataset, DaskDataset]) +def test_does_not_start_with_is_null_safe(dataset_type): + df = dataset_type.from_dict({"target": ["Att", None, "Ctt"]}) + dataframe_type = DataframeType({"value": df}) + opts = {"target": "target", "comparator": "A", "value_is_literal": True} + assert dataframe_type.starts_with(opts).equals( + df.convert_to_series([True, False, False]) + ) + assert dataframe_type.does_not_start_with(opts).equals( + df.convert_to_series([False, True, True]) + ) + + +@pytest.mark.parametrize("dataset_type", [PandasDataset, DaskDataset]) +def test_does_not_end_with_is_null_safe(dataset_type): + df = dataset_type.from_dict({"target": ["Att", None, "Cxx"]}) + dataframe_type = DataframeType({"value": df}) + opts = {"target": "target", "comparator": "tt", "value_is_literal": True} + assert dataframe_type.ends_with(opts).equals( + df.convert_to_series([True, False, False]) + ) + assert dataframe_type.does_not_end_with(opts).equals( + df.convert_to_series([False, True, True]) + ) + + @pytest.mark.parametrize( "data,comparator,dataset_type,expected_result", [