From 14dda84c11c7b36c66c0a9607068a921fe7be6dd Mon Sep 17 00:00:00 2001 From: Rdag15 Date: Sat, 27 Jun 2026 16:49:12 -0400 Subject: [PATCH 1/2] feat: add does_not_start_with and does_not_end_with operators Adds the negated complements of the existing starts_with / ends_with string operators, following the established negation pattern in this module (e.g. does_not_contain -> ~self.contains(...)). - dataframe_operators.py: does_not_start_with / does_not_end_with - resources/schema/rule/Operator.json: operator enum entries - resources/schema/rule/Operator.md: operator documentation - resources/schema/rule-merged/Operator.json: regenerated via scripts/merge_schema_markdown.py + prettier (no manual edits) - tests/unit/test_check_operators/test_string_comparison.py: parametrized tests across PandasDataset and DaskDataset Co-Authored-By: Claude Opus 4.8 (1M context) --- .../check_operators/dataframe_operators.py | 10 ++++ resources/schema/rule-merged/Operator.json | 20 +++++++ resources/schema/rule/Operator.json | 10 ++++ resources/schema/rule/Operator.md | 24 +++++++++ .../test_string_comparison.py | 52 +++++++++++++++++++ 5 files changed, 116 insertions(+) diff --git a/cdisc_rules_engine/check_operators/dataframe_operators.py b/cdisc_rules_engine/check_operators/dataframe_operators.py index ddff41a59..a70e5b6f2 100644 --- a/cdisc_rules_engine/check_operators/dataframe_operators.py +++ b/cdisc_rules_engine/check_operators/dataframe_operators.py @@ -857,6 +857,16 @@ def ends_with(self, other_value): results = self.value[target].str.endswith(comparison_data) return results + @log_operator_execution + @type_operator(FIELD_DATAFRAME) + def does_not_start_with(self, other_value): + return ~self.starts_with(other_value) + + @log_operator_execution + @type_operator(FIELD_DATAFRAME) + def does_not_end_with(self, other_value): + return ~self.ends_with(other_value) + @log_operator_execution @type_operator(FIELD_DATAFRAME) def has_equal_length(self, other_value: dict): diff --git a/resources/schema/rule-merged/Operator.json b/resources/schema/rule-merged/Operator.json index 1c1f13a64..073da9d34 100644 --- a/resources/schema/rule-merged/Operator.json +++ b/resources/schema/rule-merged/Operator.json @@ -131,6 +131,16 @@ "required": ["operator", "value"], "type": "object" }, + { + "properties": { + "operator": { + "const": "does_not_end_with", + "markdownDescription": "\nComplement of `ends_with`. Returns True when the target does NOT end with the value.\n\n> DOMAIN not ending with 'FOOBAR'\n\n```yaml\n- name: \"DOMAIN\"\n operator: \"does_not_end_with\"\n value: \"FOOBAR\"\n```\n" + } + }, + "required": ["operator", "value"], + "type": "object" + }, { "properties": { "operator": { @@ -151,6 +161,16 @@ "required": ["operator", "ordering", "value", "within"], "type": "object" }, + { + "properties": { + "operator": { + "const": "does_not_start_with", + "markdownDescription": "\nComplement of `starts_with`. Returns True when the target does NOT start with the value.\n\n> DOMAIN not beginning with 'AP'\n\n```yaml\n- name: \"DOMAIN\"\n operator: \"does_not_start_with\"\n value: \"AP\"\n```\n" + } + }, + "required": ["operator", "value"], + "type": "object" + }, { "properties": { "operator": { diff --git a/resources/schema/rule/Operator.json b/resources/schema/rule/Operator.json index 61ed71f2f..19f08743b 100644 --- a/resources/schema/rule/Operator.json +++ b/resources/schema/rule/Operator.json @@ -72,6 +72,11 @@ "required": ["operator", "value"], "type": "object" }, + { + "properties": { "operator": { "const": "does_not_end_with" } }, + "required": ["operator", "value"], + "type": "object" + }, { "properties": { "operator": { "const": "does_not_equal_string_part" } }, "required": ["operator", "value", "regex"], @@ -84,6 +89,11 @@ "required": ["operator", "ordering", "value", "within"], "type": "object" }, + { + "properties": { "operator": { "const": "does_not_start_with" } }, + "required": ["operator", "value"], + "type": "object" + }, { "properties": { "operator": { "const": "empty" } }, "required": ["operator"], diff --git a/resources/schema/rule/Operator.md b/resources/schema/rule/Operator.md index 4685cac0f..d62ba944c 100644 --- a/resources/schema/rule/Operator.md +++ b/resources/schema/rule/Operator.md @@ -267,6 +267,30 @@ Substring matching value: "FOOBAR" ``` +### does_not_start_with + +Complement of `starts_with`. Returns True when the target does NOT start with the value. + +> DOMAIN not beginning with 'AP' + +```yaml +- name: "DOMAIN" + operator: "does_not_start_with" + value: "AP" +``` + +### does_not_end_with + +Complement of `ends_with`. Returns True when the target does NOT end with the value. + +> DOMAIN not ending with 'FOOBAR' + +```yaml +- name: "DOMAIN" + operator: "does_not_end_with" + value: "FOOBAR" +``` + ### prefix_equal_to True if the `prefix` number of characters beginning a string in `name` match the string in `value` diff --git a/tests/unit/test_check_operators/test_string_comparison.py b/tests/unit/test_check_operators/test_string_comparison.py index 1ec3277d3..9579f72d6 100644 --- a/tests/unit/test_check_operators/test_string_comparison.py +++ b/tests/unit/test_check_operators/test_string_comparison.py @@ -134,6 +134,58 @@ def test_ends_with(data, comparator, dataset_type, expected_result): assert result.equals(df.convert_to_series(expected_result)) +@pytest.mark.parametrize( + "data,comparator,dataset_type,expected_result", + [ + ( + {"target": ["Att", "Btt", "Ctt"], "VAR2": ["A", "B", "D"]}, + "VAR2", + PandasDataset, + [False, False, True], + ), + ( + {"target": ["Att", "Btt", "Ctt"], "VAR2": ["A", "B", "D"]}, + "VAR2", + DaskDataset, + [False, False, True], + ), + ], +) +def test_does_not_start_with(data, comparator, dataset_type, expected_result): + df = dataset_type.from_dict(data) + dataframe_type = DataframeType({"value": df}) + result = dataframe_type.does_not_start_with( + {"target": "target", "comparator": comparator} + ) + assert result.equals(df.convert_to_series(expected_result)) + + +@pytest.mark.parametrize( + "data,comparator,dataset_type,expected_result", + [ + ( + {"target": ["Att", "Btt", "Ctt"], "VAR2": ["tt", "B", "D"]}, + "VAR2", + PandasDataset, + [False, False, False], + ), + ( + {"target": ["Att", "Btt", "Ctt"], "VAR2": ["tt", "B", "D"]}, + "VAR2", + DaskDataset, + [False, False, False], + ), + ], +) +def test_does_not_end_with(data, comparator, dataset_type, expected_result): + df = dataset_type.from_dict(data) + dataframe_type = DataframeType({"value": df}) + result = dataframe_type.does_not_end_with( + {"target": "target", "comparator": comparator} + ) + assert result.equals(df.convert_to_series(expected_result)) + + @pytest.mark.parametrize( "data,comparator,dataset_type,expected_result", [ From f865a425d5d6b7e761128826a08cfeff1a50b4da Mon Sep 17 00:00:00 2001 From: Rdag15 Date: Mon, 29 Jun 2026 19:22:35 -0400 Subject: [PATCH 2/2] fix: make starts_with/ends_with null-safe; add value_is_literal tests Addresses review on #1791. - starts_with/ends_with returned raw pandas string-method output, which is object/NaN when target values are missing. Negating that (does_not_start_with = ~self.starts_with) raised "bad operand type for unary ~: 'NoneType'". Pass na=False so the result is always a clean boolean Series (missing -> False), matching how other negating operators rely on a normalized boolean input. - Add tests for value_is_literal=True (starts/ends and their negations) plus null-safety regression tests, across PandasDataset and DaskDataset. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../check_operators/dataframe_operators.py | 8 ++- .../test_string_comparison.py | 63 +++++++++++++++++++ 2 files changed, 69 insertions(+), 2 deletions(-) diff --git a/cdisc_rules_engine/check_operators/dataframe_operators.py b/cdisc_rules_engine/check_operators/dataframe_operators.py index a70e5b6f2..19fb1e3aa 100644 --- a/cdisc_rules_engine/check_operators/dataframe_operators.py +++ b/cdisc_rules_engine/check_operators/dataframe_operators.py @@ -841,7 +841,9 @@ def starts_with(self, other_value): if self.value.is_series(comparison_data): # need to convert series to tuple to make startswith operator work correctly comparison_data: Tuple[str] = tuple(comparison_data) - results = self.value[target].str.startswith(comparison_data) + # na=False keeps the result a clean boolean Series when target values are + # missing, so negating operators (does_not_start_with) stay null-safe. + results = self.value[target].str.startswith(comparison_data, na=False) return results @log_operator_execution @@ -854,7 +856,9 @@ def ends_with(self, other_value): if self.value.is_series(comparison_data): # need to convert series to tuple to make endswith operator work correctly comparison_data: Tuple[str] = tuple(comparison_data) - results = self.value[target].str.endswith(comparison_data) + # na=False keeps the result a clean boolean Series when target values are + # missing, so negating operators (does_not_end_with) stay null-safe. + results = self.value[target].str.endswith(comparison_data, na=False) return results @log_operator_execution diff --git a/tests/unit/test_check_operators/test_string_comparison.py b/tests/unit/test_check_operators/test_string_comparison.py index 9579f72d6..324ce0ad4 100644 --- a/tests/unit/test_check_operators/test_string_comparison.py +++ b/tests/unit/test_check_operators/test_string_comparison.py @@ -186,6 +186,69 @@ def test_does_not_end_with(data, comparator, dataset_type, expected_result): assert result.equals(df.convert_to_series(expected_result)) +# value_is_literal=True: comparator is a literal string, not a column name. +@pytest.mark.parametrize( + "data,comparator,dataset_type,expected", + [ + ({"target": ["Att", "Btt", "Axx"]}, "A", PandasDataset, [True, False, True]), + ({"target": ["Att", "Btt", "Axx"]}, "A", DaskDataset, [True, False, True]), + ], +) +def test_starts_with_literal(data, comparator, dataset_type, expected): + df = dataset_type.from_dict(data) + dataframe_type = DataframeType({"value": df}) + opts = {"target": "target", "comparator": comparator, "value_is_literal": True} + assert dataframe_type.starts_with(opts).equals(df.convert_to_series(expected)) + assert dataframe_type.does_not_start_with(opts).equals( + df.convert_to_series([not b for b in expected]) + ) + + +@pytest.mark.parametrize( + "data,comparator,dataset_type,expected", + [ + ({"target": ["Att", "Btt", "Cxx"]}, "tt", PandasDataset, [True, True, False]), + ({"target": ["Att", "Btt", "Cxx"]}, "tt", DaskDataset, [True, True, False]), + ], +) +def test_ends_with_literal(data, comparator, dataset_type, expected): + df = dataset_type.from_dict(data) + dataframe_type = DataframeType({"value": df}) + opts = {"target": "target", "comparator": comparator, "value_is_literal": True} + assert dataframe_type.ends_with(opts).equals(df.convert_to_series(expected)) + assert dataframe_type.does_not_end_with(opts).equals( + df.convert_to_series([not b for b in expected]) + ) + + +# Missing target values must stay null-safe: starts_with/ends_with return a clean +# boolean Series (missing -> False) so the negating operators don't crash on `~`. +@pytest.mark.parametrize("dataset_type", [PandasDataset, DaskDataset]) +def test_does_not_start_with_is_null_safe(dataset_type): + df = dataset_type.from_dict({"target": ["Att", None, "Ctt"]}) + dataframe_type = DataframeType({"value": df}) + opts = {"target": "target", "comparator": "A", "value_is_literal": True} + assert dataframe_type.starts_with(opts).equals( + df.convert_to_series([True, False, False]) + ) + assert dataframe_type.does_not_start_with(opts).equals( + df.convert_to_series([False, True, True]) + ) + + +@pytest.mark.parametrize("dataset_type", [PandasDataset, DaskDataset]) +def test_does_not_end_with_is_null_safe(dataset_type): + df = dataset_type.from_dict({"target": ["Att", None, "Cxx"]}) + dataframe_type = DataframeType({"value": df}) + opts = {"target": "target", "comparator": "tt", "value_is_literal": True} + assert dataframe_type.ends_with(opts).equals( + df.convert_to_series([True, False, False]) + ) + assert dataframe_type.does_not_end_with(opts).equals( + df.convert_to_series([False, True, True]) + ) + + @pytest.mark.parametrize( "data,comparator,dataset_type,expected_result", [