Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 4 additions & 3 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -58,13 +58,14 @@ This prints every operation with a short description, for example:

```
convert_units
Convert numeric values between units using `pint`.
Convert numeric values from `source_unit` to `target_unit` — for example
`inch` to `cm`, or `kilogram` to `pound`.

do_nothing
Operator that does nothing.
Pass the value through unchanged. Takes no settings.
```

The descriptions are taken from the primitive implementations themselves, so the listing always matches the operations the installed version actually supports.
The descriptions are taken from the primitive implementations themselves, so the listing always matches the operations the installed version actually supports. Each description names the operation's settings (in backticks), matching the fields used in rules files.

For a machine-readable listing, add `--format json`:

Expand Down
5 changes: 4 additions & 1 deletion src/harmonization_framework/primitives/bin_primitive.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,10 @@ def __init__(self, label: Any, lower: int, upper: int, left=None, right=None):

class Bin(PrimitiveOperation):
"""
Assign values into histogram bins.
Bucket numeric values into non-overlapping labelled ranges and return the
matching bin's label. `bins` is a list of `{label, start, end}` entries
with inclusive bounds.

Performs a range query using an interval tree. Bins must not overlap.
"""
def __init__(self, bins: List[Tuple[Any, Tuple[int, int]]]):
Expand Down
4 changes: 2 additions & 2 deletions src/harmonization_framework/primitives/cast.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,9 @@ class CastType(Enum):

class Cast(PrimitiveOperation):
"""
Cast values between supported primitive types.
Convert values from `source` type to `target` type: `text`, `integer`,
`boolean`, `decimal`, or `float`.

Supported targets: "text", "integer", "boolean", "decimal", "float".
Boolean casting accepts common string/number representations.
"""
def __init__(self, source: str, target: str):
Expand Down
3 changes: 2 additions & 1 deletion src/harmonization_framework/primitives/dates.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,8 @@

class ConvertDate(PrimitiveOperation):
"""
Convert between date/time string formats using strptime/strftime.
Convert date/time strings from `source_format` to `target_format`, both
given as strftime patterns.

Examples:
- source_format="%Y-%m-%d", target_format="%m/%d/%Y"
Expand Down
6 changes: 5 additions & 1 deletion src/harmonization_framework/primitives/donothing.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,11 @@

class DoNothing(PrimitiveOperation):
"""
Operator that does nothing.
Pass the value through unchanged. Takes no settings.

Useful as an explicit placeholder where an operation chain is required
but no transformation is wanted — for example a `case`/`coalesce` branch
that uses a column as-is.
"""
def __str__(self):
return "Do Nothing"
Expand Down
7 changes: 4 additions & 3 deletions src/harmonization_framework/primitives/enum2enum.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,11 @@

class EnumToEnum(PrimitiveOperation):
"""
Operator that maps an input based on its prescribed mapping.
Map discrete values to replacement values via `mapping`, serialized as a
list of `{from, to}` entries.

If strict is True, missing mappings raise a KeyError.
If strict is False, missing mappings return the configured default (or None).
If `strict` is true, unmapped values raise a KeyError.
If `strict` is false, unmapped values return `default` (or None).
"""
def __init__(self, mapping: Dict[Any, Any], default: Any = None, strict: bool = False):
"""
Expand Down
3 changes: 2 additions & 1 deletion src/harmonization_framework/primitives/extract_regex.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,8 @@ def _resolve_flags(flag_names: Optional[Iterable[str]]) -> int:

class ExtractRegex(PrimitiveOperation):
"""
Extract a value from a string using a regex capture group.
Extract capture group `group` from the match of regex `expression`
against string values.

Common harmonization use cases include pulling identifiers out of free
text (e.g., MRN: A12-99) or numeric suffixes out of structured codes.
Expand Down
3 changes: 2 additions & 1 deletion src/harmonization_framework/primitives/format_number.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,10 @@

class FormatNumber(PrimitiveOperation):
"""
Format numeric values to a fixed number of decimal places.
Format numeric values as text with exactly `precision` decimal places.

Output is a string, intended for stable presentation (e.g., CSV output).
Use `round` instead to keep the value numeric.
"""
def __init__(self, precision: int):
if not isinstance(precision, int):
Expand Down
2 changes: 1 addition & 1 deletion src/harmonization_framework/primitives/map_each.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@

class MapEach(PrimitiveOperation):
"""
Apply a nested chain of operations to each element of a list.
Apply a nested chain of `operations` to each element of a list.

Useful for multi-source rules where each source value needs the same
per-element transform (e.g. cast each one-hot flag to int) before a
Expand Down
4 changes: 3 additions & 1 deletion src/harmonization_framework/primitives/missing_code.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,9 @@

class MissingCode(PrimitiveOperation):
"""
Operator that turns a column's missing-value codes into real nulls.
Map a column's declared missing-value `codes` (e.g. `-999`, `"UNK"`) to
real nulls; every other value passes through unchanged. Serialize codes as
a list of `{code, label}` entries, and place this first in a rule's chain.

Real datasets frequently encode "missing" as an in-band sentinel value — a
numeric code like -999 or a token like "UNK" — rather than as an empty cell.
Expand Down
3 changes: 2 additions & 1 deletion src/harmonization_framework/primitives/normalize.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,8 @@ class Normalization(Enum):

class NormalizeText(PrimitiveOperation):
"""
Perform a text normalization operation.
Apply a single text `normalization`: `strip`, `lower`, `upper`,
`remove_accents`, `remove_punctuation`, or `remove_special_characters`.
"""
def __init__(self, normalization: Normalization):
self.normalization = normalization
Expand Down
4 changes: 3 additions & 1 deletion src/harmonization_framework/primitives/normalize_boolean.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,9 @@

class NormalizeBoolean(PrimitiveOperation):
"""
Normalize common truthy/falsy representations to booleans.
Normalize truthy/falsy representations to booleans, using the `truthy`
and `falsy` value lists (sensible defaults provided). With `strict` false,
unrecognized values become `default` instead of raising.

This primitive is intended for datasets that encode booleans as strings
or numeric flags (e.g., "Yes", "y", "1", "no", "0").
Expand Down
2 changes: 1 addition & 1 deletion src/harmonization_framework/primitives/offset.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@

class Offset(PrimitiveOperation):
"""
Operator that applies an offset to a numerical value.
Add a constant `offset` (number) to numeric values.
"""
def __init__(self, offset: Union[int, float]):
if not isinstance(offset, (int, float)):
Expand Down
8 changes: 5 additions & 3 deletions src/harmonization_framework/primitives/parse_array.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,11 +7,13 @@

class ParseArray(PrimitiveOperation):
"""
Parse array-like values into Python lists.
Parse array-like text (e.g. "[8,8,6]" or "8|8|6") into a list, optionally
casting elements to `item_type`. Chain before a list-consuming operation
like `reduce`.

Supported formats:
Supported `format` values:
- json: parse JSON arrays from strings (default)
- delimiter: split strings by a configured delimiter
- delimiter: split strings by the configured `delimiter`
"""

SUPPORTED_FORMATS = {"json", "delimiter"}
Expand Down
8 changes: 5 additions & 3 deletions src/harmonization_framework/primitives/reduce.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,10 +18,12 @@ class Reduction(Enum):

class Reduce(PrimitiveOperation):
"""
Reduction operation that transforms N inputs to 1 output.
Reduce a list of values to a single value using `reduction`: `any`,
`none`, `all`, `one-hot`, or `sum`.

This primitive expects a single list/tuple of values as input and returns
one reduced value (e.g., sum, any, all). It does not accept scalar input.
This primitive expects a single list/tuple of values as input (e.g. from
`parse_array` or a multi-source rule) and returns one reduced value. It
does not accept scalar input.
"""
def __init__(self, reduction: Reduction):
self.reduction = reduction
Expand Down
2 changes: 1 addition & 1 deletion src/harmonization_framework/primitives/round_decimal.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@

class Round(PrimitiveOperation):
"""
Round numeric values to a specified decimal precision.
Round numeric values to `precision` decimal places.

Precision follows Python's built-in `round` behavior.
Precision must be a non-negative integer.
Expand Down
2 changes: 1 addition & 1 deletion src/harmonization_framework/primitives/scale.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@

class Scale(PrimitiveOperation):
"""
Operator that applies a scaling factor to a numerical value.
Multiply numeric values by `scaling_factor` (number).
"""
def __init__(self, scaling_factor: Union[int, float]):
if not isinstance(scaling_factor, (int, float)):
Expand Down
3 changes: 2 additions & 1 deletion src/harmonization_framework/primitives/substitute.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,8 @@

class Substitute(PrimitiveOperation):
"""
Apply a text substitution based on a regex pattern.
Replace every match of the regex `expression` with `substitution` in
string values.
"""
def __init__(self, expression: str, substitution: str):
"""
Expand Down
3 changes: 2 additions & 1 deletion src/harmonization_framework/primitives/threshold.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,8 @@

class Threshold(PrimitiveOperation):
"""
Operator that thresholds a numerical value.
Clamp numeric values to the inclusive range [`lower`, `upper`]: values
below `lower` become `lower`, values above `upper` become `upper`.
"""
def __init__(self, lower: Union[int, float], upper: Union[int, float]):
if not isinstance(lower, (int, float)) or not isinstance(upper, (int, float)):
Expand Down
2 changes: 1 addition & 1 deletion src/harmonization_framework/primitives/truncate.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

class Truncate(PrimitiveOperation):
"""
Operator that truncates a string by cutting off the tail.
Shorten strings to at most `length` characters by cutting off the tail.
"""
def __init__(self, length: int):
if not isinstance(length, int):
Expand Down
6 changes: 4 additions & 2 deletions src/harmonization_framework/primitives/units.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,9 +32,11 @@ def __init__(self, value):

class ConvertUnits(PrimitiveOperation):
"""
Convert numeric values between units using `pint`.
Convert numeric values from `source_unit` to `target_unit` — for example
`inch` to `cm`, or `kilogram` to `pound`.

Supports built-in `Unit` enum values or custom unit strings recognized by pint.
Units are built-in `Unit` enum values or any unit string recognized by
the `pint` library.
"""
def __init__(self, source: Union[Unit, str], target: Union[Unit, str]):
if isinstance(source, str):
Expand Down
3 changes: 2 additions & 1 deletion src/harmonization_framework/primitives/validate_pattern.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,8 @@

class ValidatePattern(PrimitiveOperation):
"""
Assert that a string matches a regex pattern.
Assert that string values match the regex `expression`, using `mode`
`match` (default), `fullmatch`, or `search`.

Returns the original value on success. On failure: raises `ValueError`
if `strict=True`, else returns `default`. Use as a data-quality gate in
Expand Down
2 changes: 1 addition & 1 deletion tests/test_list_operations.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ def test_cli_list_operations(capsys):
for name in OPERATION_CLASSES:
assert f"\n{name}\n" in f"\n{out}"
# Spot-check that help text accompanies the names.
assert "Convert numeric values between units" in out
assert "Convert numeric values from `source_unit` to `target_unit`" in out


def test_cli_list_operations_json(capsys):
Expand Down
Loading