Share nested metadata values between parameter clones

changelog.d/share-parameter-metadata-on-clone.changed.md

@@ -0,0 +1 @@
+Parameter clones copy each metadata dict without deep-copying the values in it, which makes cloning a large tax-benefit system faster and smaller. `interpolate_parameters` now records its work in `Parameter.interpolated` instead of writing `completed` into the `interpolation` metadata, so it interpolates every parameter a node propagates `interpolation` to, not only the first.

docs/usage/parameters.md

@@ -57,6 +57,16 @@ metadata:
           href: https://www.gov.uk/government/publications/child-benefit-rates-and-thresholds/child-benefit-rates-and-thresholds
 ```
 
+### Metadata in copies
+
+`clone()` on a parameter tree, which `TaxBenefitSystem.clone()`, reforms and simulation clones use, gives every node, parameter, dated value and scale in the copy its own metadata dict. The values in that dict are the same objects as in the source. Setting, updating or removing a key changes only the tree it is made on. To change a nested value, such as one entry of a `reference` list, replace the key with a new value instead of editing the list or dict in place, which would change every copy that shares it:
+
+```python
+parameter.metadata["reference"] = [*parameter.metadata["reference"], new_reference]
+```
+
+The same holds within one tree: a dated value written in dictionary form (`value:`, with or without `metadata:`) holds its parameter's metadata values, and `propagate_parameter_metadata` gives a node's descendants the node's values.
+
 ### Metadata for parameters
 
 ```{eval-rst}

policyengine_core/parameters/operations/interpolate_parameters.py

@@ -9,6 +9,10 @@
 def interpolate_parameters(root: ParameterNode) -> ParameterNode:
     """Interpolates parameters according to their metadata.
 
+    Each parameter interpolates once, however many times this runs. To
+    interpolate one again, set its ``interpolated`` attribute to ``False`` and
+    give it ``interpolation`` metadata without a true ``completed`` key.
+
     Args:
         root (ParameterNode): The root of the parameter tree.
 
@@ -17,9 +21,13 @@ def interpolate_parameters(root: ParameterNode) -> ParameterNode:
     """
     for parameter in root.get_descendants():
         if isinstance(parameter, Parameter):
-            if "interpolation" in parameter.metadata and not parameter.metadata[
-                "interpolation"
-            ].get("completed", False):
+            if (
+                "interpolation" in parameter.metadata
+                and not parameter.interpolated
+                # Core no longer writes this key, but a tree pickled by an
+                # older Core has it, and so can metadata written by hand.
+                and not parameter.metadata["interpolation"].get("completed", False)
+            ):
                 interpolated_entries = []
                 for i in range(len(parameter.values_list) - 1):
                     # For each gap in parameter values
@@ -56,5 +64,8 @@ def interpolate_parameters(root: ParameterNode) -> ParameterNode:
                 for entry in interpolated_entries:
                     parameter.values_list.append(entry)
                 parameter.values_list.sort(key=lambda x: x.instant_str, reverse=True)
-                parameter.metadata["interpolation"]["completed"] = True
+                # Mark the parameter, not its ``interpolation`` dict. Its clones
+                # and every parameter under a propagating node share that dict,
+                # and propagating metadata again replaces this parameter's.
+                parameter.interpolated = True
     return root

policyengine_core/parameters/parameter.py

@@ -19,6 +19,8 @@ class Parameter(AtInstantLike):
 
     Attributes:
         values_list: List of the values, in reverse chronological order.
+        interpolated: Whether :func:`interpolate_parameters` has filled in this
+            parameter's values.
 
 
     Args:
@@ -45,6 +47,10 @@ class Parameter(AtInstantLike):
 
     """
 
+    # A class default rather than an ``__init__`` assignment, so a subclass that
+    # skips ``__init__`` and a tree pickled before this existed can read it.
+    interpolated: bool = False
+
     def __init__(self, name: str, data: dict, file_path: Optional[str] = None) -> None:
         self.name: str = name
         self.file_path: Optional[str] = file_path
@@ -124,7 +130,8 @@ def clone(self):
         clone = empty_clone(self)
         clone.__dict__ = self.__dict__.copy()
 
-        clone.metadata = copy.deepcopy(self.metadata)
+        # Own dict, shared values: see ParameterAtInstant.clone.
+        clone.metadata = copy.copy(self.metadata)
         clone.values_list = [
             parameter_at_instant.clone() for parameter_at_instant in self.values_list
         ]

policyengine_core/parameters/parameter_at_instant.py

@@ -76,5 +76,7 @@ def __repr__(self) -> str:
     def clone(self) -> "ParameterAtInstant":
         clone = commons.empty_clone(self)
         clone.__dict__ = self.__dict__.copy()
-        clone.metadata = copy.deepcopy(self.metadata)
+        # The clone gets its own metadata dict, but shares the values in it
+        # with this one. To change a nested value, replace its key.
+        clone.metadata = copy.copy(self.metadata)
         return clone

policyengine_core/parameters/parameter_node.py

@@ -204,7 +204,8 @@ def clone(self) -> "ParameterNode":
         clone = commons.empty_clone(self)
         clone.__dict__ = self.__dict__.copy()
 
-        clone.metadata = copy.deepcopy(self.metadata)
+        # Own dict, shared values: see ParameterAtInstant.clone.
+        clone.metadata = copy.copy(self.metadata)
         clone.children = {key: child.clone() for key, child in self.children.items()}
         for child_key, child in clone.children.items():
             setattr(clone, child_key, child)

policyengine_core/parameters/parameter_scale.py

@@ -122,7 +122,8 @@ def clone(self) -> "ParameterScale":
         clone.brackets = [bracket.clone() for bracket in self.brackets]
         for bracket in clone.brackets:
             bracket.parent = clone
-        clone.metadata = copy.deepcopy(self.metadata)
+        # Own dict, shared values: see ParameterAtInstant.clone.
+        clone.metadata = copy.copy(self.metadata)
 
         return clone