diff --git a/graphtage/batch_distance.py b/graphtage/batch_distance.py index 6cbfd1d..8c3d28d 100644 --- a/graphtage/batch_distance.py +++ b/graphtage/batch_distance.py @@ -22,7 +22,12 @@ Set the ``GRAPHTAGE_BATCH_BACKEND`` environment variable to a backend name to pin the choice, which is useful for benchmarking one backend against another and for testing that they agree. -Nothing in Graphtage calls this module yet. +Graphtage reaches the batch through :func:`cost`, which prices a single pair the way +:func:`graphtage.levenshtein.levenshtein_distance` does but answers from a pre-computed block when one holds the +pair. :class:`graphtage.multiset.MultiSetEdit` and :class:`graphtage.levenshtein.EditDistance` each face a cross +product of node pairs whose edits are constructed one at a time, so they call :func:`preprice` or +:func:`preprice_product` before constructing any of them and let every later :func:`cost` read the answer out of the +block. """ @@ -63,6 +68,49 @@ _KERNEL_ITEM_BYTES = 4 """The width of one dynamic programming cell, which the scan holds as :class:`numpy.int32`.""" +BLOCK_STACK_LIMIT = 8 +"""The number of pre-priced blocks :func:`cost` searches before the oldest is dropped. + +A block is a pure cache of a deterministic function, so dropping one costs a recomputation and nothing else. The +limit is what keeps the structure bounded: an edit installs a block in its constructor and reads from it across +however many later calls it takes to tighten its bounds, and nothing tells the cache when that edit is finished. + +""" + +PREPRICE_MIN_PAIRS = 32 +"""The number of pairs below which pre-pricing is skipped and every pair goes through the scalar dynamic program. + +Building a block costs a pass over the pairs, a batch, and the two dictionaries that index it, none of which the +scalar path pays. Measured on an Apple M-series laptop over lists of 6 to 24 character strings, diffing gets +faster from about 36 pairs and slower below about 25, which is the same crossing as :const:`NUMPY_MIN_PAIRS`: +below it a batch is answered by the ``python`` backend, so a block buys the overhead of vectorizing without the +vectorization. + +""" + +PREPRICE_MAX_CELLS = 4 * 1024 * 1024 +"""The largest block, in cells, that pre-pricing will build. + +The pairs a caller offers need not fill the rectangle their distinct sides span. A collection holding both leaves +and key/value pairs, for instance, spans a rectangle several times larger than the number of pairs in it. Cells +that no pair fills cost four bytes each and hold no answer, so a batch whose rectangle is this large is left to the +scalar path rather than given a block of 16 MB. + +""" + +VECTORIZED_MIN_CELLS = 4096 +"""The size of Levenshtein matrix above which one pair alone is worth handing to a vectorized backend. + +The batched scan costs one array pass per row rather than one interpreter step per cell, so it beats the scalar +dynamic program on a single pair of long strings even though there is nothing to amortize the array setup over. +Measured on an Apple M-series laptop, the two meet at about 52 by 52 characters and the scan is 2.4 times faster +by 128 by 128. This threshold, 64 by 64, sits just above the crossing. + +""" + +_UNPRICED = -1 +"""The value of a block cell that no pair filled. Distances are never negative, so it cannot be mistaken for one.""" + _STR_PAD = 0x110000 """The padding symbol for :class:`str` batches: the first value that is not a Unicode code point.""" @@ -538,3 +586,284 @@ def all_flat( raise ValueError(f"all_flat needs two collections of the same length, but got {len(a)} and {len(b)}") unique_pairs, index = _dedupe(list(zip(a, b, strict=True))) return _pair_distances(unique_pairs, backend)[index] + + +class _Block: + """A rectangle of pre-computed distances, indexed by the strings themselves. + + A :class:`dict` keyed on pairs of strings would hold the same answers, but it would hold each key too: for the + cross product of two collections of four hundred strings that is tens of megabytes of tuples and hashes against + the ``rows * columns * 4`` bytes of the matrix here. + + """ + + __slots__ = ('_columns', '_matrix', '_rows') + + def __init__(self, rows: dict[str | bytes, int], columns: dict[str | bytes, int], matrix: np.ndarray): + """Initializes the block. + + Args: + rows: Each string that indexes a row, mapped to its row. + columns: Each string that indexes a column, mapped to its column. + matrix: A ``(len(rows), len(columns))`` array of :class:`numpy.int32`, whose unfilled cells hold + :const:`_UNPRICED`. + + """ + self._rows = rows + self._columns = columns + self._matrix = matrix + + def lookup(self, a: str | bytes, b: str | bytes) -> int | None: + """Returns the distance between two strings, or :const:`None` if this block does not hold it. + + Args: + a: The string to measure from. + b: The string to measure to. + + Returns: + Optional[int]: The distance, or :const:`None`. + + """ + row = self._rows.get(a) + if row is None: + return None + column = self._columns.get(b) + if column is None: + return None + value = int(self._matrix[row, column]) + if value == _UNPRICED: + return None + return value + + +_blocks: tuple[_Block, ...] = () +"""The pre-priced blocks, newest first. Rebound rather than mutated, so that a reader never sees a partial update.""" + + +def _install(block: _Block) -> None: + """Adds a block to the front of the stack, dropping the oldest once the stack is full. + + The stack is replaced rather than mutated. A reader holding the old tuple still sees a consistent stack, and a + writer that loses a race only costs the block it was installing, which is a recomputation rather than a wrong + answer. + + Args: + block: The block to install. + + """ + global _blocks + _blocks = (block, *_blocks[:BLOCK_STACK_LIMIT - 1]) + + +def clear() -> None: + """Discards every pre-priced block. + + :func:`cost` answers the same afterwards, because the blocks only hold what it would otherwise compute. + + """ + global _blocks + _blocks = () + + +def cost(a: str | bytes, b: str | bytes) -> int: + """Returns the Levenshtein distance between two strings, reading a pre-priced block when one holds it. + + This is what :func:`graphtage.levenshtein.exact_string_distance` calls, and it returns exactly what + :func:`graphtage.levenshtein.levenshtein_distance` returns for the same pair. + + A pair that no block holds is computed and *not* recorded. Recording it would make the cache grow with the + number of distinct pairs a diff asks about, which is the whole cross product, and nothing would ever evict it. + Blocks are installed deliberately, by a caller that knows it is about to ask for a whole rectangle of pairs. + + Args: + a: The string to measure from. + b: The string to measure to. + + Returns: + int: The Levenshtein edit distance between the two strings. + + """ + if a == b: + return 0 + elif not a: + return len(b) + elif not b: + return len(a) + for block in _blocks: + value = block.lookup(a, b) + if value is not None: + return value + return _uncached_cost(a, b) + + +def _strip_shared_affixes(a: str | bytes, b: str | bytes) -> tuple[str | bytes, str | bytes]: + """Removes the shared prefix and suffix of two strings, which quadratically shrinks the matrix between them. + + Dropping them cannot change the distance: a character that aligns with itself is free, and no optimal alignment + crosses such a position. + + Args: + a: The string to measure from. + b: The string to measure to. + + Returns: + Tuple[str | bytes, str | bytes]: What is left of each string. + + """ + overlap = min(len(a), len(b)) + prefix = 0 + while prefix < overlap and a[prefix] == b[prefix]: + prefix += 1 + suffix = 0 + while suffix < overlap - prefix and a[len(a) - suffix - 1] == b[len(b) - suffix - 1]: + suffix += 1 + return a[prefix:len(a) - suffix], b[prefix:len(b) - suffix] + + +def _same_kind(a: str | bytes, b: str | bytes) -> bool: + """Returns whether two strings are both :class:`str` or both :class:`bytes`. + + A pair that mixes the two still has a well defined distance, because a :class:`str` character never equals a + :class:`bytes` element, but the backends reject it rather than pick one of the two readings of its symbols. + + Args: + a: The string to measure from. + b: The string to measure to. + + Returns: + bool: Whether a backend will accept the pair. + + """ + if isinstance(a, str): + return isinstance(b, str) + return isinstance(a, bytes) and isinstance(b, bytes) + + +def _preferred_backend() -> str: + """Returns the backend to run a batch whose pairs are big but whose pair count is small. + + :func:`_select_backend` skips a backend whose :attr:`BatchBackend.min_pairs` a batch does not meet, which is the + right rule for a batch of short strings and the wrong one for a batch of one long pair. Naming the backend + outright bypasses that rule while still honoring ``GRAPHTAGE_BATCH_BACKEND``. + + Returns: + str: The name of the fastest available backend, or of the one the environment pins. + + """ + return os.environ.get(BACKEND_ENV_VAR) or available_backends()[0] + + +def _uncached_cost(a: str | bytes, b: str | bytes) -> int: + """Prices one pair that no block holds. + + Args: + a: The string to measure from, which differs from ``b`` and is not empty. + b: The string to measure to, which is not empty. + + Returns: + int: The Levenshtein edit distance between the two strings. + + """ + left, right = _strip_shared_affixes(a, b) + if not left: + return len(right) + elif not right: + return len(left) + elif len(left) * len(right) < VECTORIZED_MIN_CELLS or not _same_kind(left, right): + return levenshtein_distance(left, right) + return int(_pair_distances([(left, right)], _preferred_backend())[0]) + + +def _index(values: Sequence[str | bytes]) -> tuple[list[str | bytes], dict[str | bytes, int]]: + """Numbers the distinct values of a collection in first-seen order. + + Args: + values: The strings to number. + + Returns: + Tuple[List[str | bytes], Dict[str | bytes, int]]: The distinct strings, and each one mapped to its position. + + """ + positions: dict[str | bytes, int] = {} + for value in values: + if value not in positions: + positions[value] = len(positions) + return list(positions), positions + + +def _product_matrix(rows: list[str | bytes], columns: list[str | bytes]) -> np.ndarray: + """Prices a whole rectangle, one batch per kind of string. + + :class:`str` and :class:`bytes` rows are batched separately, and the cells where one meets the other are left + unfilled so that :meth:`_Block.lookup` reports them as absent. + + Args: + rows: The distinct strings indexing the rows. + columns: The distinct strings indexing the columns. + + Returns: + numpy.ndarray: A ``(len(rows), len(columns))`` array of :class:`numpy.int32`. + + """ + matrix = np.full((len(rows), len(columns)), _UNPRICED, dtype=np.int32) + for kind in (str, bytes): + row_positions = [position for position, value in enumerate(rows) if isinstance(value, kind)] + column_positions = [position for position, value in enumerate(columns) if isinstance(value, kind)] + if row_positions and column_positions: + matrix[np.ix_(row_positions, column_positions)] = all_pairs( + [rows[position] for position in row_positions], + [columns[position] for position in column_positions], + ) + return matrix + + +def preprice_product( + from_strings: Sequence[str | bytes], + to_strings: Sequence[str | bytes], +) -> None: + """Prices every pair drawn from two collections and keeps the answers for :func:`cost`. + + Call this before constructing the edits that will ask for those pairs. The block outlives this call, because + the edits that read it are constructed and tightened long afterwards. + + Nothing is installed for a rectangle larger than :const:`PREPRICE_MAX_CELLS`, and the pairs then go through the + scalar path one at a time. + + Args: + from_strings: The strings to measure from, which may repeat. + to_strings: The strings to measure to, which may repeat. + + """ + rows, row_index = _index(from_strings) + columns, column_index = _index(to_strings) + if not rows or not columns or len(rows) * len(columns) > PREPRICE_MAX_CELLS: + return + _install(_Block(row_index, column_index, _product_matrix(rows, columns))) + + +def preprice(pairs: Sequence[Pair]) -> None: + """Prices a list of pairs and keeps the answers for :func:`cost`. + + This is the form for a caller whose pairs are not a whole rectangle, such as + :class:`graphtage.multiset.MultiSetEdit`, which draws one pair from two leaves but two from two key/value + pairs. Pairs that mix :class:`str` with :class:`bytes` are left out and go through the scalar path. + + Nothing is installed for a rectangle larger than :const:`PREPRICE_MAX_CELLS`. + + Args: + pairs: The pairs to price, which may repeat. + + """ + priceable = [(left, right) for left, right in pairs if _same_kind(left, right)] + if not priceable: + return + rows, row_index = _index([left for left, _ in priceable]) + columns, column_index = _index([right for _, right in priceable]) + if len(rows) * len(columns) > PREPRICE_MAX_CELLS: + return + matrix = np.full((len(rows), len(columns)), _UNPRICED, dtype=np.int32) + matrix[ + [row_index[left] for left, _ in priceable], + [column_index[right] for _, right in priceable], + ] = all_flat([left for left, _ in priceable], [right for _, right in priceable]) + _install(_Block(row_index, column_index, matrix)) diff --git a/graphtage/graphtage.py b/graphtage/graphtage.py index 545db0a..c887d9d 100644 --- a/graphtage/graphtage.py +++ b/graphtage/graphtage.py @@ -1034,7 +1034,9 @@ def string_edit_distance(s1: str, s2: str) -> EditDistance: """ list1 = ListNode([StringNode(c) for c in s1]) list2 = ListNode([StringNode(c) for c in s2]) - return EditDistance(list1, list2, list1.children(), list2.children(), insert_remove_penalty=0) + return EditDistance( + list1, list2, list1.children(), list2.children(), insert_remove_penalty=0, preprice=False + ) FILETYPES_BY_MIME: dict[str, 'Filetype'] = {} diff --git a/graphtage/levenshtein.py b/graphtage/levenshtein.py index 98be1ac..b5a4b13 100644 --- a/graphtage/levenshtein.py +++ b/graphtage/levenshtein.py @@ -76,6 +76,11 @@ def exact_string_distance(s: str | bytes, t: str | bytes) -> int: :class:`int` byte values, which never compare equal to a :class:`str` character, so a mixed pair costs one per aligned position exactly as the lattice charges for it. + The answer comes from :func:`graphtage.batch_distance.cost`, which reads a block that was priced in advance + when one holds this pair and computes the pair when none does. That is how a caller facing a whole cross + product of pairs pays for them in one vectorized batch rather than one at a time, and the number is the same + either way. + Args: s: the string from which to match. t: the string to which to match. @@ -84,23 +89,58 @@ def exact_string_distance(s: str | bytes, t: str | bytes) -> int: int: The Levenshtein edit distance metric between the two strings. """ - if s == t: - return 0 - elif not s: - return len(t) - elif not t: - return len(s) - # Stripping a shared prefix and suffix quadratically reduces the size of the matrix that - # levenshtein_distance builds. It cannot change the answer: characters that align with themselves are - # always free, and no optimal alignment crosses them. - overlap = min(len(s), len(t)) - prefix = 0 - while prefix < overlap and s[prefix] == t[prefix]: - prefix += 1 - suffix = 0 - while suffix < overlap - prefix and s[len(s) - suffix - 1] == t[len(t) - suffix - 1]: - suffix += 1 - return levenshtein_distance(s[prefix:len(s) - suffix], t[prefix:len(t) - suffix]) + # Deferred to break the cycle: batch_distance calls levenshtein_distance, above. + from . import batch_distance + return batch_distance.cost(s, t) + + +def _leaf_strings(nodes: Sequence[TreeNode]) -> list[str | bytes]: + """Projects the leaves of a sequence onto the strings that pricing an edit against them compares. + + This mirrors what :meth:`graphtage.LeafNode.edits` and :meth:`graphtage.StringNode.edits` pass to + :func:`exact_string_distance`: a :class:`graphtage.StringNode` is compared by its wrapped object, and every + other leaf by the string representation of it. Nodes that are not leaves are left out, because the edit between + two of them prices its own children rather than a pair of strings. + + Args: + nodes: The nodes to project. + + Returns: + List[str | bytes]: One string per leaf, in order. + + """ + from .graphtage import LeafNode, StringNode + strings: list[str | bytes] = [] + for node in nodes: + if isinstance(node, StringNode): + strings.append(node.object) + elif isinstance(node, LeafNode): + strings.append(str(node.object)) + return strings + + +def _preprice_leaves(from_seq: Sequence[TreeNode], to_seq: Sequence[TreeNode], enabled: bool) -> None: + """Prices the cross product of two sequences' leaves in one batch. + + :meth:`EditDistance._add_node` fills each cell of the Levenshtein matrix with + ``from_seq[column - 1].edits(to_seq[row - 1])``, and a leaf's edit knows its cost as soon as it is constructed. + Those are the same pairs a batch prices at once, so collecting them before the first cell is built replaces one + pure Python dynamic program per cell with one vectorized pass over all of them. + + Args: + from_seq: The nodes to match from, after any shared prefix and suffix have been stripped. + to_seq: The nodes to match to, after any shared prefix and suffix have been stripped. + enabled: Whether to price anything at all, which is the ``preprice`` argument of + :meth:`EditDistance.__init__`. + + """ + from . import batch_distance + if not enabled or len(from_seq) * len(to_seq) < batch_distance.PREPRICE_MIN_PAIRS: + return + from_strings = _leaf_strings(from_seq) + to_strings = _leaf_strings(to_seq) + if len(from_strings) * len(to_strings) >= batch_distance.PREPRICE_MIN_PAIRS: + batch_distance.preprice_product(from_strings, to_strings) class EditDistance(SequenceEdit): @@ -143,6 +183,8 @@ def __init__( from_seq: Sequence[TreeNode], to_seq: Sequence[TreeNode], insert_remove_penalty: int = 1, + *, + preprice: bool = True, ): """Initializes the edit distance edit. @@ -152,6 +194,11 @@ def __init__( from_seq: A sequence of nodes that comprise :obj:`from_node`. to_seq: A sequence of nodes that comprise :obj:`to_node`. insert_remove_penalty: The penalty for inserting or removing a node (default is 1). + preprice: Whether to price the string pairs of the two sequences' leaves in one batch before any cell + of the matrix is built. Pass :const:`False` when the elements are single characters, as + :func:`graphtage.string_edit_distance` does: a character-level lattice is itself an + :class:`EditDistance`, and collecting its pairs would cost a pass over every cell of every string + that gets rendered in exchange for a batch of one-character pairs that cost nothing to begin with. """ self.penalty: int = insert_remove_penalty @@ -184,6 +231,7 @@ def __init__( len(self.shared_prefix):len(to_seq)-len(self.reversed_shared_suffix) ] log.debug(f"Levenshtein len(shared prefix)={len(self.shared_prefix)}, len(shared suffix)={len(self.reversed_shared_suffix)}, len(from_seq)={len(self.from_seq)}, len(to_seq)={len(self.to_seq)}") + _preprice_leaves(self.from_seq, self.to_seq, preprice) constant_cost = 0 if len(from_seq) != len(to_seq): sizes: FibonacciHeap[TreeNode, int] = FibonacciHeap(key=lambda node: node.total_size) diff --git a/graphtage/matching.py b/graphtage/matching.py index cd3f650..0fcd927 100644 --- a/graphtage/matching.py +++ b/graphtage/matching.py @@ -636,12 +636,24 @@ def bounds(self) -> Range: return self._bounds def _make_edges_distinct(self): + """Tightens the edges until their bounds are definitive or non-overlapping. + + Edges that already have definitive bounds are skipped altogether. :func:`graphtage.bounds.make_distinct` + would return without tightening anything in that case, but only after building an interval tree over every + edge, which for a large matching costs more than the matching itself. + + Returns: + bool: :const:`True` the first time it is called, matching the protocol that + :func:`graphtage.bounds.repeat_until_tightened` expects. + + """ if self._edges_are_distinct: return False - else: - make_distinct(*itertools.chain(*self.edges)) - self._edges_are_distinct = True - return True + edges = list(itertools.chain(*self.edges)) + if not all(edge.bounds().definitive() for edge in edges): + make_distinct(*edges) + self._edges_are_distinct = True + return True @property def matching(self) -> Mapping[T, tuple[T, Bounded]]: diff --git a/graphtage/multiset.py b/graphtage/multiset.py index d2ddc45..0222d03 100644 --- a/graphtage/multiset.py +++ b/graphtage/multiset.py @@ -6,10 +6,12 @@ """ import logging -from collections.abc import Iterator +from collections.abc import Iterator, Sequence import graphtage +from . import batch_distance +from .batch_distance import Pair from .bounds import Range from .edits import Insert, Match, Remove from .matching import WeightedBipartiteMatcher @@ -19,8 +21,131 @@ log = logging.getLogger(__name__) -MATCHING_SIZE_WARNING_THRESHOLD = 400 -"""The number of candidate pairs above which matching two unordered collections is reported as slow.""" +MATCHING_SIZE_WARNING_THRESHOLD = 250_000 +"""The number of candidate pairs above which matching two unordered collections is reported as slow. + +The pairs are priced in one batch by :mod:`graphtage.batch_distance`, so this counts the pairs a diff can price in +roughly a second on an Apple M-series laptop rather than the few hundred that were worth warning about when each +pair cost its own pure Python dynamic program. + +""" + + +def _first_with_key(to_set: HashableCounter[TreeNode], key: TreeNode) -> TreeNode | None: + """Returns the first key/value pair in a multiset whose key equals the one given. + + Args: + to_set: The multiset to search. + key: The key to look for. + + Returns: + Optional[TreeNode]: The matching key/value pair, or :const:`None` if the multiset holds none. + + """ + for t in to_set: + if isinstance(t, graphtage.KeyValuePairNode) and t.key == key: + return t + return None + + +def _match_by_key( + from_set: HashableCounter[TreeNode], + to_set: HashableCounter[TreeNode], +) -> list[tuple[TreeNode, TreeNode, int]]: + """Pairs off key/value pairs whose keys are equal, and takes what it pairs out of both multisets. + + This decides the pairing without constructing any edit for it, so that the caller can price every string the + resulting edits will ask about in one batch before it builds the first of them. + + Args: + from_set: The multiset to match from. Its counts are decremented in place. + to_set: The multiset to match to. Its counts are decremented in place. + + Returns: + List[Tuple[TreeNode, TreeNode, int]]: Each matched pair and the number of times it matched. + + """ + matched: list[tuple[TreeNode, TreeNode, int]] = [] + for f in from_set: + if not isinstance(f, graphtage.KeyValuePairNode): + continue + t = _first_with_key(to_set, f.key) + if t is None: + continue + num_matched = min(from_set[f], to_set[t]) + matched.append((f, t, num_matched)) + to_set[t] -= num_matched + for f, _, num_matched in matched: + from_set[f] -= num_matched + return matched + + +def _leaf_pair(from_node: TreeNode, to_node: TreeNode) -> Pair | None: + """Returns the pair of strings that costing an edit between two leaves compares. + + Args: + from_node: The leaf to match from. + to_node: The leaf to match to. + + Returns: + Optional[Pair]: The pair, or :const:`None` when one side is a :class:`str` and the other :class:`bytes`, + which a batch will not price. + + """ + if isinstance(from_node, graphtage.StringNode) and isinstance(to_node, graphtage.StringNode): + if isinstance(from_node.object, str) == isinstance(to_node.object, str): + return from_node.object, to_node.object + return None + return str(from_node.object), str(to_node.object) + + +def _collect_pairs(from_node: TreeNode, to_node: TreeNode, pairs: list[Pair]) -> None: + """Appends the string pairs that costing an edit between two nodes will ask about. + + Two key/value pairs contribute their keys and their values, because + :class:`graphtage.KeyValuePairEdit` always matches key to key and value to value. Two containers contribute + nothing: the edit between them prices its own cross product when it is constructed. + + Args: + from_node: The node to match from. + to_node: The node to match to. + pairs: The list to append to. + + """ + if isinstance(from_node, graphtage.LeafNode) and isinstance(to_node, graphtage.LeafNode): + pair = _leaf_pair(from_node, to_node) + if pair is not None: + pairs.append(pair) + elif isinstance(from_node, graphtage.KeyValuePairNode) and isinstance(to_node, graphtage.KeyValuePairNode): + _collect_pairs(from_node.key, to_node.key, pairs) + _collect_pairs(from_node.value, to_node.value, pairs) + + +def _preprice_matching( + matched: Sequence[tuple[TreeNode, TreeNode, int]], + to_remove: HashableCounter[TreeNode], + to_insert: HashableCounter[TreeNode], + num_pairs: int, +) -> None: + """Prices in one batch every string pair that costing a multiset matching will ask about. + + Args: + matched: The key/value pairs that :func:`_match_by_key` paired off. + to_remove: The nodes the matcher will match from. + to_insert: The nodes the matcher will match to. + num_pairs: The number of candidate pairs the matcher faces, used to decide whether a batch is worth it. + + """ + if num_pairs + len(matched) < batch_distance.PREPRICE_MIN_PAIRS: + return + pairs: list[Pair] = [] + for f, t, _ in matched: + _collect_pairs(f, t, pairs) + for f in to_remove: + for t in to_insert: + _collect_pairs(f, t, pairs) + if pairs: + batch_distance.preprice(pairs) class MultiSetEdit(SequenceEdit): @@ -55,28 +180,11 @@ def __init__( this to `False` will require a significant amount more computation for larger dictionaries. """ - self._matched_kvp_edits: list[Edit] = [] + matched_key_pairs: list[tuple[TreeNode, TreeNode, int]] = [] if auto_match_keys: to_set = HashableCounter(to_set) from_set = HashableCounter(from_set) - to_remove_from = [] - for f in from_set: - if not isinstance(f, graphtage.KeyValuePairNode): - continue - for t in to_set: - if not isinstance(t, graphtage.KeyValuePairNode): - continue - if f.key == t.key: - num_matched = min(from_set[f], to_set[t]) - for _ in range(num_matched): - self._matched_kvp_edits.append(f.edits(t)) - to_remove_from.append((f, num_matched)) - break - else: - continue - to_set[t] -= num_matched - for f, num_matched in to_remove_from: - from_set[f] -= num_matched + matched_key_pairs = _match_by_key(from_set, to_set) self.to_insert = to_set - from_set """The set of nodes in :obj:`to_set` that do not exist in :obj:`from_set`.""" self.to_remove = from_set - to_set @@ -88,6 +196,10 @@ def __init__( "Matching %d unordered elements against %d requires costing %d pairs, which can take a long time", sum(self.to_remove.values()), sum(self.to_insert.values()), num_pairs ) + _preprice_matching(matched_key_pairs, self.to_remove, self.to_insert, num_pairs) + self._matched_kvp_edits: list[Edit] = [ + f.edits(t) for f, t, num_matched in matched_key_pairs for _ in range(num_matched) + ] self._edits: list[Edit] = [Match(n, n, 0) for n in to_match.elements()] self._matcher = WeightedBipartiteMatcher( from_nodes=self.to_remove.elements(), diff --git a/test/test_batch_distance.py b/test/test_batch_distance.py index aa757dc..aab786f 100644 --- a/test/test_batch_distance.py +++ b/test/test_batch_distance.py @@ -1,11 +1,14 @@ import os import random import time +from io import StringIO from unittest import TestCase from unittest.mock import patch import numpy as np +import graphtage +from graphtage import batch_distance from graphtage.batch_distance import ( BACKEND_ENV_VAR, NUMPY_MIN_PAIRS, @@ -13,9 +16,11 @@ all_flat, all_pairs, available_backends, + cost, register_backend, ) from graphtage.levenshtein import levenshtein_distance +from graphtage.printer import Printer ALPHABET = 'abcdefghijklmnopqrstuvwxyz' UNICODE_ALPHABET = 'a\u00e9\u6f22\u00df\u0301\U0001F600\U00010330' @@ -320,3 +325,341 @@ def test_other_types_are_rejected(self): with self.assertRaises(TypeError) as context: all_pairs([1], [2]) self.assertIn('int', str(context.exception)) + + +class TestCostOracle(TestCase): + """Tests for :func:`graphtage.batch_distance.cost` and the blocks it reads.""" + + def setUp(self): + batch_distance.clear() + + def tearDown(self): + batch_distance.clear() + + def assert_cost_is_exact(self, from_strings, to_strings): + """Asserts that :func:`graphtage.batch_distance.cost` answers every pair exactly. + + Args: + from_strings: The strings to measure from. + to_strings: The strings to measure to. + + """ + for left in from_strings: + for right in to_strings: + with self.subTest(left=left, right=right): + self.assertEqual(levenshtein_distance(left, right), cost(left, right)) + + def test_cost_matches_levenshtein_distance_without_a_block(self): + """Every pair gets the same answer the scalar dynamic program gives, with nothing pre-priced. + + Prevents the short circuits and the shared affix stripping in the miss path from changing the metric. The + corpus holds empty strings, repeated strings, strings that share an affix, non-ASCII text, and characters + outside the Basic Multilingual Plane. + + """ + self.assert_cost_is_exact(mixed_corpus(), mixed_corpus()) + + def test_cost_matches_levenshtein_distance_from_a_block(self): + """A pre-priced block answers with exactly what computing the pair would have answered. + + Prevents a block from being indexed by the wrong axis, which a square grid of a symmetric metric would + hide, and prevents its :class:`numpy.int32` cells from being read back as anything but the distance. + + """ + rng = random.Random(51966) + from_strings = random_strings(rng, 14, ALPHABET, 0, 16) + to_strings = random_strings(rng, 11, ALPHABET, 0, 16) + batch_distance.preprice_product(from_strings, to_strings) + self.assert_cost_is_exact(from_strings, to_strings) + + def test_a_block_answers_instead_of_the_scalar_path(self): + """A pre-priced pair is answered from the block rather than computed again. + + Prevents the block from being installed but never consulted, which no comparison of answers can detect + because both paths return the same number. + + """ + rng = random.Random(31337) + from_strings = random_strings(rng, 8, ALPHABET, 4, 16) + to_strings = random_strings(rng, 8, ALPHABET, 4, 16) + batch_distance.preprice_product(from_strings, to_strings) + with patch.object(batch_distance, '_uncached_cost', side_effect=AssertionError('block was not consulted')): + self.assert_cost_is_exact(from_strings, to_strings) + + def test_eviction_does_not_change_answers(self): + """Filling the block stack past its limit drops the oldest blocks without corrupting any answer. + + Prevents an eviction policy that leaves a stale block in place or that drops the wrong end of the stack. + The pairs of the evicted blocks have to be recomputed, and those answers have to agree with the ones that + are still cached. + + """ + rng = random.Random(0xBADF00D) + batches = [ + (random_strings(rng, 7, ALPHABET, 3, 12), random_strings(rng, 7, ALPHABET, 3, 12)) + for _ in range(batch_distance.BLOCK_STACK_LIMIT + 4) + ] + for from_strings, to_strings in batches: + batch_distance.preprice_product(from_strings, to_strings) + self.assertEqual(batch_distance.BLOCK_STACK_LIMIT, len(batch_distance._blocks)) + for from_strings, to_strings in batches: + self.assert_cost_is_exact(from_strings, to_strings) + + def test_misses_are_not_memoized(self): + """Pricing a pair that no block holds leaves the cache exactly as it was. + + Prevents the cache from growing with the number of distinct pairs a diff asks about, which is the whole + cross product, and which nothing would ever evict. + + """ + rng = random.Random(8675309) + for left, right in zip( + random_strings(rng, 200, ALPHABET, 1, 12), + random_strings(rng, 200, ALPHABET, 1, 12), + strict=True, + ): + cost(left, right) + self.assertEqual((), batch_distance._blocks) + + def test_clear_discards_every_block(self): + """Clearing the cache leaves the answers alone and the stack empty.""" + rng = random.Random(24601) + from_strings = random_strings(rng, 8, ALPHABET, 2, 10) + to_strings = random_strings(rng, 8, ALPHABET, 2, 10) + batch_distance.preprice_product(from_strings, to_strings) + self.assertEqual(1, len(batch_distance._blocks)) + batch_distance.clear() + self.assertEqual((), batch_distance._blocks) + self.assert_cost_is_exact(from_strings, to_strings) + + def test_pre_pricing_a_flat_list_leaves_unasked_pairs_absent(self): + """A pair the caller never offered is not read out of the rectangle its sides happen to span. + + Prevents the unfilled cells of a sparse block from being mistaken for distances of zero, which is what a + zero-filled matrix would report for every pair the caller did not ask about. + + """ + batch_distance.preprice([('kitten', 'sitting'), ('flaw', 'lawn')]) + self.assertEqual(3, cost('kitten', 'sitting')) + self.assertEqual(2, cost('flaw', 'lawn')) + self.assertEqual(levenshtein_distance('kitten', 'lawn'), cost('kitten', 'lawn')) + self.assertEqual(levenshtein_distance('flaw', 'sitting'), cost('flaw', 'sitting')) + + def test_pairs_that_mix_str_and_bytes_are_left_to_the_scalar_path(self): + """A block never answers a pair whose sides are of different types, and the pair is still priced. + + Prevents a mixed pair from being given one of the two readings of its symbols: ``ord('a')`` and + ``b'a'[0]`` are both 97 while ``'a' == b'a'`` is :const:`False`, so the batch declines such a pair and the + scalar path has to pick it up. + + """ + batch_distance.preprice([('abc', b'abc'), ('abc', 'abd')]) + self.assertEqual(1, cost('abc', 'abd')) + self.assertEqual(3, cost('abc', b'abc')) + self.assertEqual(levenshtein_distance('abc', b'abc'), cost('abc', b'abc')) + + def test_bytes_and_str_share_a_block_without_colliding(self): + """A block holding both kinds of string keeps each kind's answers to itself.""" + batch_distance.preprice_product(['abc', b'abc'], ['abd', b'abxd']) + self.assertEqual(1, cost('abc', 'abd')) + self.assertEqual(2, cost(b'abc', b'abxd')) + self.assertEqual(levenshtein_distance('abc', b'abxd'), cost('abc', b'abxd')) + + def test_an_oversized_rectangle_is_not_pre_priced(self): + """A batch whose distinct sides span too large a rectangle is left to the scalar path. + + Prevents a sparse batch from allocating a matrix far larger than the number of answers it holds. + + """ + rng = random.Random(4096) + from_strings = random_strings(rng, 6, ALPHABET, 2, 8) + to_strings = random_strings(rng, 6, ALPHABET, 2, 8) + with patch.object(batch_distance, 'PREPRICE_MAX_CELLS', 8): + batch_distance.preprice_product(from_strings, to_strings) + batch_distance.preprice(list(zip(from_strings, to_strings, strict=True))) + self.assertEqual((), batch_distance._blocks) + self.assert_cost_is_exact(from_strings, to_strings) + + def test_one_big_pair_goes_to_a_vectorized_backend(self): + """A single pair large enough to pay for the array setup is scanned rather than looped over. + + Prevents the single-pair route from being gated on the number of pairs alone, which would send every long + pair that no block holds back through the pure Python dynamic program. + + """ + rng = random.Random(1024) + long_left = ''.join(rng.choices(ALPHABET, k=128)) + long_right = ''.join(rng.choices(ALPHABET, k=128)) + short_left, short_right = 'kitten', 'sitting' + self.assertGreaterEqual(len(long_left) * len(long_right), batch_distance.VECTORIZED_MIN_CELLS) + self.assertLess(len(short_left) * len(short_right), batch_distance.VECTORIZED_MIN_CELLS) + with patch.object(NumpyBackend, 'distances', autospec=True, side_effect=NumpyBackend.distances) as batched: + self.assertEqual(levenshtein_distance(short_left, short_right), cost(short_left, short_right)) + batched.assert_not_called() + self.assertEqual(levenshtein_distance(long_left, long_right), cost(long_left, long_right)) + self.assertTrue(batched.called, 'a pair this large should have been scanned') + + def test_the_environment_variable_still_pins_a_single_big_pair(self): + """``GRAPHTAGE_BATCH_BACKEND`` is honored on the single-pair route as well as on a whole batch.""" + rng = random.Random(2048) + left = ''.join(rng.choices(ALPHABET, k=96)) + right = ''.join(rng.choices(ALPHABET, k=96)) + with patch.object(NumpyBackend, 'distances', autospec=True, side_effect=NumpyBackend.distances) as batched: + with patch.dict(os.environ, {BACKEND_ENV_VAR: 'python'}): + self.assertEqual(levenshtein_distance(left, right), cost(left, right)) + batched.assert_not_called() + + def test_every_backend_pre_prices_the_same_answers(self): + """Pinning each backend in turn fills a block with identical answers. + + Prevents a backend from being exact on its own entry points but wrong through the block, for instance by + returning a dtype that the block truncates. + + """ + rng = random.Random(0xFEEDFACE) + from_strings = random_strings(rng, 9, ALPHABET, 0, 14) + to_strings = random_strings(rng, 9, ALPHABET, 0, 14) + expected = oracle_pairs(from_strings, to_strings) + for backend in available_backends(): + with self.subTest(backend=backend), patch.dict(os.environ, {BACKEND_ENV_VAR: backend}): + batch_distance.clear() + batch_distance.preprice_product(from_strings, to_strings) + answered = np.array( + [[cost(left, right) for right in to_strings] for left in from_strings], dtype=np.int64 + ) + self.assertTrue(np.array_equal(answered, expected)) + + +def mutated_words(rng: random.Random, count: int, alphabet: str = ALPHABET) -> tuple[list[str], list[str]]: + """Draws a collection of strings and a copy of it with one substitution in each.""" + words = random_strings(rng, count, alphabet, 6, 18) + mutated = [] + for word in words: + position = rng.randrange(len(word)) + mutated.append(word[:position] + rng.choice(alphabet) + word[position + 1:]) + return words, mutated + + +def diff_shapes() -> list[tuple[str, object, object, bool]]: + """Builds the documents that the pre-pricing parity tests diff. + + Every shape is large enough to be pre-priced, and together they cover the ordered and unordered paths, keys as + well as values, and the kinds of string a leaf can wrap. + + Returns: + List[Tuple[str, object, object, bool]]: A name, the document to diff from, the document to diff to, and + whether to build lists as unordered collections. + + """ + rng = random.Random(0x5CA1AB1E) + words, mutated = mutated_words(rng, 24) + keys, mutated_keys = mutated_words(rng, 40) + values, mutated_values = mutated_words(rng, 40) + non_ascii, mutated_non_ascii = mutated_words(rng, 16, UNICODE_ALPHABET) + multiline = [f"{left}\n{right}\n" for left, right in zip(words, mutated, strict=True)] + remultiline = [f"{right}\n{left}\n" for left, right in zip(words, mutated, strict=True)] + return [ + ('ordered', words, mutated, False), + ('unordered', words, rng.sample(mutated, len(mutated)), True), + ('ragged', words, mutated[:17], False), + ('dict', dict(zip(keys, values, strict=True)), dict(zip(mutated_keys, mutated_values, strict=True)), False), + ('dict-shared-keys', dict(zip(keys, values, strict=True)), dict(zip(keys, mutated_values, strict=True)), + False), + ('bytes', [word.encode() for word in words], [word.encode() for word in mutated], False), + ('non-ascii', non_ascii, mutated_non_ascii, False), + ('multiline', multiline, remultiline, False), + ('nested', [{'k': word} for word in words], [{'k': word} for word in mutated], False), + ] + + +class TestPrePricedDiffs(TestCase): + """Checks that pre-pricing changes how a diff is computed and never what it produces.""" + + NOT_PRE_PRICED = frozenset({'nested'}) + """Shapes whose string pairs are spread over many small edits, none of them holding enough to batch. + + A list of single-entry dictionaries is one: the outer sequence holds containers, which contribute no pair, and + each inner dictionary offers one pair of its own. + + """ + + def setUp(self): + batch_distance.clear() + + def tearDown(self): + batch_distance.clear() + + @staticmethod + def render(from_obj, to_obj, unordered: bool) -> tuple[str, int]: + """Diffs two Python objects and returns the rendered diff and the total cost of its edits. + + Args: + from_obj: The document to diff from. + to_obj: The document to diff to. + unordered: Whether to build lists as unordered collections. + + Returns: + Tuple[str, int]: The rendered diff, and the sum of the costs of every edit in it. + + """ + options = graphtage.BuildOptions(ignore_list_order=unordered) + stream = StringIO() + printer = Printer(out_stream=stream, ansi_color=False, quiet=True) + diff = graphtage.json.build_tree(from_obj, options).diff(graphtage.json.build_tree(to_obj, options)) + total = sum( + edit.bounds().upper_bound + for node in diff.dfs() for edit in node.edit_list if edit.has_non_zero_cost() + ) + graphtage.FILETYPES_BY_TYPENAME['json'].get_default_formatter().print(printer, diff) + printer.flush(final=True) + return stream.getvalue(), total + + def test_pre_pricing_does_not_change_a_diff(self): + """A pre-priced diff renders the same text at the same cost as one priced a pair at a time. + + Prevents pre-pricing from feeding the matcher a different cost matrix than the scalar path would have, + which would change which nodes a diff matches. It also prevents the collection pass from projecting a node + onto a string that the edit it stands for never compares, which would leave the block holding an answer + for the wrong question. + + """ + for name, from_obj, to_obj, unordered in diff_shapes(): + with self.subTest(shape=name): + priced = self.render(from_obj, to_obj, unordered) + if name not in self.NOT_PRE_PRICED: + self.assertTrue(batch_distance._blocks, f"the {name} shape never got pre-priced") + batch_distance.clear() + with patch.object(batch_distance, 'PREPRICE_MIN_PAIRS', 1 << 40): + scalar = self.render(from_obj, to_obj, unordered) + self.assertEqual((), batch_distance._blocks) + self.assertEqual(scalar, priced) + + def test_a_pre_priced_diff_reads_its_costs_out_of_the_block(self): + """A diff whose pairs were pre-priced does not go back through the scalar dynamic program for them. + + Prevents :func:`graphtage.levenshtein.exact_string_distance` from being left wired straight to + :func:`graphtage.levenshtein.levenshtein_distance`, which would install blocks that nothing ever reads and + show up as no change at all in what the diff produces. This watches the module level name that + :mod:`graphtage.batch_distance` does not use, so only the unbatched route trips it. + + """ + words, mutated = mutated_words(random.Random(0xB10CC), 24) + with patch('graphtage.levenshtein.levenshtein_distance', wraps=levenshtein_distance) as scalar: + self.render(words, mutated, False) + self.assertEqual(0, scalar.call_count, 'the pre-priced pairs were priced one at a time as well') + + def test_every_backend_produces_the_same_diff(self): + """Pinning each backend in turn renders the same diff. + + Prevents a backend from being exact in isolation and wrong once a diff depends on it, and makes + ``GRAPHTAGE_BATCH_BACKEND`` safe to use for benchmarking. + + """ + for name, from_obj, to_obj, unordered in diff_shapes(): + rendered = {} + for backend in available_backends(): + with self.subTest(shape=name, backend=backend), patch.dict(os.environ, {BACKEND_ENV_VAR: backend}): + batch_distance.clear() + rendered[backend] = self.render(from_obj, to_obj, unordered) + with self.subTest(shape=name): + self.assertEqual(1, len(set(rendered.values())), f"the backends disagree on the {name} shape") diff --git a/test/test_levenshtein.py b/test/test_levenshtein.py index 411b274..65b17dd 100644 --- a/test/test_levenshtein.py +++ b/test/test_levenshtein.py @@ -3,7 +3,8 @@ from tqdm import trange -from graphtage import EditDistance, string_edit_distance +import graphtage +from graphtage import EditDistance, batch_distance, string_edit_distance from graphtage.edits import Edit, Insert, Match, Remove from graphtage.levenshtein import exact_string_distance, levenshtein_distance @@ -352,3 +353,38 @@ def test_agrees_with_the_lattice(self): while lattice.tighten_bounds(): pass self.assertEqual(lattice.bounds().upper_bound, exact_string_distance(s, t), f"{s!r} -> {t!r}") + + +class TestPrePricing(TestCase): + """Covers the ``preprice`` argument of :class:`graphtage.levenshtein.EditDistance`.""" + + def setUp(self): + batch_distance.clear() + + def tearDown(self): + batch_distance.clear() + + def test_a_sequence_of_leaves_is_pre_priced(self): + """The cross product of two sequences of leaves is priced in one batch before any cell is built. + + Prevents the pre-pricing call from being dropped from the constructor, which no comparison of results can + detect because both routes return the same distances. + + """ + from_node = graphtage.json.build_tree([f"item number {index}" for index in range(12)]) + to_node = graphtage.json.build_tree([f"item no. {index}" for index in range(12)]) + self.assertIsInstance(from_node.edits(to_node), EditDistance) + self.assertTrue(batch_distance._blocks) + + def test_a_character_lattice_is_not_pre_priced(self): + """The lattice over the characters of two strings prices nothing in advance. + + Prevents the character level :class:`EditDistance` from paying for a collection pass over every cell of + every string a diff renders, in exchange for a batch of single-character pairs whose distances are + settled by the equality short circuit before any dynamic program runs. + + """ + lattice = string_edit_distance('kittens are nice', 'sitting is nicer') + while lattice.tighten_bounds(): + pass + self.assertEqual((), batch_distance._blocks) diff --git a/test/test_matching.py b/test/test_matching.py index b23ba35..d7f8583 100644 --- a/test/test_matching.py +++ b/test/test_matching.py @@ -1,11 +1,12 @@ import itertools import random from unittest import TestCase +from unittest.mock import patch import numpy as np from tqdm import tqdm, trange -from graphtage.bounds import Range +from graphtage.bounds import ConstantBound, Range, make_distinct from graphtage.matching import ( MatchingFromNode, MatchingToNode, @@ -118,3 +119,90 @@ def test_getitem_populates_the_cache(self): def test_edges_agrees_with_getitem(self): self.assertEqual([self.from_node[self.to_node]], list(self.from_node.edges())) + + +class TestMakeEdgesDistinct(TestCase): + """Covers the shortcut in :meth:`graphtage.matching.WeightedBipartiteMatcher._make_edges_distinct`. + + Every edge whose weight is a :class:`graphtage.StringEdit` knows its cost the moment it is constructed, so a + matching over leaves hands :func:`graphtage.bounds.make_distinct` nothing to tighten. It would still build an + interval tree over every edge before finding that out, which on a matching of a few hundred elements costs + more than solving the matching. + + """ + + WEIGHTS = ( + (7, 2, 9, 4), + (3, 8, 1, 6), + (5, 5, 2, 2), + (9, 1, 4, 8), + ) + + def constant_matcher(self, weights=None) -> WeightedBipartiteMatcher[int]: + """Builds a matcher whose every edge already has a definitive bound.""" + weights = self.WEIGHTS if weights is None else weights + return WeightedBipartiteMatcher( + from_nodes=list(range(len(weights))), + to_nodes=list(range(len(weights[0]))), + get_edge=lambda f, t, weights=weights: ConstantBound(weights[f][t]), + ) + + @staticmethod + def solve(matcher: WeightedBipartiteMatcher[int]) -> dict[int, tuple[int, int]]: + """Tightens a matcher to completion and returns the pairs it chose with their weights.""" + while matcher.tighten_bounds(): + pass + return {f: (t, edge.bounds().upper_bound) for f, (t, edge) in matcher.matching.items()} + + def test_definitive_edges_do_not_reach_make_distinct(self): + """A matching whose edges are all definitive skips the interval tree entirely. + + Prevents the shortcut from being written but never taken, which no comparison of matchings can detect + because :func:`graphtage.bounds.make_distinct` tightens nothing in this case either. + + """ + with patch('graphtage.matching.make_distinct') as distinct: + matching = self.solve(self.constant_matcher()) + distinct.assert_not_called() + self.assertEqual(4, len(matching)) + + def test_indefinite_edges_still_reach_make_distinct(self): + """An edge whose bounds still overlap another's is tightened before the matching is solved. + + Prevents the shortcut from swallowing the case it does not apply to, which would hand + :func:`scipy.optimize.linear_sum_assignment` a matrix of upper bounds that overstate some pairs. + + """ + random.seed(0x5EED) + edges = [[RandomDecreasingRange() for _ in range(4)] for _ in range(4)] + matcher = WeightedBipartiteMatcher( + from_nodes=list(range(4)), + to_nodes=list(range(4)), + get_edge=lambda f, t: edges[f][t], + ) + with patch('graphtage.matching.make_distinct', wraps=make_distinct) as distinct: + self.solve(matcher) + self.assertTrue(distinct.called) + + def test_the_shortcut_does_not_change_the_matching(self): + """The pairs chosen are the same whether or not :func:`graphtage.bounds.make_distinct` runs. + + Prevents the shortcut from leaving out a step that the matching depends on. The comparison is against the + implementation the shortcut replaced, run over the same weights. + + """ + def always_make_distinct(matcher): + if matcher._edges_are_distinct: + return False + make_distinct(*itertools.chain(*matcher.edges)) + matcher._edges_are_distinct = True + return True + + random.seed(0xDEC1DE) + for _ in range(20): + rows, columns = random.randint(1, 7), random.randint(1, 7) + weights = tuple(tuple(random.randint(0, 40) for _ in range(columns)) for _ in range(rows)) + shortcut = self.solve(self.constant_matcher(weights)) + with patch.object(WeightedBipartiteMatcher, '_make_edges_distinct', always_make_distinct): + forced = self.solve(self.constant_matcher(weights)) + self.assertEqual(forced, shortcut, f"weights={weights!r}") diff --git a/test/test_multiset.py b/test/test_multiset.py index b5f58f8..719ae33 100644 --- a/test/test_multiset.py +++ b/test/test_multiset.py @@ -63,6 +63,29 @@ def test_set_versus_set(self): def test_dict_with_set_value(self): self.assertMultiSetEdit({"a": {1, 2}}, {"a": {1, 3}}, 1) + @staticmethod + def kvp(key: str, value: int) -> graphtage.KeyValuePairNode: + """Builds a key/value pair node outside any dictionary.""" + return graphtage.KeyValuePairNode(graphtage.StringNode(key), graphtage.IntegerNode(value)) + + def test_repeated_key_value_pairs_match_as_many_times_as_they_repeat(self): + """A key that appears several times on each side is matched as many times as the smaller side holds it. + + Prevents the multiplicity from being lost when the loop that decides which pairs share a key is separated + from the loop that builds their edits. A multiset can hold the same key/value pair more than once, and the + surplus on the longer side has to be left for the matcher rather than silently matched or dropped. + + """ + from_node = graphtage.MultiSetNode([self.kvp('a', 1) for _ in range(3)]) + to_node = graphtage.MultiSetNode([self.kvp('a', 2) for _ in range(2)]) + edit = from_node.edits(to_node) + self.assertIsInstance(edit, MultiSetEdit) + self.assertEqual(2, len(edit._matched_kvp_edits)) + self.assertEqual(1, sum(edit.to_remove.values())) + self.assertEqual(0, sum(edit.to_insert.values())) + for kvp_edit in edit._matched_kvp_edits: + self.assertEqual(1, kvp_edit.bounds().upper_bound) + class TestMatchingOptimality(TestCase): """Checks that the matching a diff of two string sets produces is optimal under exact edit costs.