From 1c9d1dc498debb630e97a47e5a30a9330ec0ea77 Mon Sep 17 00:00:00 2001 From: Roei Date: Sat, 20 Jun 2026 18:14:43 +0300 Subject: [PATCH 01/16] Add headers and unit tests for the PB recommendation-system algorithms Implements the function headers (with doctests and empty bodies) and the pytest test suite for the algorithms in "A Recommendation System for Participatory Budgeting" (Leibiker & Talmon, 2023), integrating with the pabutools data structures. Co-Authored-By: Claude Opus 4.8 --- pb_recommendation.py | 755 ++++++++++++++++++++++++++++++++++++++ test_pb_recommendation.py | 500 +++++++++++++++++++++++++ 2 files changed, 1255 insertions(+) create mode 100644 pb_recommendation.py create mode 100644 test_pb_recommendation.py diff --git a/pb_recommendation.py b/pb_recommendation.py new file mode 100644 index 0000000..965c462 --- /dev/null +++ b/pb_recommendation.py @@ -0,0 +1,755 @@ +""" +An implementation of the algorithms in: +"A Recommendation System for Participatory Budgeting", +by Gil Leibiker and Nimrod Talmon (2023), https://optlearnmas23.github.io/files/p17.pdf + +Programmer: Roei Yanku +Date: 2026-06-20. +""" + +from __future__ import annotations + +from pabutools.election import ( + Instance, + Project, + ApprovalProfile, + ApprovalBallot, +) +from pabutools.rules import BudgetAllocation + + +# --------------------------------------------------------------------------- +# Section 2.3 - Partial ballots (three-state approval ballots). +# --------------------------------------------------------------------------- +class PartialApprovalBallot: + """ + A three-state approval ballot (Section 2.3), splitting the projects into the + approval set A_v (``approved``), the disapproval set D_v (``disapproved``) + and the unknown set H_v (``hidden``). They partition P (A_v ∪ D_v ∪ H_v = P) + and the exposed set is E_v = A_v ∪ D_v. pabutools' ApprovalBallot cannot + express this (no disapproved/unknown distinction), so this helper class is + added here. + + Parameters + ---------- + approved : Iterable[:py:class:`~pabutools.election.instance.Project`], optional + The projects the voter approves (A_v). Defaults to ``()``. + disapproved : Iterable[:py:class:`~pabutools.election.instance.Project`], optional + The projects the voter disapproves (D_v). Defaults to ``()``. + hidden : Iterable[:py:class:`~pabutools.election.instance.Project`], optional + The projects whose preference is unknown (H_v). Defaults to ``()``. + + Attributes + ---------- + approved : set[:py:class:`~pabutools.election.instance.Project`] + The approval set A_v. + disapproved : set[:py:class:`~pabutools.election.instance.Project`] + The disapproval set D_v. + hidden : set[:py:class:`~pabutools.election.instance.Project`] + The hidden set H_v. + + Examples + -------- + >>> p1, p2, p3 = Project("p1", 1), Project("p2", 1), Project("p3", 1) + >>> b = PartialApprovalBallot(approved={p1}, disapproved={p2}, hidden={p3}) + >>> b.exposed == {p1, p2} + True + >>> (b.approved | b.disapproved | b.hidden) == {p1, p2, p3} + True + """ + + def __init__(self, approved=(), disapproved=(), hidden=()) -> None: + self.approved: set[Project] = set() # Empty implementation + self.disapproved: set[Project] = set() # Empty implementation + self.hidden: set[Project] = set() # Empty implementation + + @property + def exposed(self) -> set[Project]: + """The exposed set E_v = A_v ∪ D_v (the projects the voter answered).""" + return set() # Empty implementation + + def as_approval_ballot(self) -> ApprovalBallot: + """ + The pabutools approval ballot made of the approved projects A_v, so that + a (completed) partial ballot can be fed to approval-based voting rules. + """ + return ApprovalBallot() # Empty implementation + + def __eq__(self, other) -> bool: + return ( + isinstance(other, PartialApprovalBallot) + and self.approved == other.approved + and self.disapproved == other.disapproved + and self.hidden == other.hidden + ) + + def __repr__(self) -> str: + return ( + f"PartialApprovalBallot(approved={sorted(map(str, self.approved))}, " + f"disapproved={sorted(map(str, self.disapproved))}, " + f"hidden={sorted(map(str, self.hidden))})" + ) + + +def reveal_ballot( + instance: Instance, + true_approval: set[Project], + exposed: set[Project], +) -> PartialApprovalBallot: + """ + Build the partial ballot exposing a set of projects of a voter whose true + approval set is known (Section 2.3): A_v = exposed ∩ true, + D_v = exposed \\ true, H_v = P \\ exposed. + + Parameters + ---------- + instance : :py:class:`~pabutools.election.instance.Instance` + The PB instance (used as the universe of projects P). + true_approval : set[:py:class:`~pabutools.election.instance.Project`] + The projects the voter truly approves. + exposed : set[:py:class:`~pabutools.election.instance.Project`] + The projects revealed for this voter (E_v). + + Returns + ------- + PartialApprovalBallot + The corresponding three-state ballot. + + Examples + -------- + Example 2.4 from the paper: P = {p1, p2, p3, p4}, the voter truly approves + {p1, p2}, and {p1, p3} is exposed. Then p1 is approved, p3 disapproved, and + p2, p4 remain hidden. + + >>> p1, p2, p3, p4 = (Project("p1", 1), Project("p2", 1), + ... Project("p3", 1), Project("p4", 1)) + >>> inst = Instance([p1, p2, p3, p4], budget_limit=4) + >>> b = reveal_ballot(inst, {p1, p2}, {p1, p3}) + >>> b.approved == {p1}, b.disapproved == {p3}, b.hidden == {p2, p4} + (True, True, True) + """ + return PartialApprovalBallot() # Empty implementation + + +# --------------------------------------------------------------------------- +# Section 2.2.3 - Popularity and consensus (primitives used by every module). +# --------------------------------------------------------------------------- +def approval_scores( + instance: Instance, profile: ApprovalProfile +) -> dict[Project, int]: + """ + Definition 2.1 (Approval scores): the approval score of a project is the + number of voters that approve it. + + Parameters + ---------- + instance : :py:class:`~pabutools.election.instance.Instance` + The PB instance (the set of projects and the budget limit). + profile : :py:class:`~pabutools.election.profile.approvalprofile.ApprovalProfile` + The approval profile, i.e. the ballots of the voters. + + Returns + ------- + dict[:py:class:`~pabutools.election.instance.Project`, int] + A mapping from each project to its approval score. + + Examples + -------- + Example 1 from the paper (P = {p1, p2, p3}, three full ballots): + + >>> p1, p2, p3 = Project("p1", 1), Project("p2", 1), Project("p3", 2) + >>> inst = Instance([p1, p2, p3], budget_limit=3) + >>> prof = ApprovalProfile([ApprovalBallot([p1, p2]), + ... ApprovalBallot([p1, p3]), + ... ApprovalBallot([p2])]) + >>> s = approval_scores(inst, prof) + >>> [s[p] for p in (p1, p2, p3)] + [2, 2, 1] + """ + return {} # Empty implementation + + +def consensus_levels( + instance: Instance, profile: ApprovalProfile +) -> dict[Project, int]: + """ + Definition 2.2 (Consensus levels): the absolute difference between the number + of voters approving a project and the number disapproving it (so a unanimous + approval or a unanimous rejection both reach the maximum n). + + Parameters + ---------- + instance : :py:class:`~pabutools.election.instance.Instance` + The PB instance. + profile : :py:class:`~pabutools.election.profile.approvalprofile.ApprovalProfile` + The approval profile. + + Returns + ------- + dict[:py:class:`~pabutools.election.instance.Project`, int] + A mapping from each project to its consensus level. + + Examples + -------- + Example 7 from the paper (4 LV voters): p1 approved 4/0, p2 split 2/2, + p3 rejected 0/4. Both p1 and p3 are in full consensus, p2 in none. + + >>> p1, p2, p3 = Project("p1", 1), Project("p2", 1), Project("p3", 1) + >>> inst = Instance([p1, p2, p3], budget_limit=3) + >>> prof = ApprovalProfile([ApprovalBallot([p1, p2]), ApprovalBallot([p1]), + ... ApprovalBallot([p1, p2]), ApprovalBallot([p1])]) + >>> c = consensus_levels(inst, prof) + >>> [c[p] for p in (p1, p2, p3)] + [4, 0, 4] + """ + return {} # Empty implementation + + +# --------------------------------------------------------------------------- +# Section 2.2.5 - The voting rule (greedy approval). +# --------------------------------------------------------------------------- +def greedy_approval( + instance: Instance, profile: ApprovalProfile +) -> BudgetAllocation: + """ + Greedy approval voting rule (Section 2.2.5). Projects are considered in + decreasing order of their approval score and funded while the budget + allows; a project that would exceed the remaining budget is skipped. Ties + are broken lexicographically by project name. + + .. note:: + Implemented from scratch; it does *not* reuse pabutools' + :py:func:`~pabutools.rules.greedy_utilitarian_welfare`, which ranks by + satisfaction **divided by cost** (density). The paper ranks by the + **raw** approval score, so the two give different bundles. + + Parameters + ---------- + instance : :py:class:`~pabutools.election.instance.Instance` + The PB instance. + profile : :py:class:`~pabutools.election.profile.approvalprofile.ApprovalProfile` + The (completed) approval profile. + + Returns + ------- + :py:class:`~pabutools.rules.budgetallocation.BudgetAllocation` + The winning bundle. + + Examples + -------- + Example 1 from the paper: scores p1 = p2 = 2, p3 = 1, budget 3. p1 and p2 + are funded (cost 1 each); p3 (cost 2) no longer fits. + + >>> p1, p2, p3 = Project("p1", 1), Project("p2", 1), Project("p3", 2) + >>> inst = Instance([p1, p2, p3], budget_limit=3) + >>> prof = ApprovalProfile([ApprovalBallot([p1, p2]), + ... ApprovalBallot([p1, p3]), + ... ApprovalBallot([p2])]) + >>> sorted(greedy_approval(inst, prof), key=str) + [p1, p2] + """ + return BudgetAllocation() # Empty implementation + + +# --------------------------------------------------------------------------- +# Section 3.1.1 - Random setup. +# --------------------------------------------------------------------------- +def random_setup( + instance: Instance, + tv_ballots: dict[str, set[Project]], + k: int, + seed: int | None = None, +) -> dict[str, set[Project]]: + """ + Algorithm 1 - Random setup (Section 3.1.1): expose k projects chosen + uniformly at random from P for each TV voter (the rest is predicted later). + + Parameters + ---------- + instance : :py:class:`~pabutools.election.instance.Instance` + The PB instance. + tv_ballots : dict[str, set[:py:class:`~pabutools.election.instance.Project`]] + The true (hidden) approval set of every TV voter, keyed by voter id. + Projects outside the set are disapproved by that voter. + k : int + The number of projects to expose per voter. + seed : int, optional + Seed for the random generator, for reproducibility. + + Returns + ------- + dict[str, set[:py:class:`~pabutools.election.instance.Project`]] + For each TV voter, the set of k exposed projects. + + Examples + -------- + Example 5 from the paper: a single TV voter v4, k = 2. Whatever the draw, + exactly two projects are exposed and they are real projects of the instance. + + >>> p1, p2, p3, p4 = (Project("p1", 2), Project("p2", 2), + ... Project("p3", 3), Project("p4", 3)) + >>> inst = Instance([p1, p2, p3, p4], budget_limit=6) + >>> exposed = random_setup(inst, {"v4": {p1, p2}}, k=2, seed=0) + >>> len(exposed["v4"]) == 2 and exposed["v4"] <= {p1, p2, p3, p4} + True + """ + return {} # Empty implementation + + +# --------------------------------------------------------------------------- +# Section 3.1.2 - Offline setup (popularity / consensus / controversiality). +# --------------------------------------------------------------------------- +def offline_popularity( + instance: Instance, lv_profile: ApprovalProfile, k: int +) -> list[Project]: + """ + Algorithm 2 - Offline revealing by popularity (Section 3.1.2). Exposes, for + every TV voter, the k most approved projects among the LV voters, i.e. + {sigma_1, ..., sigma_k}. + + Parameters + ---------- + instance : :py:class:`~pabutools.election.instance.Instance` + The PB instance. + lv_profile : :py:class:`~pabutools.election.profile.approvalprofile.ApprovalProfile` + The full ballots of the LV voters. + k : int + The number of projects to expose. + + Returns + ------- + list[:py:class:`~pabutools.election.instance.Project`] + The k most popular projects, ordered by score (ties by name). + + Examples + -------- + Example 6 from the paper: LV scores p1 = 3, p2 = 1, p3 = 1, so the single + most popular project is p1. + + >>> p1, p2, p3 = Project("p1", 1), Project("p2", 1), Project("p3", 1) + >>> inst = Instance([p1, p2, p3], budget_limit=3) + >>> lv = ApprovalProfile([ApprovalBallot([p1, p3]), + ... ApprovalBallot([p1, p2]), + ... ApprovalBallot([p1])]) + >>> offline_popularity(inst, lv, k=1) + [p1] + """ + return [] # Empty implementation + + +def offline_consensus( + instance: Instance, lv_profile: ApprovalProfile, k: int +) -> list[Project]: + """ + Algorithm 3 - Offline revealing by consensus (Section 3.1.2). Exposes the k + projects with the highest consensus level among the LV voters, i.e. + {gamma_1, ..., gamma_k}. + + Parameters + ---------- + instance : :py:class:`~pabutools.election.instance.Instance` + The PB instance. + lv_profile : :py:class:`~pabutools.election.profile.approvalprofile.ApprovalProfile` + The full ballots of the LV voters. + k : int + The number of projects to expose. + + Returns + ------- + list[:py:class:`~pabutools.election.instance.Project`] + The k projects most in consensus, ordered by level (ties by name). + + Examples + -------- + Example 7 from the paper: p1 and p3 both have consensus 4 (p1 unanimously + approved, p3 unanimously rejected); the tie is broken by name towards p1. + + >>> p1, p2, p3 = Project("p1", 1), Project("p2", 1), Project("p3", 1) + >>> inst = Instance([p1, p2, p3], budget_limit=3) + >>> lv = ApprovalProfile([ApprovalBallot([p1, p2]), ApprovalBallot([p1]), + ... ApprovalBallot([p1, p2]), ApprovalBallot([p1])]) + >>> offline_consensus(inst, lv, k=1) + [p1] + """ + return [] # Empty implementation + + +def offline_controversiality( + instance: Instance, lv_profile: ApprovalProfile, k: int +) -> list[Project]: + """ + Algorithm 4 - Offline revealing by controversiality (Section 3.1.2): expose + the k projects *least* in consensus among the LV voters, + {gamma_{m-k+1}, ..., gamma_m} (the hardest to predict, so asked directly). + + Parameters + ---------- + instance : :py:class:`~pabutools.election.instance.Instance` + The PB instance. + lv_profile : :py:class:`~pabutools.election.profile.approvalprofile.ApprovalProfile` + The full ballots of the LV voters. + k : int + The number of projects to expose. + + Returns + ------- + list[:py:class:`~pabutools.election.instance.Project`] + The k most controversial projects, ordered by increasing consensus. + + Examples + -------- + Example 8 from the paper (same data as Example 7): p2 is split exactly in + half (consensus 0) and is therefore the single most controversial project. + + >>> p1, p2, p3 = Project("p1", 1), Project("p2", 1), Project("p3", 1) + >>> inst = Instance([p1, p2, p3], budget_limit=3) + >>> lv = ApprovalProfile([ApprovalBallot([p1, p2]), ApprovalBallot([p1]), + ... ApprovalBallot([p1, p2]), ApprovalBallot([p1])]) + >>> offline_controversiality(inst, lv, k=1) + [p2] + """ + return [] # Empty implementation + + +# --------------------------------------------------------------------------- +# Section 3.1.3 - Online setup (adaptive controversial). +# --------------------------------------------------------------------------- +def online_adaptive_controversial( + instance: Instance, + lv_profile: ApprovalProfile, + tv_ballot: set[Project], + k: int, +) -> list[Project]: + """ + Algorithm 5 - Online adaptive-controversial setup (Section 3.1.3): in each of + k iterations recompute the most controversial project given the LV ballots + and the answers already revealed, then ask about it (each answer affects the + next question). + + Parameters + ---------- + instance : :py:class:`~pabutools.election.instance.Instance` + The PB instance. + lv_profile : :py:class:`~pabutools.election.profile.approvalprofile.ApprovalProfile` + The full ballots of the LV voters. + tv_ballot : set[:py:class:`~pabutools.election.instance.Project`] + The true (hidden) approval set of the TV voter being queried. + k : int + The number of iterations / projects to expose. + + Returns + ------- + list[:py:class:`~pabutools.election.instance.Project`] + The k exposed projects, in the order they were asked. + + Examples + -------- + Example 9 from the paper: 4 LV voters, one TV voter v5, k = 2. p1, p2, p3 + are all tied as most controversial; the first question (tie broken by name) + is p1, and after the voter's answer the second question is p2. + + >>> p1, p2, p3, p4 = (Project("p1", 1), Project("p2", 1), + ... Project("p3", 1), Project("p4", 1)) + >>> inst = Instance([p1, p2, p3, p4], budget_limit=4) + >>> lv = ApprovalProfile([ApprovalBallot([p1, p2]), ApprovalBallot([p1, p3]), + ... ApprovalBallot([p2, p3]), ApprovalBallot([])]) + >>> online_adaptive_controversial(inst, lv, {p1, p2}, k=2) + [p1, p2] + """ + return [] # Empty implementation + + +# --------------------------------------------------------------------------- +# Section 2.1 / examples - Prediction module (majority rule over LV). +# --------------------------------------------------------------------------- +def predict_by_majority( + instance: Instance, + lv_profile: ApprovalProfile, + ballot: PartialApprovalBallot, +) -> ApprovalBallot: + """ + Prediction module (Section 2.1). Completes a single partial TV ballot into a + full approval ballot: every project in the hidden set H_v is predicted as + approved iff its approval rate among the LV voters is at least 50%. The + exposed approvals A_v are kept; the exposed disapprovals D_v stay rejected. + + Parameters + ---------- + instance : :py:class:`~pabutools.election.instance.Instance` + The PB instance. + lv_profile : :py:class:`~pabutools.election.profile.approvalprofile.ApprovalProfile` + The full ballots of the LV voters, used as the training data. + ballot : PartialApprovalBallot + The partial ballot of the TV voter to complete. + + Returns + ------- + :py:class:`~pabutools.election.ballot.approvalballot.ApprovalBallot` + The full predicted approval ballot of the TV voter. + + Examples + -------- + Example 11 from the paper: 2 LV voters both approve {p1, p2}. A TV voter with + nothing exposed (all three projects hidden) is predicted to approve p1 (100%) + and p2 (100%) but not p3 (0%). + + >>> p1, p2, p3 = Project("p1", 4), Project("p2", 4), Project("p3", 6) + >>> inst = Instance([p1, p2, p3], budget_limit=6) + >>> lv = ApprovalProfile([ApprovalBallot([p1, p2]), ApprovalBallot([p1, p2])]) + >>> partial = PartialApprovalBallot(hidden={p1, p2, p3}) + >>> predict_by_majority(inst, lv, partial) == {p1, p2} + True + """ + return ApprovalBallot() # Empty implementation + + +def predict_by_classification( + instance: Instance, + lv_profile: ApprovalProfile, + ballot: PartialApprovalBallot, +) -> ApprovalBallot: + """ + Prediction module - binary classification (Section 2.1.1): predict each + hidden project with a per-project binary classifier trained on the LV ballots + (features = votes on the exposed projects, label = vote on the target), then + applied to the TV voter. Exposed approvals A_v are kept and exposed + disapprovals D_v stay rejected. Backed by the external ``xgboost`` library + (:py:class:`xgboost.XGBClassifier`, class-weighted loss for the imbalanced + data), imported inside the implementation. + + Parameters + ---------- + instance : :py:class:`~pabutools.election.instance.Instance` + The PB instance. + lv_profile : :py:class:`~pabutools.election.profile.approvalprofile.ApprovalProfile` + The full ballots of the LV voters, used as the training data. + ballot : PartialApprovalBallot + The partial ballot of the TV voter to complete. + + Returns + ------- + :py:class:`~pabutools.election.ballot.approvalballot.ApprovalBallot` + The full predicted approval ballot of the TV voter. + + Examples + -------- + Example 11 from the paper: 2 LV voters both approve {p1, p2}. A TV voter with + nothing exposed is completed to {p1, p2} - on such an unambiguous pattern any + correct predictor (XGBoost included) agrees with the LV majority. + + >>> p1, p2, p3 = Project("p1", 4), Project("p2", 4), Project("p3", 6) + >>> inst = Instance([p1, p2, p3], budget_limit=6) + >>> lv = ApprovalProfile([ApprovalBallot([p1, p2]), ApprovalBallot([p1, p2])]) + >>> partial = PartialApprovalBallot(hidden={p1, p2, p3}) + >>> predict_by_classification(inst, lv, partial) == {p1, p2} + True + """ + return ApprovalBallot() # Empty implementation + + +def predict_by_matrix_factorization( + instance: Instance, + lv_profile: ApprovalProfile, + ballot: PartialApprovalBallot, +) -> ApprovalBallot: + """ + Prediction module - collaborative filtering via Matrix Factorization + (Section 2.1.2): build the sparse user-item matrix from the LV ballots and + exposed TV votes (approve=1, disapprove=0), factorise it, and predict a + hidden project as approved iff its reconstructed score is >= 0.5. Exposed + approvals A_v are kept and exposed disapprovals D_v stay rejected. Backed by + the external ``scikit-surprise`` library (:py:class:`surprise.SVD`). + + Parameters + ---------- + instance : :py:class:`~pabutools.election.instance.Instance` + The PB instance. + lv_profile : :py:class:`~pabutools.election.profile.approvalprofile.ApprovalProfile` + The full ballots of the LV voters, used as the training data. + ballot : PartialApprovalBallot + The partial ballot of the TV voter to complete. + + Returns + ------- + :py:class:`~pabutools.election.ballot.approvalballot.ApprovalBallot` + The full predicted approval ballot of the TV voter. + + Examples + -------- + Example 10 from the paper: three LV voters all support the cheap pair + {p1, p2} and reject {p3, p4}. The reconstructed matrix completes a TV voter + with nothing exposed to {p1, p2}. + + >>> p1, p2, p3, p4 = (Project("p1", 3), Project("p2", 3), + ... Project("p3", 4), Project("p4", 4)) + >>> inst = Instance([p1, p2, p3, p4], budget_limit=6) + >>> lv = ApprovalProfile([ApprovalBallot([p1, p2])] * 3) + >>> partial = PartialApprovalBallot(hidden={p1, p2, p3, p4}) + >>> predict_by_matrix_factorization(inst, lv, partial) == {p1, p2} + True + """ + return ApprovalBallot() # Empty implementation + + +def predict_by_factorization_machines( + instance: Instance, + lv_profile: ApprovalProfile, + ballot: PartialApprovalBallot, +) -> ApprovalBallot: + """ + Prediction module - hybrid Factorization Machines (Section 2.1.2): like MF + but with a linear term plus pairwise latent interactions and optional side + features, predicting a hidden project as approved iff the FM score is >= 0.5. + Exposed approvals A_v are kept and exposed disapprovals D_v stay rejected. + Backed by an external FM library (e.g. ``lightfm`` / ``fastFM``). + + Parameters + ---------- + instance : :py:class:`~pabutools.election.instance.Instance` + The PB instance. + lv_profile : :py:class:`~pabutools.election.profile.approvalprofile.ApprovalProfile` + The full ballots of the LV voters, used as the training data. + ballot : PartialApprovalBallot + The partial ballot of the TV voter to complete. + + Returns + ------- + :py:class:`~pabutools.election.ballot.approvalballot.ApprovalBallot` + The full predicted approval ballot of the TV voter. + + Examples + -------- + Example 11 from the paper: 2 LV voters both approve {p1, p2}. A TV voter with + nothing exposed is completed to {p1, p2}. + + >>> p1, p2, p3 = Project("p1", 4), Project("p2", 4), Project("p3", 6) + >>> inst = Instance([p1, p2, p3], budget_limit=6) + >>> lv = ApprovalProfile([ApprovalBallot([p1, p2]), ApprovalBallot([p1, p2])]) + >>> partial = PartialApprovalBallot(hidden={p1, p2, p3}) + >>> predict_by_factorization_machines(inst, lv, partial) == {p1, p2} + True + """ + return ApprovalBallot() # Empty implementation + + +# --------------------------------------------------------------------------- +# Full pipeline (sampling -> prediction -> greedy approval). +# --------------------------------------------------------------------------- +def recommend( + instance: Instance, + lv_profile: ApprovalProfile, + tv_ballots: dict[str, set[Project]], + k: int, +) -> BudgetAllocation: + """ + The complete Section 3 pipeline with the offline-popularity sampler: expose + the k most popular LV projects to each TV voter, predict the hidden votes by + LV majority, then run greedy approval on the LV plus completed TV ballots. + + Parameters + ---------- + instance : :py:class:`~pabutools.election.instance.Instance` + The PB instance. + lv_profile : :py:class:`~pabutools.election.profile.approvalprofile.ApprovalProfile` + The full ballots of the LV voters. + tv_ballots : dict[str, set[:py:class:`~pabutools.election.instance.Project`]] + The true (hidden) approval set of every TV voter, keyed by voter id. + k : int + The number of projects to expose per TV voter. + + Returns + ------- + :py:class:`~pabutools.rules.budgetallocation.BudgetAllocation` + The predicted winning bundle. + + Examples + -------- + Example 10 from the paper - the "perfect" case. Three LV voters and one TV + voter all support the two cheap projects {p1, p2}; the predicted bundle + coincides with the real one. + + >>> p1, p2, p3, p4 = (Project("p1", 3), Project("p2", 3), + ... Project("p3", 4), Project("p4", 4)) + >>> inst = Instance([p1, p2, p3, p4], budget_limit=6) + >>> lv = ApprovalProfile([ApprovalBallot([p1, p2])] * 3) + >>> sorted(recommend(inst, lv, {"v4": {p1, p2}}, k=1), key=str) + [p1, p2] + """ + return BudgetAllocation() # Empty implementation + + +# --------------------------------------------------------------------------- +# Section 5.2 - Bundle evaluation metrics. +# --------------------------------------------------------------------------- +def symmetric_distance( + real_bundle: set[Project], predicted_bundle: set[Project] +) -> int: + """ + Section 5.2.1 - Symmetric distance between the real winning bundle and the + predicted one: the size of their symmetric difference. + + Parameters + ---------- + real_bundle : set[:py:class:`~pabutools.election.instance.Project`] + The bundle obtained from the real (full) ballots. + predicted_bundle : set[:py:class:`~pabutools.election.instance.Project`] + The bundle obtained from the predicted ballots. + + Returns + ------- + int + The number of projects in exactly one of the two bundles. + + Examples + -------- + Example 10 (a perfect prediction) and Example 11 (a complete failure): + + >>> p1, p2, p3 = Project("p1", 1), Project("p2", 1), Project("p3", 1) + >>> symmetric_distance({p1, p2}, {p1, p2}) + 0 + >>> symmetric_distance({p1}, {p3}) + 2 + """ + return 0 # Empty implementation + + +def fractional_allocation_score( + real_bundle: set[Project], predicted_bundle: set[Project], budget_limit: int +) -> float: + """ + Definition 5.1 (Fractional Allocation score): the total cost of the + projects predicted correctly (those in both bundles) divided by the budget + limit, FA = lambda / B with lambda = sum of cost(p) over p in pb ∩ rb. + + Parameters + ---------- + real_bundle : set[:py:class:`~pabutools.election.instance.Project`] + The bundle obtained from the real (full) ballots. + predicted_bundle : set[:py:class:`~pabutools.election.instance.Project`] + The bundle obtained from the predicted ballots. + budget_limit : int + The budget limit B of the instance. + + Returns + ------- + float + The fractional allocation score, in [0, 1]. + + Examples + -------- + Example 10: pb = rb = {p1, p2}, costs 3 and 3, budget 6, so FA = 6/6 = 1.0. + Example 11: disjoint bundles, so FA = 0/6 = 0.0. + + >>> p1, p2, p3 = Project("p1", 3), Project("p2", 3), Project("p3", 6) + >>> fractional_allocation_score({p1, p2}, {p1, p2}, budget_limit=6) + 1.0 + >>> fractional_allocation_score({p3}, {p1}, budget_limit=6) + 0.0 + """ + return 0.0 # Empty implementation + + +if __name__ == "__main__": + import doctest + + doctest.testmod(verbose=True) diff --git a/test_pb_recommendation.py b/test_pb_recommendation.py new file mode 100644 index 0000000..0791b41 --- /dev/null +++ b/test_pb_recommendation.py @@ -0,0 +1,500 @@ +""" +Unit tests for `pb_recommendation`, the implementation of the algorithms in +"A Recommendation System for Participatory Budgeting" (Leibiker & Talmon, 2023). + +Run with: pytest test_pb_recommendation.py -v + +Each test is the specification its function must satisfy. They come in three +kinds: + * small, hand-checked instances (the examples from the paper); + * edge cases (empty profile, single project, full ballots, k bounds); + * large inputs, checked against an independent reference or via invariants. + +Programmer: Roei Yanku +Date: 2026-06-20. +""" + +import random +from collections import Counter + +import pytest + +from pabutools.election import ( + Instance, + Project, + ApprovalProfile, + ApprovalBallot, +) + +from pb_recommendation import ( + PartialApprovalBallot, + reveal_ballot, + approval_scores, + consensus_levels, + greedy_approval, + random_setup, + offline_popularity, + offline_consensus, + offline_controversiality, + online_adaptive_controversial, + predict_by_majority, + predict_by_classification, + predict_by_matrix_factorization, + predict_by_factorization_machines, + recommend, + symmetric_distance, + fractional_allocation_score, +) + + +# --------------------------------------------------------------------------- +# Helpers / fixtures +# --------------------------------------------------------------------------- +def make_projects(specs): + """specs: list of (name, cost) -> dict name -> Project.""" + return {name: Project(name, cost) for name, cost in specs} + + +@pytest.fixture +def example1(): + """P = {p1, p2, p3}, costs 1,1,2, B = 3; v1={p1,p2}, v2={p1,p3}, v3={p2}.""" + p = make_projects([("p1", 1), ("p2", 1), ("p3", 2)]) + inst = Instance(p.values(), budget_limit=3) + prof = ApprovalProfile( + [ + ApprovalBallot([p["p1"], p["p2"]]), + ApprovalBallot([p["p1"], p["p3"]]), + ApprovalBallot([p["p2"]]), + ] + ) + return p, inst, prof + + +@pytest.fixture +def consensus_data(): + """4 LV voters: p1 approved 4/0, p2 split 2/2, p3 rejected 0/4.""" + p = make_projects([("p1", 1), ("p2", 1), ("p3", 1)]) + inst = Instance(p.values(), budget_limit=3) + lv = ApprovalProfile( + [ + ApprovalBallot([p["p1"], p["p2"]]), + ApprovalBallot([p["p1"]]), + ApprovalBallot([p["p1"], p["p2"]]), + ApprovalBallot([p["p1"]]), + ] + ) + return p, inst, lv + + +def random_instance(n_projects, n_voters, budget, seed): + """Build a random instance + approval profile for property testing.""" + rng = random.Random(seed) + projects = [Project(f"p{i}", rng.randint(1, 10)) for i in range(n_projects)] + inst = Instance(projects, budget_limit=budget) + ballots = [] + for _ in range(n_voters): + approved = [p for p in projects if rng.random() < 0.3] + ballots.append(ApprovalBallot(approved)) + return projects, inst, ApprovalProfile(ballots) + + +def padded_projects(m, cost): + """m unit-named projects p000, p001, ... so str-order == index-order.""" + width = len(str(m - 1)) + return [Project(f"p{i:0{width}d}", cost) for i in range(m)] + + +def manual_scores(projects, profile): + """Independent (Counter-based) reimplementation of the approval scores.""" + counts = Counter() + for ballot in profile: + for proj in ballot: + counts[proj] += 1 + return {proj: counts[proj] for proj in projects} + + +def reference_greedy_approval(projects, profile, budget): + """ + Independent reference implementation of greedy approval, used to cross-check + `greedy_approval` on random inputs (the "compare to another algorithm" + strategy). Projects are taken in decreasing score, ties broken by name, and + funded while they fit the budget. + """ + counts = manual_scores(projects, profile) + ordered = sorted(projects, key=lambda p: (-counts[p], str(p))) + chosen, spent = [], 0 + for p in ordered: + if spent + p.cost <= budget: + chosen.append(p) + spent += p.cost + return chosen + + +# --------------------------------------------------------------------------- +# approval_scores +# --------------------------------------------------------------------------- +class TestApprovalScores: + def test_example1(self, example1): + p, inst, prof = example1 + scores = approval_scores(inst, prof) + assert scores[p["p1"]] == 2 + assert scores[p["p2"]] == 2 + assert scores[p["p3"]] == 1 + + def test_empty_profile(self): + p = make_projects([("p1", 1), ("p2", 1)]) + inst = Instance(p.values(), budget_limit=2) + scores = approval_scores(inst, ApprovalProfile([])) + assert all(scores[proj] == 0 for proj in p.values()) + + def test_large_structured_all_approve_all(self): + # "Complete-approval" analogue: 200 projects, 500 voters, everyone + # approves everything -> every score is exactly 500. + projects = padded_projects(200, cost=1) + inst = Instance(projects, budget_limit=10) + prof = ApprovalProfile([ApprovalBallot(projects)] * 500) + scores = approval_scores(inst, prof) + assert all(scores[p] == 500 for p in projects) + + def test_matches_manual_count_random(self): + # Cross-check against an independent counting implementation. + projects, inst, prof = random_instance(40, 120, 100, seed=31) + assert approval_scores(inst, prof) == manual_scores(projects, prof) + + +# --------------------------------------------------------------------------- +# consensus_levels +# --------------------------------------------------------------------------- +class TestConsensusLevels: + def test_example7(self, consensus_data): + p, inst, lv = consensus_data + c = consensus_levels(inst, lv) + assert c[p["p1"]] == 4 + assert c[p["p2"]] == 0 + assert c[p["p3"]] == 4 + + def test_nonnegative_and_bounded_random(self): + projects, inst, prof = random_instance(15, 40, 80, seed=2) + c = consensus_levels(inst, prof) + assert all(0 <= c[p] <= len(prof) for p in projects) + + def test_large_structured_even_split_is_zero(self): + # Exactly half approve and half reject each project -> consensus 0. + projects = padded_projects(20, cost=1) + inst = Instance(projects, budget_limit=10) + prof = ApprovalProfile( + [ApprovalBallot(projects)] * 50 + [ApprovalBallot([])] * 50 + ) + c = consensus_levels(inst, prof) + assert all(c[p] == 0 for p in projects) + + +# --------------------------------------------------------------------------- +# greedy_approval (voting rule) +# --------------------------------------------------------------------------- +class TestGreedyApproval: + def test_example1(self, example1): + p, inst, prof = example1 + bundle = set(greedy_approval(inst, prof)) + assert bundle == {p["p1"], p["p2"]} + + def test_skips_unaffordable_but_funds_affordable(self): + # A project that exceeds the budget is skipped, while a cheaper one that + # fits is funded. + p = make_projects([("cheap", 10), ("dear", 100)]) + inst = Instance(p.values(), budget_limit=10) + prof = ApprovalProfile([ApprovalBallot([p["cheap"], p["dear"]])]) + assert set(greedy_approval(inst, prof)) == {p["cheap"]} + + def test_large_structured_known_bundle(self): + # 100 unit-cost projects, all approved by everyone, budget 30 -> exactly + # the 30 name-first projects are funded (tie-break decides everything). + projects = padded_projects(100, cost=1) + inst = Instance(projects, budget_limit=30) + prof = ApprovalProfile([ApprovalBallot(projects)] * 20) + bundle = set(greedy_approval(inst, prof)) + assert bundle == set(projects[:30]) + + def test_matches_reference_random(self): + # Cross-check against the independent greedy reference, several seeds. + for seed in range(5): + projects, inst, prof = random_instance(30, 50, 40, seed=200 + seed) + got = set(greedy_approval(inst, prof)) + expected = set(reference_greedy_approval(projects, prof, inst.budget_limit)) + assert got == expected + + +# --------------------------------------------------------------------------- +# random_setup +# --------------------------------------------------------------------------- +class TestRandomSetup: + def test_exposes_exactly_k(self): + p = make_projects([("p1", 2), ("p2", 2), ("p3", 3), ("p4", 3)]) + inst = Instance(p.values(), budget_limit=6) + exposed = random_setup(inst, {"v4": set(p.values())}, k=2, seed=0) + assert len(exposed["v4"]) == 2 + assert exposed["v4"] <= set(p.values()) + + def test_all_tv_voters_covered(self): + projects, inst, _ = random_instance(10, 1, 30, seed=5) + tv = {f"v{i}": set() for i in range(5)} + exposed = random_setup(inst, tv, k=3, seed=7) + assert set(exposed.keys()) == set(tv.keys()) + assert all(len(exposed[v]) == 3 for v in tv) + + def test_k_zero(self): + p = make_projects([("p1", 1), ("p2", 1)]) + inst = Instance(p.values(), budget_limit=2) + exposed = random_setup(inst, {"v1": set()}, k=0, seed=0) + assert exposed["v1"] == set() + + +# --------------------------------------------------------------------------- +# offline_popularity / consensus / controversiality +# --------------------------------------------------------------------------- +class TestOfflineSamplers: + def test_popularity_example6(self): + p = make_projects([("p1", 1), ("p2", 1), ("p3", 1)]) + inst = Instance(p.values(), budget_limit=3) + lv = ApprovalProfile( + [ + ApprovalBallot([p["p1"], p["p3"]]), + ApprovalBallot([p["p1"], p["p2"]]), + ApprovalBallot([p["p1"]]), + ] + ) + assert offline_popularity(inst, lv, k=1) == [p["p1"]] + + def test_consensus_example7(self, consensus_data): + p, inst, lv = consensus_data + assert offline_consensus(inst, lv, k=1) == [p["p1"]] + + def test_controversiality_example8(self, consensus_data): + p, inst, lv = consensus_data + assert offline_controversiality(inst, lv, k=1) == [p["p2"]] + + def test_popularity_matches_manual_topk_random(self): + # Cross-check the popularity sampler against an independent top-k. + projects, inst, lv = random_instance(20, 40, 50, seed=33) + k = 5 + counts = manual_scores(projects, lv) + expected = sorted(projects, key=lambda p: (-counts[p], str(p)))[:k] + assert offline_popularity(inst, lv, k=k) == expected + + +# --------------------------------------------------------------------------- +# online_adaptive_controversial +# --------------------------------------------------------------------------- +class TestOnlineAdaptive: + def test_example9(self): + p = make_projects([("p1", 1), ("p2", 1), ("p3", 1), ("p4", 1)]) + inst = Instance(p.values(), budget_limit=4) + lv = ApprovalProfile( + [ + ApprovalBallot([p["p1"], p["p2"]]), + ApprovalBallot([p["p1"], p["p3"]]), + ApprovalBallot([p["p2"], p["p3"]]), + ApprovalBallot([]), + ] + ) + assert online_adaptive_controversial( + inst, lv, {p["p1"], p["p2"]}, k=2 + ) == [p["p1"], p["p2"]] + + def test_returns_k_distinct(self): + projects, inst, lv = random_instance(15, 30, 60, seed=9) + result = online_adaptive_controversial(inst, lv, set(projects[:5]), k=4) + assert len(result) == 4 + assert len(set(result)) == 4 + assert set(result) <= set(projects) + + +# --------------------------------------------------------------------------- +# PartialApprovalBallot + reveal_ballot (three-state ballots, Section 2.3) +# --------------------------------------------------------------------------- +class TestPartialBallot: + def test_states_partition_projects(self): + p = make_projects([("p1", 1), ("p2", 1), ("p3", 1)]) + b = PartialApprovalBallot( + approved={p["p1"]}, disapproved={p["p2"]}, hidden={p["p3"]} + ) + assert b.approved == {p["p1"]} + assert b.disapproved == {p["p2"]} + assert b.hidden == {p["p3"]} + assert b.exposed == {p["p1"], p["p2"]} + assert (b.approved | b.disapproved | b.hidden) == set(p.values()) + + def test_as_approval_ballot_keeps_only_approvals(self): + p = make_projects([("p1", 1), ("p2", 1)]) + b = PartialApprovalBallot(approved={p["p1"]}, disapproved={p["p2"]}) + ab = b.as_approval_ballot() + assert isinstance(ab, ApprovalBallot) + assert set(ab) == {p["p1"]} + + def test_reveal_example2_4(self): + # Example 2.4: true approvals {p1,p2}, exposed {p1,p3}. + p = make_projects([("p1", 1), ("p2", 1), ("p3", 1), ("p4", 1)]) + inst = Instance(p.values(), budget_limit=4) + b = reveal_ballot(inst, {p["p1"], p["p2"]}, {p["p1"], p["p3"]}) + assert b.approved == {p["p1"]} + assert b.disapproved == {p["p3"]} + assert b.hidden == {p["p2"], p["p4"]} + assert b.exposed == {p["p1"], p["p3"]} + + +# --------------------------------------------------------------------------- +# predict_by_majority +# --------------------------------------------------------------------------- +class TestPredictByMajority: + def test_example11(self): + p = make_projects([("p1", 4), ("p2", 4), ("p3", 6)]) + inst = Instance(p.values(), budget_limit=6) + lv = ApprovalProfile( + [ApprovalBallot([p["p1"], p["p2"]]), ApprovalBallot([p["p1"], p["p2"]])] + ) + partial = PartialApprovalBallot(hidden=set(p.values())) + pred = predict_by_majority(inst, lv, partial) + assert set(pred) == {p["p1"], p["p2"]} + + def test_exposed_approvals_are_kept(self): + p = make_projects([("p1", 1), ("p2", 1), ("p3", 1)]) + inst = Instance(p.values(), budget_limit=3) + # LV would reject p3 (0%), but the voter explicitly approved it. + lv = ApprovalProfile([ApprovalBallot([p["p1"]]), ApprovalBallot([p["p1"]])]) + partial = PartialApprovalBallot( + approved={p["p3"]}, hidden={p["p1"], p["p2"]} + ) + pred = predict_by_majority(inst, lv, partial) + assert p["p3"] in pred + + def test_exposed_disapprovals_stay_rejected(self): + p = make_projects([("p1", 1), ("p2", 1)]) + inst = Instance(p.values(), budget_limit=2) + # LV approves both p1 and p2 unanimously, but the voter explicitly + # disapproved p1; p2 is hidden and should be predicted as approved. + lv = ApprovalProfile([ApprovalBallot([p["p1"], p["p2"]])] * 3) + partial = PartialApprovalBallot(disapproved={p["p1"]}, hidden={p["p2"]}) + pred = predict_by_majority(inst, lv, partial) + assert p["p1"] not in pred # the explicit disapproval is honoured + assert p["p2"] in pred # hidden + LV majority -> approved + + +# --------------------------------------------------------------------------- +# Library-backed predictors: classification (XGBoost), MF, FM (Section 2.1). +# They share the contract of predict_by_majority, so the same behavioural +# invariants are checked for each one. +# --------------------------------------------------------------------------- +LIBRARY_PREDICTORS = [ + predict_by_classification, + predict_by_matrix_factorization, + predict_by_factorization_machines, +] + + +class TestLibraryPredictors: + @pytest.mark.parametrize("predictor", LIBRARY_PREDICTORS) + def test_strong_pattern_recovered(self, predictor): + # LV unanimously approve {p1, p2} and reject p3; a TV voter with nothing + # exposed must be completed to {p1, p2} by any correct predictor. + p = make_projects([("p1", 4), ("p2", 4), ("p3", 6)]) + inst = Instance(p.values(), budget_limit=6) + lv = ApprovalProfile([ApprovalBallot([p["p1"], p["p2"]])] * 4) + partial = PartialApprovalBallot(hidden=set(p.values())) + pred = predictor(inst, lv, partial) + assert set(pred) == {p["p1"], p["p2"]} + + @pytest.mark.parametrize("predictor", LIBRARY_PREDICTORS) + def test_exposed_votes_are_respected(self, predictor): + # The exposed approval p3 is kept and the exposed disapproval p1 stays + # rejected, regardless of what the model would otherwise predict. + p = make_projects([("p1", 1), ("p2", 1), ("p3", 1)]) + inst = Instance(p.values(), budget_limit=3) + lv = ApprovalProfile([ApprovalBallot([p["p1"]])] * 3) + partial = PartialApprovalBallot( + approved={p["p3"]}, disapproved={p["p1"]}, hidden={p["p2"]} + ) + pred = predictor(inst, lv, partial) + assert p["p3"] in pred # exposed approval kept + assert p["p1"] not in pred # exposed disapproval rejected + + +# --------------------------------------------------------------------------- +# recommend (full pipeline) +# --------------------------------------------------------------------------- +class TestRecommend: + def test_example10_perfect(self): + p = make_projects([("p1", 3), ("p2", 3), ("p3", 4), ("p4", 4)]) + inst = Instance(p.values(), budget_limit=6) + lv = ApprovalProfile([ApprovalBallot([p["p1"], p["p2"]])] * 3) + bundle = set(recommend(inst, lv, {"v4": {p["p1"], p["p2"]}}, k=1)) + assert bundle == {p["p1"], p["p2"]} + + def test_pipeline_respects_budget_random(self): + # Cheap, widely-approved projects so the pipeline must fund something. + p = make_projects([("p1", 2), ("p2", 2), ("p3", 9), ("p4", 9)]) + inst = Instance(p.values(), budget_limit=6) + lv = ApprovalProfile([ApprovalBallot([p["p1"], p["p2"]])] * 4) + tv = {"v1": {p["p1"], p["p2"]}, "v2": {p["p1"]}} + bundle = list(recommend(inst, lv, tv, k=2)) + assert set(bundle) <= set(p.values()) + assert sum(proj.cost for proj in bundle) <= inst.budget_limit + assert len(bundle) > 0 + + +# --------------------------------------------------------------------------- +# symmetric_distance +# --------------------------------------------------------------------------- +class TestSymmetricDistance: + def test_identical_bundles(self): + p = make_projects([("p1", 1), ("p2", 1)]) + assert symmetric_distance(set(p.values()), set(p.values())) == 0 + # bundles differing by one project have distance 1. + assert symmetric_distance(set(p.values()), {p["p1"]}) == 1 + + def test_paper_toy_examples(self): + a, b, c, d, e = (Project(x, 1) for x in "abcde") + assert symmetric_distance({a, b, c}, {b, a, c}) == 0 + # NB: the paper's text prints 1 here, but the symmetric difference of + # {a,b,c} and {a,c,d} is {b,d}, i.e. 2 (a typo in the paper). + assert symmetric_distance({a, b, c}, {a, c, d}) == 2 + assert symmetric_distance({a, b, c}, {a, d, e}) == 4 + + def test_large_structured_known_value(self): + # rb = first 60, pb = last 60 of 100 -> overlap 20, symmetric diff 80. + a = padded_projects(100, cost=1) + assert symmetric_distance(set(a[:60]), set(a[40:])) == 80 + + +# --------------------------------------------------------------------------- +# fractional_allocation_score +# --------------------------------------------------------------------------- +class TestFractionalAllocation: + def test_example11_zero_score(self): + p = make_projects([("p1", 4), ("p3", 6)]) + # disjoint bundles -> 0.0; a full overlap of cost 6 over budget 6 -> 1.0. + assert fractional_allocation_score({p["p3"]}, {p["p1"]}, budget_limit=6) == 0.0 + assert fractional_allocation_score({p["p3"]}, {p["p3"]}, budget_limit=6) == 1.0 + + def test_partial_overlap(self): + p = make_projects([("p1", 2), ("p2", 3), ("p3", 5)]) + # overlap is {p2}, cost 3, budget 10 -> 0.3 + score = fractional_allocation_score( + {p["p1"], p["p2"]}, {p["p2"], p["p3"]}, budget_limit=10 + ) + assert score == pytest.approx(0.3) + + def test_value_matches_overlap_random(self): + projects, inst, _ = random_instance(12, 1, 40, seed=14) + rng = random.Random(15) + rb = set(rng.sample(projects, 6)) + pb = set(rng.sample(projects, 6)) + score = fractional_allocation_score(rb, pb, budget_limit=inst.budget_limit) + expected = sum(p.cost for p in (rb & pb)) / inst.budget_limit + assert 0.0 <= score <= 1.0 + assert score == pytest.approx(expected) + + +if __name__ == "__main__": + import sys + + sys.exit(pytest.main([__file__, "-v"])) From 826bde4a74b89928dd71e340db37e10508b78181 Mon Sep 17 00:00:00 2001 From: Roei Date: Wed, 1 Jul 2026 19:06:25 +0300 Subject: [PATCH 02/16] Restructure PB recommendation per maintainer/professor feedback - Represent three-state partial ballots with pabutools' CardinalBallot (+1 approve / -1 disapprove / 0 hidden) instead of a new ballot type - Split model training into a separate, ballot-agnostic module (pb_model_training.py); ML libs are an optional "recommendation" extra imported lazily - Reuse pabutools where the primitive exists; keep empty stubs for this stage - Samplers return the exposed set E_v as set[Project]; random_setup drops the unused ballots argument; remove symmetric_distance (per instructor) - Use the paper's vocabulary throughout (Learning/Target Voters, full/partial ballot, ideal instance, A_v/D_v/E_v/H_v) and update the unit tests Co-Authored-By: Claude Opus 4.8 --- pb_model_training.py | 195 ++++++++++++++++++ pb_recommendation.py | 418 ++++++++++++++++++++------------------ pyproject.toml | 5 + test_pb_recommendation.py | 133 ++++++------ 4 files changed, 494 insertions(+), 257 deletions(-) create mode 100644 pb_model_training.py diff --git a/pb_model_training.py b/pb_model_training.py new file mode 100644 index 0000000..2d6def6 --- /dev/null +++ b/pb_model_training.py @@ -0,0 +1,195 @@ +""" +Model *training* for the learning-based prediction modules of +"A Recommendation System for Participatory Budgeting" +(Leibiker & Talmon, 2023), Section 2.1. + +This module is intentionally separate from the module that *uses* the models +(``pb_recommendation``): a ``train_*`` function fits an estimator and returns an +opaque model object, while the matching ``predict_by_*`` function in +``pb_recommendation`` consumes it to complete a single Target Voter's partial +ballot. Keeping them apart also lets the (heavy) ML dependencies be imported only +on the training side. + +What each model trains on follows the paper (Section 3.1: predictions use the +preferences of LV *and* the preferences in E_TV): + +* :py:func:`train_classification` is supervised, so it trains on the **Learning + Voters' full ballots only** (features = votes on the exposed projects). The + Target Voter's exposed votes enter later, as features, at prediction time - so + one trained classifier can be reused across many Target Voters. +* :py:func:`train_matrix_factorization` and + :py:func:`train_factorization_machines` are collaborative filtering, so the + Target Voter must be **inside** the fitted user-item matrix: they train on the + Learning Voters' full ballots **and** that Target Voter's exposed set E_v + (passed in as the ``approved`` / ``disapproved`` project sets), and are + therefore (re)fitted per Target Voter. + +.. note:: + Per the pabutools maintainer (Simon Rey, issue thread): completing a partial + vote should live in a **separate module** and the ML libraries must **not be + a hard requirement** - exactly how pabutools treats Jinja for rule + explanations. So every ML dependency here is imported lazily inside the + ``train_*`` function that needs it, raising a friendly ``ImportError`` if it + is missing, and is declared only as an optional extra in ``pyproject.toml``. + +Programmer: Roei Yanku +Date: 2026-06-20. +""" + +from __future__ import annotations + +from pabutools.election import ( + Instance, + Project, + ApprovalProfile, +) + +# This module is deliberately *ballot-agnostic*: it takes plain project sets, not +# the partial CardinalBallot. The caller (``pb_recommendation``) decodes the +# ballot into approved / disapproved / exposed sets and passes those in. That +# keeps the +1/-1/0 convention in one place (``pb_recommendation``) and the +# import one-directional (``pb_recommendation`` -> here), with no cycle. + + +# --------------------------------------------------------------------------- +# Section 2.1.1 - Binary classification (one classifier per project). +# --------------------------------------------------------------------------- +def train_classification( + instance: Instance, + lv_profile: ApprovalProfile, + exposed: set[Project], +) -> object: + """ + Train the per-project binary classifiers of Section 2.1.1 on the LV ballots. + For every project the voter might be asked to predict, a classifier is fitted + whose features are the votes on the ``exposed`` projects and whose label is + the vote on the target project. Backed by the external ``xgboost`` library + (:py:class:`xgboost.XGBClassifier`, class-weighted loss for the imbalanced + data), imported inside the implementation. + + Parameters + ---------- + instance : :py:class:`~pabutools.election.instance.Instance` + The PB instance. + lv_profile : :py:class:`~pabutools.election.profile.approvalprofile.ApprovalProfile` + The full ballots of the LV voters, used as the training data. + exposed : set[:py:class:`~pabutools.election.instance.Project`] + The projects used as features (those exposed to the TV voters). + + Returns + ------- + object + A fitted model (e.g. a mapping from each hidden project to its + trained classifier) to be passed to + :py:func:`pb_recommendation.predict_by_classification`. + """ + try: + import xgboost # noqa: F401 (optional dependency, used once implemented) + except ImportError: + raise ImportError( + "You need to install xgboost to use the classification predictor " + "(pip install pabutools[recommendation])." + ) + return None # Empty implementation + + +# --------------------------------------------------------------------------- +# Section 2.1.2 - Collaborative filtering via Matrix Factorization. +# --------------------------------------------------------------------------- +def train_matrix_factorization( + instance: Instance, + lv_profile: ApprovalProfile, + approved: set[Project], + disapproved: set[Project], +) -> object: + """ + Train the Matrix Factorization model of Section 2.1.2. Collaborative + filtering needs the Target Voter inside the matrix, so the model is fitted on + **both** the Learning Voters' full ballots **and** the Target Voter's exposed + set E_v (paper Section 3.1: predictions use the preferences of LV *and* the + preferences in E_TV): build the sparse user-item rating matrix with one row + per LV voter (full) plus one row for this Target Voter holding only her + exposed votes (``approved`` -> 1, ``disapproved`` -> 0; the hidden projects + are left empty), then factorise it. The reconstructed cells of the Target + Voter's row over the hidden projects are what gets read off at prediction + time. Without her exposed votes the model could only reproduce the LV + average. Backed by the external ``scikit-surprise`` library + (:py:class:`surprise.SVD`), imported inside the implementation. + + Parameters + ---------- + instance : :py:class:`~pabutools.election.instance.Instance` + The PB instance. + lv_profile : :py:class:`~pabutools.election.profile.approvalprofile.ApprovalProfile` + The full ballots of the Learning Voters (LV). + approved : set[:py:class:`~pabutools.election.instance.Project`] + The Target Voter's exposed approvals A_v (known cells set to 1). + disapproved : set[:py:class:`~pabutools.election.instance.Project`] + The Target Voter's exposed disapprovals D_v (known cells set to 0). + + Returns + ------- + object + A fitted factorisation model to be passed to + :py:func:`pb_recommendation.predict_by_matrix_factorization`. + """ + try: + import surprise # noqa: F401 (optional dependency, used once implemented) + except ImportError: + raise ImportError( + "You need to install scikit-surprise to use the matrix-factorization " + "predictor (pip install pabutools[recommendation])." + ) + return None # Empty implementation + + +# --------------------------------------------------------------------------- +# Section 2.1.2 - Hybrid Factorization Machines. +# --------------------------------------------------------------------------- +def train_factorization_machines( + instance: Instance, + lv_profile: ApprovalProfile, + approved: set[Project], + disapproved: set[Project], +) -> object: + """ + Train the Factorization Machines model of Section 2.1.2: like Matrix + Factorization but with a linear term plus pairwise latent interactions and + optional side features. Being collaborative filtering, it is fitted on + **both** the Learning Voters' full ballots **and** the Target Voter's exposed + set E_v (``approved`` -> 1, ``disapproved`` -> 0), exactly as in + :py:func:`train_matrix_factorization`; the hidden projects are what the fitted + model predicts. Backed by an external FM library (e.g. ``lightfm`` / + ``fastFM``), imported inside the implementation. + + Parameters + ---------- + instance : :py:class:`~pabutools.election.instance.Instance` + The PB instance. + lv_profile : :py:class:`~pabutools.election.profile.approvalprofile.ApprovalProfile` + The full ballots of the Learning Voters (LV). + approved : set[:py:class:`~pabutools.election.instance.Project`] + The Target Voter's exposed approvals A_v (known cells set to 1). + disapproved : set[:py:class:`~pabutools.election.instance.Project`] + The Target Voter's exposed disapprovals D_v (known cells set to 0). + + Returns + ------- + object + A fitted FM model to be passed to + :py:func:`pb_recommendation.predict_by_factorization_machines`. + """ + try: + import lightfm # noqa: F401 (optional dependency, used once implemented) + except ImportError: + raise ImportError( + "You need to install lightfm to use the factorization-machines " + "predictor (pip install pabutools[recommendation])." + ) + return None # Empty implementation + + +if __name__ == "__main__": + import doctest + + doctest.testmod(verbose=True) diff --git a/pb_recommendation.py b/pb_recommendation.py index 965c462..640f9f2 100644 --- a/pb_recommendation.py +++ b/pb_recommendation.py @@ -3,121 +3,120 @@ "A Recommendation System for Participatory Budgeting", by Gil Leibiker and Nimrod Talmon (2023), https://optlearnmas23.github.io/files/p17.pdf +Terminology follows the paper: the voters are partitioned into the **Learning +Voters (LV)**, who already provided their full ballots, and the **Target Voters +(TV)**, who provide partial ballots. The goal is to estimate the **ideal +instance** (where every voter provided a full ballot) from the partial one: for +each TV voter the algorithm reveals k projects into her **exposed set** E_v +(splitting into the **approval set** A_v and **disapproval set** D_v) and +predicts her **hidden set** H_v. + Programmer: Roei Yanku Date: 2026-06-20. """ from __future__ import annotations +from collections.abc import Iterable + from pabutools.election import ( Instance, Project, ApprovalProfile, ApprovalBallot, + CardinalBallot, ) from pabutools.rules import BudgetAllocation +# Model training lives in a separate module (the professor's note b: separate +# files for training the model and using it). The learning-based predictors +# below fit their model there and only *use* it here. ``pb_model_training`` is +# ballot-agnostic (it takes plain project sets), so the import is one-directional +# and there is no circular dependency. +from pb_model_training import ( + train_classification, + train_matrix_factorization, + train_factorization_machines, +) -# --------------------------------------------------------------------------- -# Section 2.3 - Partial ballots (three-state approval ballots). -# --------------------------------------------------------------------------- -class PartialApprovalBallot: - """ - A three-state approval ballot (Section 2.3), splitting the projects into the - approval set A_v (``approved``), the disapproval set D_v (``disapproved``) - and the unknown set H_v (``hidden``). They partition P (A_v ∪ D_v ∪ H_v = P) - and the exposed set is E_v = A_v ∪ D_v. pabutools' ApprovalBallot cannot - express this (no disapproved/unknown distinction), so this helper class is - added here. - Parameters - ---------- - approved : Iterable[:py:class:`~pabutools.election.instance.Project`], optional - The projects the voter approves (A_v). Defaults to ``()``. - disapproved : Iterable[:py:class:`~pabutools.election.instance.Project`], optional - The projects the voter disapproves (D_v). Defaults to ``()``. - hidden : Iterable[:py:class:`~pabutools.election.instance.Project`], optional - The projects whose preference is unknown (H_v). Defaults to ``()``. - - Attributes - ---------- - approved : set[:py:class:`~pabutools.election.instance.Project`] - The approval set A_v. - disapproved : set[:py:class:`~pabutools.election.instance.Project`] - The disapproval set D_v. - hidden : set[:py:class:`~pabutools.election.instance.Project`] - The hidden set H_v. +# =========================================================================== +# Section 2.3 - Partial (three-state) ballots, encoded as a CardinalBallot. +# =========================================================================== +# A Target Voter answers only k projects, so "unknown" must be distinct from +# "disapproved" - which a plain approval set (just the approved projects) cannot +# express. On the pabutools maintainer's advice (Simon Rey) we do NOT add a new +# ballot type; instead we reuse the existing +# :py:class:`~pabutools.election.ballot.cardinalballot.CardinalBallot` (a dict of +# project -> score) under this sign convention: +# +# score > 0 -> APPROVED (A_v) we store +1 (APPROVAL) +# score < 0 -> DISAPPROVED (D_v) we store -1 (DISAPPROVAL) +# score == 0 -> HIDDEN (H_v) we store 0 (HIDDEN) +# +# A project simply absent from the ballot is also HIDDEN. This convention is not +# self-evident, so never hand-write the scores: build ballots with +# ``partial_ballot`` / ``reveal_ballot`` and read them back with the +# ``*_projects`` accessors / ``as_approval_ballot``. + +#: Score stored for an approved project (A_v). Any strictly positive score works. +APPROVAL = 1 +#: Score stored for a disapproved project (D_v). Any strictly negative score works. +DISAPPROVAL = -1 +#: Score stored for a hidden project (H_v); absent projects mean the same. +HIDDEN = 0 + + +def partial_ballot( + approved: Iterable[Project] = (), + disapproved: Iterable[Project] = (), + hidden: Iterable[Project] = (), +) -> CardinalBallot: + """ + Build a partial ballot from the three explicit sets, hiding the convention: + approved projects get :data:`APPROVAL`, disapproved get :data:`DISAPPROVAL`, + hidden get :data:`HIDDEN`. This is the intended way to create a partial + ballot - callers name the three sets instead of writing raw scores. Examples -------- >>> p1, p2, p3 = Project("p1", 1), Project("p2", 1), Project("p3", 1) - >>> b = PartialApprovalBallot(approved={p1}, disapproved={p2}, hidden={p3}) - >>> b.exposed == {p1, p2} - True - >>> (b.approved | b.disapproved | b.hidden) == {p1, p2, p3} + >>> b = partial_ballot(approved={p1}, disapproved={p2}, hidden={p3}) + >>> b[p1], b[p2], b[p3] + (1, -1, 0) + >>> exposed_projects(b) == {p1, p2} True """ - - def __init__(self, approved=(), disapproved=(), hidden=()) -> None: - self.approved: set[Project] = set() # Empty implementation - self.disapproved: set[Project] = set() # Empty implementation - self.hidden: set[Project] = set() # Empty implementation - - @property - def exposed(self) -> set[Project]: - """The exposed set E_v = A_v ∪ D_v (the projects the voter answered).""" - return set() # Empty implementation - - def as_approval_ballot(self) -> ApprovalBallot: - """ - The pabutools approval ballot made of the approved projects A_v, so that - a (completed) partial ballot can be fed to approval-based voting rules. - """ - return ApprovalBallot() # Empty implementation - - def __eq__(self, other) -> bool: - return ( - isinstance(other, PartialApprovalBallot) - and self.approved == other.approved - and self.disapproved == other.disapproved - and self.hidden == other.hidden - ) - - def __repr__(self) -> str: - return ( - f"PartialApprovalBallot(approved={sorted(map(str, self.approved))}, " - f"disapproved={sorted(map(str, self.disapproved))}, " - f"hidden={sorted(map(str, self.hidden))})" - ) + return CardinalBallot() # Empty implementation def reveal_ballot( instance: Instance, - true_approval: set[Project], + full_ballot: set[Project], exposed: set[Project], -) -> PartialApprovalBallot: +) -> CardinalBallot: """ - Build the partial ballot exposing a set of projects of a voter whose true - approval set is known (Section 2.3): A_v = exposed ∩ true, - D_v = exposed \\ true, H_v = P \\ exposed. + Build the partial ballot exposing a set of projects of a voter whose full + ballot in the ideal instance is known (Section 2.3): A_v = E_v ∩ full_ballot, + D_v = E_v \\ full_ballot, H_v = P \\ E_v. Parameters ---------- instance : :py:class:`~pabutools.election.instance.Instance` The PB instance (used as the universe of projects P). - true_approval : set[:py:class:`~pabutools.election.instance.Project`] - The projects the voter truly approves. + full_ballot : set[:py:class:`~pabutools.election.instance.Project`] + The voter's full ballot - her approval set A_v in the ideal instance. exposed : set[:py:class:`~pabutools.election.instance.Project`] - The projects revealed for this voter (E_v). + The projects revealed for this voter (the exposed set E_v). Returns ------- - PartialApprovalBallot - The corresponding three-state ballot. + :py:class:`~pabutools.election.ballot.cardinalballot.CardinalBallot` + The corresponding three-state ballot under the sign convention. Examples -------- - Example 2.4 from the paper: P = {p1, p2, p3, p4}, the voter truly approves + Example 2.4 from the paper: P = {p1, p2, p3, p4}, the voter's full ballot is {p1, p2}, and {p1, p3} is exposed. Then p1 is approved, p3 disapproved, and p2, p4 remain hidden. @@ -125,10 +124,43 @@ def reveal_ballot( ... Project("p3", 1), Project("p4", 1)) >>> inst = Instance([p1, p2, p3, p4], budget_limit=4) >>> b = reveal_ballot(inst, {p1, p2}, {p1, p3}) - >>> b.approved == {p1}, b.disapproved == {p3}, b.hidden == {p2, p4} - (True, True, True) + >>> approved_projects(b) == {p1}, disapproved_projects(b) == {p3} + (True, True) + >>> hidden_projects(b, inst) == {p2, p4} + True """ - return PartialApprovalBallot() # Empty implementation + return CardinalBallot() # Empty implementation + + +def approved_projects(ballot: CardinalBallot) -> set[Project]: + """The approval set A_v: the projects with a strictly positive score.""" + return set() # Empty implementation + + +def disapproved_projects(ballot: CardinalBallot) -> set[Project]: + """The disapproval set D_v: the projects with a strictly negative score.""" + return set() # Empty implementation + + +def exposed_projects(ballot: CardinalBallot) -> set[Project]: + """The exposed set E_v = A_v ∪ D_v: the projects with a non-zero score.""" + return set() # Empty implementation + + +def hidden_projects(ballot: CardinalBallot, instance: Instance) -> set[Project]: + """ + The hidden set H_v = P \\ E_v: every project of the instance that the voter + was not asked about (score 0 or absent from the ballot). + """ + return set() # Empty implementation + + +def as_approval_ballot(ballot: CardinalBallot) -> ApprovalBallot: + """ + The pabutools approval ballot made of the approved projects A_v, so that a + (completed) partial ballot can be fed to approval-based voting rules. + """ + return ApprovalBallot() # Empty implementation # --------------------------------------------------------------------------- @@ -141,6 +173,13 @@ def approval_scores( Definition 2.1 (Approval scores): the approval score of a project is the number of voters that approve it. + This is exactly pabutools' + :py:meth:`~pabutools.election.profile.approvalprofile.AbstractApprovalProfile.approval_scores`, + so we delegate to the library instead of re-counting the ballots ourselves. + The only addition is that projects approved by nobody (absent from the + library's dictionary) are filled in with a score of 0, so every project of + the instance is present in the result. + Parameters ---------- instance : :py:class:`~pabutools.election.instance.Instance` @@ -256,44 +295,42 @@ def greedy_approval( # --------------------------------------------------------------------------- def random_setup( instance: Instance, - tv_ballots: dict[str, set[Project]], k: int, seed: int | None = None, -) -> dict[str, set[Project]]: +) -> set[Project]: """ - Algorithm 1 - Random setup (Section 3.1.1): expose k projects chosen - uniformly at random from P for each TV voter (the rest is predicted later). + Algorithm 1 - Random setup (Section 3.1.1): the exposed set E_v of a Target + Voter, made of k projects chosen uniformly at random from P (the rest is + predicted later). Called once per TV voter, so a different draw can be drawn + for each; it needs no ballot, since the choice is random. Parameters ---------- instance : :py:class:`~pabutools.election.instance.Instance` - The PB instance. - tv_ballots : dict[str, set[:py:class:`~pabutools.election.instance.Project`]] - The true (hidden) approval set of every TV voter, keyed by voter id. - Projects outside the set are disapproved by that voter. + The PB instance (the universe of projects P). k : int - The number of projects to expose per voter. + The number of projects to expose. seed : int, optional Seed for the random generator, for reproducibility. Returns ------- - dict[str, set[:py:class:`~pabutools.election.instance.Project`]] - For each TV voter, the set of k exposed projects. + set[:py:class:`~pabutools.election.instance.Project`] + The exposed set E_v: k projects drawn uniformly at random from P. Examples -------- - Example 5 from the paper: a single TV voter v4, k = 2. Whatever the draw, - exactly two projects are exposed and they are real projects of the instance. + Example 5 from the paper: k = 2. Whatever the draw, exactly two projects are + exposed and they are real projects of the instance. >>> p1, p2, p3, p4 = (Project("p1", 2), Project("p2", 2), ... Project("p3", 3), Project("p4", 3)) >>> inst = Instance([p1, p2, p3, p4], budget_limit=6) - >>> exposed = random_setup(inst, {"v4": {p1, p2}}, k=2, seed=0) - >>> len(exposed["v4"]) == 2 and exposed["v4"] <= {p1, p2, p3, p4} + >>> exposed = random_setup(inst, k=2, seed=0) + >>> len(exposed) == 2 and exposed <= {p1, p2, p3, p4} True """ - return {} # Empty implementation + return set() # Empty implementation # --------------------------------------------------------------------------- @@ -301,25 +338,28 @@ def random_setup( # --------------------------------------------------------------------------- def offline_popularity( instance: Instance, lv_profile: ApprovalProfile, k: int -) -> list[Project]: +) -> set[Project]: """ - Algorithm 2 - Offline revealing by popularity (Section 3.1.2). Exposes, for - every TV voter, the k most approved projects among the LV voters, i.e. - {sigma_1, ..., sigma_k}. + Algorithm 2 - Offline revealing by popularity (Section 3.1.2). The exposed + set E = {sigma_1, ..., sigma_k}: the k most approved projects among the LV + voters. The same set is exposed to every Target Voter, so it depends only on + the LV profile (the score ordering sigma is used only to pick the top k; the + exposed set itself is unordered). Parameters ---------- instance : :py:class:`~pabutools.election.instance.Instance` The PB instance. lv_profile : :py:class:`~pabutools.election.profile.approvalprofile.ApprovalProfile` - The full ballots of the LV voters. + The full ballots of the Learning Voters (LV). k : int The number of projects to expose. Returns ------- - list[:py:class:`~pabutools.election.instance.Project`] - The k most popular projects, ordered by score (ties by name). + set[:py:class:`~pabutools.election.instance.Project`] + The exposed set E of the k most popular projects (ties broken by name + when picking the top k). Examples -------- @@ -331,33 +371,34 @@ def offline_popularity( >>> lv = ApprovalProfile([ApprovalBallot([p1, p3]), ... ApprovalBallot([p1, p2]), ... ApprovalBallot([p1])]) - >>> offline_popularity(inst, lv, k=1) - [p1] + >>> offline_popularity(inst, lv, k=1) == {p1} + True """ - return [] # Empty implementation + return set() # Empty implementation def offline_consensus( instance: Instance, lv_profile: ApprovalProfile, k: int -) -> list[Project]: +) -> set[Project]: """ - Algorithm 3 - Offline revealing by consensus (Section 3.1.2). Exposes the k - projects with the highest consensus level among the LV voters, i.e. - {gamma_1, ..., gamma_k}. + Algorithm 3 - Offline revealing by consensus (Section 3.1.2). The exposed set + E = {gamma_1, ..., gamma_k}: the k projects with the highest consensus level + among the LV voters. The same set is exposed to every Target Voter. Parameters ---------- instance : :py:class:`~pabutools.election.instance.Instance` The PB instance. lv_profile : :py:class:`~pabutools.election.profile.approvalprofile.ApprovalProfile` - The full ballots of the LV voters. + The full ballots of the Learning Voters (LV). k : int The number of projects to expose. Returns ------- - list[:py:class:`~pabutools.election.instance.Project`] - The k projects most in consensus, ordered by level (ties by name). + set[:py:class:`~pabutools.election.instance.Project`] + The exposed set E of the k projects most in consensus (ties broken by + name when picking the top k). Examples -------- @@ -368,33 +409,35 @@ def offline_consensus( >>> inst = Instance([p1, p2, p3], budget_limit=3) >>> lv = ApprovalProfile([ApprovalBallot([p1, p2]), ApprovalBallot([p1]), ... ApprovalBallot([p1, p2]), ApprovalBallot([p1])]) - >>> offline_consensus(inst, lv, k=1) - [p1] + >>> offline_consensus(inst, lv, k=1) == {p1} + True """ - return [] # Empty implementation + return set() # Empty implementation def offline_controversiality( instance: Instance, lv_profile: ApprovalProfile, k: int -) -> list[Project]: +) -> set[Project]: """ - Algorithm 4 - Offline revealing by controversiality (Section 3.1.2): expose - the k projects *least* in consensus among the LV voters, - {gamma_{m-k+1}, ..., gamma_m} (the hardest to predict, so asked directly). + Algorithm 4 - Offline revealing by controversiality (Section 3.1.2). The + exposed set E = {gamma_{m-k+1}, ..., gamma_m}: the k projects *least* in + consensus among the LV voters (the hardest to predict, so asked directly). + The same set is exposed to every Target Voter. Parameters ---------- instance : :py:class:`~pabutools.election.instance.Instance` The PB instance. lv_profile : :py:class:`~pabutools.election.profile.approvalprofile.ApprovalProfile` - The full ballots of the LV voters. + The full ballots of the Learning Voters (LV). k : int The number of projects to expose. Returns ------- - list[:py:class:`~pabutools.election.instance.Project`] - The k most controversial projects, ordered by increasing consensus. + set[:py:class:`~pabutools.election.instance.Project`] + The exposed set E of the k most controversial projects (ties broken + by name when picking the bottom k). Examples -------- @@ -405,10 +448,10 @@ def offline_controversiality( >>> inst = Instance([p1, p2, p3], budget_limit=3) >>> lv = ApprovalProfile([ApprovalBallot([p1, p2]), ApprovalBallot([p1]), ... ApprovalBallot([p1, p2]), ApprovalBallot([p1])]) - >>> offline_controversiality(inst, lv, k=1) - [p2] + >>> offline_controversiality(inst, lv, k=1) == {p2} + True """ - return [] # Empty implementation + return set() # Empty implementation # --------------------------------------------------------------------------- @@ -417,46 +460,49 @@ def offline_controversiality( def online_adaptive_controversial( instance: Instance, lv_profile: ApprovalProfile, - tv_ballot: set[Project], + full_ballot: set[Project], k: int, -) -> list[Project]: +) -> set[Project]: """ - Algorithm 5 - Online adaptive-controversial setup (Section 3.1.3): in each of - k iterations recompute the most controversial project given the LV ballots - and the answers already revealed, then ask about it (each answer affects the - next question). + Algorithm 5 - Online adaptive-controversial setup (Section 3.1.3): the + exposed set E_v of one Target Voter, built in k iterations. In each iteration + the most controversial project is recomputed given the LV ballots and the + answers already revealed, then asked about (each answer affects the next + question, hence "adaptive"). The voter's full ballot is needed to simulate + those answers. Parameters ---------- instance : :py:class:`~pabutools.election.instance.Instance` The PB instance. lv_profile : :py:class:`~pabutools.election.profile.approvalprofile.ApprovalProfile` - The full ballots of the LV voters. - tv_ballot : set[:py:class:`~pabutools.election.instance.Project`] - The true (hidden) approval set of the TV voter being queried. + The full ballots of the Learning Voters (LV). + full_ballot : set[:py:class:`~pabutools.election.instance.Project`] + The full ballot of the Target Voter (TV) being queried - her approval + set A_v in the ideal instance - used to answer each adaptive question. k : int The number of iterations / projects to expose. Returns ------- - list[:py:class:`~pabutools.election.instance.Project`] - The k exposed projects, in the order they were asked. + set[:py:class:`~pabutools.election.instance.Project`] + The exposed set E_v of the k projects that ended up being asked. Examples -------- Example 9 from the paper: 4 LV voters, one TV voter v5, k = 2. p1, p2, p3 are all tied as most controversial; the first question (tie broken by name) - is p1, and after the voter's answer the second question is p2. + is p1, and after the voter's answer the next is p2, so E_v = {p1, p2}. >>> p1, p2, p3, p4 = (Project("p1", 1), Project("p2", 1), ... Project("p3", 1), Project("p4", 1)) >>> inst = Instance([p1, p2, p3, p4], budget_limit=4) >>> lv = ApprovalProfile([ApprovalBallot([p1, p2]), ApprovalBallot([p1, p3]), ... ApprovalBallot([p2, p3]), ApprovalBallot([])]) - >>> online_adaptive_controversial(inst, lv, {p1, p2}, k=2) - [p1, p2] + >>> online_adaptive_controversial(inst, lv, {p1, p2}, k=2) == {p1, p2} + True """ - return [] # Empty implementation + return set() # Empty implementation # --------------------------------------------------------------------------- @@ -465,7 +511,7 @@ def online_adaptive_controversial( def predict_by_majority( instance: Instance, lv_profile: ApprovalProfile, - ballot: PartialApprovalBallot, + ballot: CardinalBallot, ) -> ApprovalBallot: """ Prediction module (Section 2.1). Completes a single partial TV ballot into a @@ -479,8 +525,9 @@ def predict_by_majority( The PB instance. lv_profile : :py:class:`~pabutools.election.profile.approvalprofile.ApprovalProfile` The full ballots of the LV voters, used as the training data. - ballot : PartialApprovalBallot - The partial ballot of the TV voter to complete. + ballot : :py:class:`~pabutools.election.ballot.cardinalballot.CardinalBallot` + The Target Voter's partial ballot to complete (the +1/-1/0 partial + ballot built by ``partial_ballot`` / ``reveal_ballot``). Returns ------- @@ -496,7 +543,7 @@ def predict_by_majority( >>> p1, p2, p3 = Project("p1", 4), Project("p2", 4), Project("p3", 6) >>> inst = Instance([p1, p2, p3], budget_limit=6) >>> lv = ApprovalProfile([ApprovalBallot([p1, p2]), ApprovalBallot([p1, p2])]) - >>> partial = PartialApprovalBallot(hidden={p1, p2, p3}) + >>> partial = partial_ballot(hidden={p1, p2, p3}) >>> predict_by_majority(inst, lv, partial) == {p1, p2} True """ @@ -506,16 +553,16 @@ def predict_by_majority( def predict_by_classification( instance: Instance, lv_profile: ApprovalProfile, - ballot: PartialApprovalBallot, + ballot: CardinalBallot, ) -> ApprovalBallot: """ Prediction module - binary classification (Section 2.1.1): predict each hidden project with a per-project binary classifier trained on the LV ballots (features = votes on the exposed projects, label = vote on the target), then applied to the TV voter. Exposed approvals A_v are kept and exposed - disapprovals D_v stay rejected. Backed by the external ``xgboost`` library - (:py:class:`xgboost.XGBClassifier`, class-weighted loss for the imbalanced - data), imported inside the implementation. + disapprovals D_v stay rejected. The classifiers are fitted by + :py:func:`pb_model_training.train_classification` (backed by ``xgboost``); + this function only *applies* the trained model. Parameters ---------- @@ -523,8 +570,9 @@ def predict_by_classification( The PB instance. lv_profile : :py:class:`~pabutools.election.profile.approvalprofile.ApprovalProfile` The full ballots of the LV voters, used as the training data. - ballot : PartialApprovalBallot - The partial ballot of the TV voter to complete. + ballot : :py:class:`~pabutools.election.ballot.cardinalballot.CardinalBallot` + The Target Voter's partial ballot to complete (the +1/-1/0 partial + ballot built by ``partial_ballot`` / ``reveal_ballot``). Returns ------- @@ -540,25 +588,27 @@ def predict_by_classification( >>> p1, p2, p3 = Project("p1", 4), Project("p2", 4), Project("p3", 6) >>> inst = Instance([p1, p2, p3], budget_limit=6) >>> lv = ApprovalProfile([ApprovalBallot([p1, p2]), ApprovalBallot([p1, p2])]) - >>> partial = PartialApprovalBallot(hidden={p1, p2, p3}) + >>> partial = partial_ballot(hidden={p1, p2, p3}) >>> predict_by_classification(inst, lv, partial) == {p1, p2} True """ + train_classification(instance, lv_profile, exposed_projects(ballot)) # model used below return ApprovalBallot() # Empty implementation def predict_by_matrix_factorization( instance: Instance, lv_profile: ApprovalProfile, - ballot: PartialApprovalBallot, + ballot: CardinalBallot, ) -> ApprovalBallot: """ Prediction module - collaborative filtering via Matrix Factorization (Section 2.1.2): build the sparse user-item matrix from the LV ballots and exposed TV votes (approve=1, disapprove=0), factorise it, and predict a hidden project as approved iff its reconstructed score is >= 0.5. Exposed - approvals A_v are kept and exposed disapprovals D_v stay rejected. Backed by - the external ``scikit-surprise`` library (:py:class:`surprise.SVD`). + approvals A_v are kept and exposed disapprovals D_v stay rejected. The model + is fitted by :py:func:`pb_model_training.train_matrix_factorization` (backed + by ``scikit-surprise``); this function only *applies* the trained model. Parameters ---------- @@ -566,8 +616,9 @@ def predict_by_matrix_factorization( The PB instance. lv_profile : :py:class:`~pabutools.election.profile.approvalprofile.ApprovalProfile` The full ballots of the LV voters, used as the training data. - ballot : PartialApprovalBallot - The partial ballot of the TV voter to complete. + ballot : :py:class:`~pabutools.election.ballot.cardinalballot.CardinalBallot` + The Target Voter's partial ballot to complete (the +1/-1/0 partial + ballot built by ``partial_ballot`` / ``reveal_ballot``). Returns ------- @@ -584,24 +635,30 @@ def predict_by_matrix_factorization( ... Project("p3", 4), Project("p4", 4)) >>> inst = Instance([p1, p2, p3, p4], budget_limit=6) >>> lv = ApprovalProfile([ApprovalBallot([p1, p2])] * 3) - >>> partial = PartialApprovalBallot(hidden={p1, p2, p3, p4}) + >>> partial = partial_ballot(hidden={p1, p2, p3, p4}) >>> predict_by_matrix_factorization(inst, lv, partial) == {p1, p2} True """ + train_matrix_factorization( + instance, lv_profile, approved_projects(ballot), disapproved_projects(ballot) + ) # model used below return ApprovalBallot() # Empty implementation def predict_by_factorization_machines( instance: Instance, lv_profile: ApprovalProfile, - ballot: PartialApprovalBallot, + ballot: CardinalBallot, ) -> ApprovalBallot: """ Prediction module - hybrid Factorization Machines (Section 2.1.2): like MF but with a linear term plus pairwise latent interactions and optional side features, predicting a hidden project as approved iff the FM score is >= 0.5. Exposed approvals A_v are kept and exposed disapprovals D_v stay rejected. - Backed by an external FM library (e.g. ``lightfm`` / ``fastFM``). + The model is fitted by + :py:func:`pb_model_training.train_factorization_machines` (backed by an + external FM library, e.g. ``lightfm`` / ``fastFM``); this function only + *applies* the trained model. Parameters ---------- @@ -609,8 +666,9 @@ def predict_by_factorization_machines( The PB instance. lv_profile : :py:class:`~pabutools.election.profile.approvalprofile.ApprovalProfile` The full ballots of the LV voters, used as the training data. - ballot : PartialApprovalBallot - The partial ballot of the TV voter to complete. + ballot : :py:class:`~pabutools.election.ballot.cardinalballot.CardinalBallot` + The Target Voter's partial ballot to complete (the +1/-1/0 partial + ballot built by ``partial_ballot`` / ``reveal_ballot``). Returns ------- @@ -625,10 +683,13 @@ def predict_by_factorization_machines( >>> p1, p2, p3 = Project("p1", 4), Project("p2", 4), Project("p3", 6) >>> inst = Instance([p1, p2, p3], budget_limit=6) >>> lv = ApprovalProfile([ApprovalBallot([p1, p2]), ApprovalBallot([p1, p2])]) - >>> partial = PartialApprovalBallot(hidden={p1, p2, p3}) + >>> partial = partial_ballot(hidden={p1, p2, p3}) >>> predict_by_factorization_machines(inst, lv, partial) == {p1, p2} True """ + train_factorization_machines( + instance, lv_profile, approved_projects(ballot), disapproved_projects(ballot) + ) # model used below return ApprovalBallot() # Empty implementation @@ -653,7 +714,8 @@ def recommend( lv_profile : :py:class:`~pabutools.election.profile.approvalprofile.ApprovalProfile` The full ballots of the LV voters. tv_ballots : dict[str, set[:py:class:`~pabutools.election.instance.Project`]] - The true (hidden) approval set of every TV voter, keyed by voter id. + The full ballot of every Target Voter (TV) - her approval set A_v in + the ideal instance - keyed by voter id. k : int The number of projects to expose per TV voter. @@ -681,38 +743,6 @@ def recommend( # --------------------------------------------------------------------------- # Section 5.2 - Bundle evaluation metrics. # --------------------------------------------------------------------------- -def symmetric_distance( - real_bundle: set[Project], predicted_bundle: set[Project] -) -> int: - """ - Section 5.2.1 - Symmetric distance between the real winning bundle and the - predicted one: the size of their symmetric difference. - - Parameters - ---------- - real_bundle : set[:py:class:`~pabutools.election.instance.Project`] - The bundle obtained from the real (full) ballots. - predicted_bundle : set[:py:class:`~pabutools.election.instance.Project`] - The bundle obtained from the predicted ballots. - - Returns - ------- - int - The number of projects in exactly one of the two bundles. - - Examples - -------- - Example 10 (a perfect prediction) and Example 11 (a complete failure): - - >>> p1, p2, p3 = Project("p1", 1), Project("p2", 1), Project("p3", 1) - >>> symmetric_distance({p1, p2}, {p1, p2}) - 0 - >>> symmetric_distance({p1}, {p3}) - 2 - """ - return 0 # Empty implementation - - def fractional_allocation_score( real_bundle: set[Project], predicted_bundle: set[Project], budget_limit: int ) -> float: diff --git a/pyproject.toml b/pyproject.toml index 47b5c62..5035b63 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,6 +36,11 @@ dependencies = [ ] [project.optional-dependencies] +recommendation = [ + "xgboost", + "scikit-surprise", + "lightfm" +] dev = [ "coverage", "Sphinx", diff --git a/test_pb_recommendation.py b/test_pb_recommendation.py index 0791b41..4dd614d 100644 --- a/test_pb_recommendation.py +++ b/test_pb_recommendation.py @@ -27,8 +27,17 @@ ) from pb_recommendation import ( - PartialApprovalBallot, + # Partial-ballot helpers + the +1/-1/0 convention constants. + partial_ballot, reveal_ballot, + approved_projects, + disapproved_projects, + exposed_projects, + hidden_projects, + as_approval_ballot, + APPROVAL, + DISAPPROVAL, + HIDDEN, approval_scores, consensus_levels, greedy_approval, @@ -42,7 +51,6 @@ predict_by_matrix_factorization, predict_by_factorization_machines, recommend, - symmetric_distance, fractional_allocation_score, ) @@ -231,22 +239,22 @@ class TestRandomSetup: def test_exposes_exactly_k(self): p = make_projects([("p1", 2), ("p2", 2), ("p3", 3), ("p4", 3)]) inst = Instance(p.values(), budget_limit=6) - exposed = random_setup(inst, {"v4": set(p.values())}, k=2, seed=0) - assert len(exposed["v4"]) == 2 - assert exposed["v4"] <= set(p.values()) + exposed = random_setup(inst, k=2, seed=0) + assert len(exposed) == 2 + assert exposed <= set(p.values()) - def test_all_tv_voters_covered(self): + def test_returns_a_set_subset_of_projects(self): projects, inst, _ = random_instance(10, 1, 30, seed=5) - tv = {f"v{i}": set() for i in range(5)} - exposed = random_setup(inst, tv, k=3, seed=7) - assert set(exposed.keys()) == set(tv.keys()) - assert all(len(exposed[v]) == 3 for v in tv) + exposed = random_setup(inst, k=3, seed=7) + assert isinstance(exposed, set) + assert len(exposed) == 3 + assert exposed <= set(projects) - def test_k_zero(self): + def test_k_equals_all_exposes_everything(self): + # Edge case: exposing k = |P| projects reveals the whole instance. p = make_projects([("p1", 1), ("p2", 1)]) inst = Instance(p.values(), budget_limit=2) - exposed = random_setup(inst, {"v1": set()}, k=0, seed=0) - assert exposed["v1"] == set() + assert random_setup(inst, k=2, seed=0) == set(p.values()) # --------------------------------------------------------------------------- @@ -263,22 +271,22 @@ def test_popularity_example6(self): ApprovalBallot([p["p1"]]), ] ) - assert offline_popularity(inst, lv, k=1) == [p["p1"]] + assert offline_popularity(inst, lv, k=1) == {p["p1"]} def test_consensus_example7(self, consensus_data): p, inst, lv = consensus_data - assert offline_consensus(inst, lv, k=1) == [p["p1"]] + assert offline_consensus(inst, lv, k=1) == {p["p1"]} def test_controversiality_example8(self, consensus_data): p, inst, lv = consensus_data - assert offline_controversiality(inst, lv, k=1) == [p["p2"]] + assert offline_controversiality(inst, lv, k=1) == {p["p2"]} def test_popularity_matches_manual_topk_random(self): # Cross-check the popularity sampler against an independent top-k. projects, inst, lv = random_instance(20, 40, 50, seed=33) k = 5 counts = manual_scores(projects, lv) - expected = sorted(projects, key=lambda p: (-counts[p], str(p)))[:k] + expected = set(sorted(projects, key=lambda p: (-counts[p], str(p)))[:k]) assert offline_popularity(inst, lv, k=k) == expected @@ -299,47 +307,70 @@ def test_example9(self): ) assert online_adaptive_controversial( inst, lv, {p["p1"], p["p2"]}, k=2 - ) == [p["p1"], p["p2"]] + ) == {p["p1"], p["p2"]} def test_returns_k_distinct(self): projects, inst, lv = random_instance(15, 30, 60, seed=9) result = online_adaptive_controversial(inst, lv, set(projects[:5]), k=4) + assert isinstance(result, set) assert len(result) == 4 - assert len(set(result)) == 4 - assert set(result) <= set(projects) + assert result <= set(projects) # --------------------------------------------------------------------------- -# PartialApprovalBallot + reveal_ballot (three-state ballots, Section 2.3) +# Partial (three-state) ballots, Section 2.3. Represented as a CardinalBallot +# under the convention: +1 approved, -1 disapproved, 0 (or absent) hidden. # --------------------------------------------------------------------------- class TestPartialBallot: + def test_scores_follow_convention(self): + # The whole point of the encoding: +1 = approve, -1 = disapprove, + # 0 = unknown. Pin it down so the convention can't silently drift. + p = make_projects([("p1", 1), ("p2", 1), ("p3", 1)]) + b = partial_ballot( + approved={p["p1"]}, disapproved={p["p2"]}, hidden={p["p3"]} + ) + assert b[p["p1"]] == APPROVAL == 1 + assert b[p["p2"]] == DISAPPROVAL == -1 + assert b[p["p3"]] == HIDDEN == 0 + def test_states_partition_projects(self): p = make_projects([("p1", 1), ("p2", 1), ("p3", 1)]) - b = PartialApprovalBallot( + inst = Instance(p.values(), budget_limit=3) + b = partial_ballot( approved={p["p1"]}, disapproved={p["p2"]}, hidden={p["p3"]} ) - assert b.approved == {p["p1"]} - assert b.disapproved == {p["p2"]} - assert b.hidden == {p["p3"]} - assert b.exposed == {p["p1"], p["p2"]} - assert (b.approved | b.disapproved | b.hidden) == set(p.values()) + assert approved_projects(b) == {p["p1"]} + assert disapproved_projects(b) == {p["p2"]} + assert hidden_projects(b, inst) == {p["p3"]} + assert exposed_projects(b) == {p["p1"], p["p2"]} + assert ( + approved_projects(b) | disapproved_projects(b) | hidden_projects(b, inst) + ) == set(p.values()) + + def test_absent_project_is_hidden(self): + # A project never mentioned in the ballot is hidden, just like an + # explicit 0 - both belong to H_v. + p = make_projects([("p1", 1), ("p2", 1), ("p3", 1)]) + inst = Instance(p.values(), budget_limit=3) + b = partial_ballot(approved={p["p1"]}, disapproved={p["p2"]}) # p3 absent + assert hidden_projects(b, inst) == {p["p3"]} def test_as_approval_ballot_keeps_only_approvals(self): p = make_projects([("p1", 1), ("p2", 1)]) - b = PartialApprovalBallot(approved={p["p1"]}, disapproved={p["p2"]}) - ab = b.as_approval_ballot() + b = partial_ballot(approved={p["p1"]}, disapproved={p["p2"]}) + ab = as_approval_ballot(b) assert isinstance(ab, ApprovalBallot) assert set(ab) == {p["p1"]} def test_reveal_example2_4(self): - # Example 2.4: true approvals {p1,p2}, exposed {p1,p3}. + # Example 2.4: full ballot {p1,p2}, exposed set {p1,p3}. p = make_projects([("p1", 1), ("p2", 1), ("p3", 1), ("p4", 1)]) inst = Instance(p.values(), budget_limit=4) b = reveal_ballot(inst, {p["p1"], p["p2"]}, {p["p1"], p["p3"]}) - assert b.approved == {p["p1"]} - assert b.disapproved == {p["p3"]} - assert b.hidden == {p["p2"], p["p4"]} - assert b.exposed == {p["p1"], p["p3"]} + assert approved_projects(b) == {p["p1"]} + assert disapproved_projects(b) == {p["p3"]} + assert hidden_projects(b, inst) == {p["p2"], p["p4"]} + assert exposed_projects(b) == {p["p1"], p["p3"]} # --------------------------------------------------------------------------- @@ -352,7 +383,7 @@ def test_example11(self): lv = ApprovalProfile( [ApprovalBallot([p["p1"], p["p2"]]), ApprovalBallot([p["p1"], p["p2"]])] ) - partial = PartialApprovalBallot(hidden=set(p.values())) + partial = partial_ballot(hidden=set(p.values())) pred = predict_by_majority(inst, lv, partial) assert set(pred) == {p["p1"], p["p2"]} @@ -361,7 +392,7 @@ def test_exposed_approvals_are_kept(self): inst = Instance(p.values(), budget_limit=3) # LV would reject p3 (0%), but the voter explicitly approved it. lv = ApprovalProfile([ApprovalBallot([p["p1"]]), ApprovalBallot([p["p1"]])]) - partial = PartialApprovalBallot( + partial = partial_ballot( approved={p["p3"]}, hidden={p["p1"], p["p2"]} ) pred = predict_by_majority(inst, lv, partial) @@ -373,7 +404,7 @@ def test_exposed_disapprovals_stay_rejected(self): # LV approves both p1 and p2 unanimously, but the voter explicitly # disapproved p1; p2 is hidden and should be predicted as approved. lv = ApprovalProfile([ApprovalBallot([p["p1"], p["p2"]])] * 3) - partial = PartialApprovalBallot(disapproved={p["p1"]}, hidden={p["p2"]}) + partial = partial_ballot(disapproved={p["p1"]}, hidden={p["p2"]}) pred = predict_by_majority(inst, lv, partial) assert p["p1"] not in pred # the explicit disapproval is honoured assert p["p2"] in pred # hidden + LV majority -> approved @@ -399,7 +430,7 @@ def test_strong_pattern_recovered(self, predictor): p = make_projects([("p1", 4), ("p2", 4), ("p3", 6)]) inst = Instance(p.values(), budget_limit=6) lv = ApprovalProfile([ApprovalBallot([p["p1"], p["p2"]])] * 4) - partial = PartialApprovalBallot(hidden=set(p.values())) + partial = partial_ballot(hidden=set(p.values())) pred = predictor(inst, lv, partial) assert set(pred) == {p["p1"], p["p2"]} @@ -410,7 +441,7 @@ def test_exposed_votes_are_respected(self, predictor): p = make_projects([("p1", 1), ("p2", 1), ("p3", 1)]) inst = Instance(p.values(), budget_limit=3) lv = ApprovalProfile([ApprovalBallot([p["p1"]])] * 3) - partial = PartialApprovalBallot( + partial = partial_ballot( approved={p["p3"]}, disapproved={p["p1"]}, hidden={p["p2"]} ) pred = predictor(inst, lv, partial) @@ -441,30 +472,6 @@ def test_pipeline_respects_budget_random(self): assert len(bundle) > 0 -# --------------------------------------------------------------------------- -# symmetric_distance -# --------------------------------------------------------------------------- -class TestSymmetricDistance: - def test_identical_bundles(self): - p = make_projects([("p1", 1), ("p2", 1)]) - assert symmetric_distance(set(p.values()), set(p.values())) == 0 - # bundles differing by one project have distance 1. - assert symmetric_distance(set(p.values()), {p["p1"]}) == 1 - - def test_paper_toy_examples(self): - a, b, c, d, e = (Project(x, 1) for x in "abcde") - assert symmetric_distance({a, b, c}, {b, a, c}) == 0 - # NB: the paper's text prints 1 here, but the symmetric difference of - # {a,b,c} and {a,c,d} is {b,d}, i.e. 2 (a typo in the paper). - assert symmetric_distance({a, b, c}, {a, c, d}) == 2 - assert symmetric_distance({a, b, c}, {a, d, e}) == 4 - - def test_large_structured_known_value(self): - # rb = first 60, pb = last 60 of 100 -> overlap 20, symmetric diff 80. - a = padded_projects(100, cost=1) - assert symmetric_distance(set(a[:60]), set(a[40:])) == 80 - - # --------------------------------------------------------------------------- # fractional_allocation_score # --------------------------------------------------------------------------- From 177327b02b57cd967626bc8ea94f2383911fee99 Mon Sep 17 00:00:00 2001 From: Roei Date: Wed, 1 Jul 2026 20:06:05 +0300 Subject: [PATCH 03/16] Make ML predictors pure empty stubs for the headers stage The predict_by_classification/matrix_factorization/factorization_machines bodies still called the train_* functions; that is real code, not an empty implementation. Remove those calls (and the now-unused pb_model_training import) so every function body is an empty stub and all tests fail, as the assignment requires. The predict/train relationship stays documented in the docstrings. Co-Authored-By: Claude Opus 4.8 --- pb_recommendation.py | 29 ++++------------------------- 1 file changed, 4 insertions(+), 25 deletions(-) diff --git a/pb_recommendation.py b/pb_recommendation.py index 640f9f2..caba493 100644 --- a/pb_recommendation.py +++ b/pb_recommendation.py @@ -3,14 +3,6 @@ "A Recommendation System for Participatory Budgeting", by Gil Leibiker and Nimrod Talmon (2023), https://optlearnmas23.github.io/files/p17.pdf -Terminology follows the paper: the voters are partitioned into the **Learning -Voters (LV)**, who already provided their full ballots, and the **Target Voters -(TV)**, who provide partial ballots. The goal is to estimate the **ideal -instance** (where every voter provided a full ballot) from the partial one: for -each TV voter the algorithm reveals k projects into her **exposed set** E_v -(splitting into the **approval set** A_v and **disapproval set** D_v) and -predicts her **hidden set** H_v. - Programmer: Roei Yanku Date: 2026-06-20. """ @@ -28,16 +20,10 @@ ) from pabutools.rules import BudgetAllocation -# Model training lives in a separate module (the professor's note b: separate -# files for training the model and using it). The learning-based predictors -# below fit their model there and only *use* it here. ``pb_model_training`` is -# ballot-agnostic (it takes plain project sets), so the import is one-directional -# and there is no circular dependency. -from pb_model_training import ( - train_classification, - train_matrix_factorization, - train_factorization_machines, -) +# Model training lives in a separate module, ``pb_model_training`` (the +# professor's note b: separate files for training the model and using it). The +# learning-based predictors below will call its ``train_*`` functions once +# implemented; at this stage their bodies are empty, so nothing is imported yet. # =========================================================================== @@ -592,7 +578,6 @@ def predict_by_classification( >>> predict_by_classification(inst, lv, partial) == {p1, p2} True """ - train_classification(instance, lv_profile, exposed_projects(ballot)) # model used below return ApprovalBallot() # Empty implementation @@ -639,9 +624,6 @@ def predict_by_matrix_factorization( >>> predict_by_matrix_factorization(inst, lv, partial) == {p1, p2} True """ - train_matrix_factorization( - instance, lv_profile, approved_projects(ballot), disapproved_projects(ballot) - ) # model used below return ApprovalBallot() # Empty implementation @@ -687,9 +669,6 @@ def predict_by_factorization_machines( >>> predict_by_factorization_machines(inst, lv, partial) == {p1, p2} True """ - train_factorization_machines( - instance, lv_profile, approved_projects(ballot), disapproved_projects(ballot) - ) # model used below return ApprovalBallot() # Empty implementation From bf85250fba9a3d36d6e66fea64677987467c54ab Mon Sep 17 00:00:00 2001 From: Roei Date: Wed, 1 Jul 2026 20:09:28 +0300 Subject: [PATCH 04/16] Turn ballot-encoding constants into comments (headers stage) Replace the APPROVAL/DISAPPROVAL/HIDDEN module constants with a comment documenting the +1/-1/0 convention, so the file contains only headers, empty implementations, and documentation - no concrete code. Update the tests to use the literal scores instead of importing the constants. Co-Authored-By: Claude Opus 4.8 --- pb_recommendation.py | 16 +++++++--------- test_pb_recommendation.py | 9 +++------ 2 files changed, 10 insertions(+), 15 deletions(-) diff --git a/pb_recommendation.py b/pb_recommendation.py index caba493..4a4b275 100644 --- a/pb_recommendation.py +++ b/pb_recommendation.py @@ -45,12 +45,10 @@ # ``partial_ballot`` / ``reveal_ballot`` and read them back with the # ``*_projects`` accessors / ``as_approval_ballot``. -#: Score stored for an approved project (A_v). Any strictly positive score works. -APPROVAL = 1 -#: Score stored for a disapproved project (D_v). Any strictly negative score works. -DISAPPROVAL = -1 -#: Score stored for a hidden project (H_v); absent projects mean the same. -HIDDEN = 0 +# Scores stored (to be defined when the module is implemented): +# approved project (A_v) -> +1 (any strictly positive score) +# disapproved project (D_v) -> -1 (any strictly negative score) +# hidden project (H_v) -> 0 (absent projects mean the same) def partial_ballot( @@ -60,9 +58,9 @@ def partial_ballot( ) -> CardinalBallot: """ Build a partial ballot from the three explicit sets, hiding the convention: - approved projects get :data:`APPROVAL`, disapproved get :data:`DISAPPROVAL`, - hidden get :data:`HIDDEN`. This is the intended way to create a partial - ballot - callers name the three sets instead of writing raw scores. + approved projects get +1, disapproved get -1, and hidden get 0. This is the + intended way to create a partial ballot - callers name the three sets instead + of writing raw scores. Examples -------- diff --git a/test_pb_recommendation.py b/test_pb_recommendation.py index 4dd614d..c5a83f6 100644 --- a/test_pb_recommendation.py +++ b/test_pb_recommendation.py @@ -35,9 +35,6 @@ exposed_projects, hidden_projects, as_approval_ballot, - APPROVAL, - DISAPPROVAL, - HIDDEN, approval_scores, consensus_levels, greedy_approval, @@ -329,9 +326,9 @@ def test_scores_follow_convention(self): b = partial_ballot( approved={p["p1"]}, disapproved={p["p2"]}, hidden={p["p3"]} ) - assert b[p["p1"]] == APPROVAL == 1 - assert b[p["p2"]] == DISAPPROVAL == -1 - assert b[p["p3"]] == HIDDEN == 0 + assert b[p["p1"]] == 1 + assert b[p["p2"]] == -1 + assert b[p["p3"]] == 0 def test_states_partition_projects(self): p = make_projects([("p1", 1), ("p2", 1), ("p3", 1)]) From 7cf878b5af88d9ce743b68bf29a517a8a393a44b Mon Sep 17 00:00:00 2001 From: Roei Date: Wed, 1 Jul 2026 20:21:08 +0300 Subject: [PATCH 05/16] Make train_* pure empty stubs (headers stage) Remove the lazy import guards (try/except ImportError for xgboost/surprise/ lightfm) from the train_* bodies - that was real code. Each body is now just `return None # Empty implementation`. The optional-dependency behaviour stays documented in the docstrings for the later implementation stage. Co-Authored-By: Claude Opus 4.8 --- pb_model_training.py | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/pb_model_training.py b/pb_model_training.py index 2d6def6..7df8fab 100644 --- a/pb_model_training.py +++ b/pb_model_training.py @@ -83,13 +83,6 @@ def train_classification( trained classifier) to be passed to :py:func:`pb_recommendation.predict_by_classification`. """ - try: - import xgboost # noqa: F401 (optional dependency, used once implemented) - except ImportError: - raise ImportError( - "You need to install xgboost to use the classification predictor " - "(pip install pabutools[recommendation])." - ) return None # Empty implementation @@ -133,13 +126,6 @@ def train_matrix_factorization( A fitted factorisation model to be passed to :py:func:`pb_recommendation.predict_by_matrix_factorization`. """ - try: - import surprise # noqa: F401 (optional dependency, used once implemented) - except ImportError: - raise ImportError( - "You need to install scikit-surprise to use the matrix-factorization " - "predictor (pip install pabutools[recommendation])." - ) return None # Empty implementation @@ -179,13 +165,6 @@ def train_factorization_machines( A fitted FM model to be passed to :py:func:`pb_recommendation.predict_by_factorization_machines`. """ - try: - import lightfm # noqa: F401 (optional dependency, used once implemented) - except ImportError: - raise ImportError( - "You need to install lightfm to use the factorization-machines " - "predictor (pip install pabutools[recommendation])." - ) return None # Empty implementation From 977b7a1a012cea5e24f1bff39d91787d09eb558a Mon Sep 17 00:00:00 2001 From: Roei Date: Mon, 6 Jul 2026 21:00:00 +0300 Subject: [PATCH 06/16] Implement all PB recommendation algorithms (full-implementation stage) - Partial-ballot layer: build/decode the +1/-1/0 CardinalBallot convention - Definitions 2.1/2.2 (approval scores via pabutools, consensus = |2*score-n|) - Greedy approval by raw score with lexicographic tie-breaking (Section 2.2.5) - Sampling modules: random, offline popularity/consensus/controversiality (top/bottom-k of the sigma/gamma orderings), online adaptive-controversial - Prediction: LV-majority, XGBoost per-project classification, and matrix-factorization / factorization-machines via a shared NumPy SVD core (scikit-surprise and lightfm wheels do not build on this Windows + NumPy 2 environment; the substitution is documented in the docstrings) - Full pipeline (sampling -> prediction -> greedy) and the FA score (Def 5.1) - INFO/DEBUG logging narrating every algorithm; helper functions with doctests All 39 pytest tests and 102 doctests pass. Validated end-to-end on the paper's Bielany 2020 Pabulib dataset, reproducing the paper's qualitative findings. --- pb_model_training.py | 232 +++++++++++++++++++++++++++--------- pb_recommendation.py | 271 ++++++++++++++++++++++++++++++++++++++----- 2 files changed, 424 insertions(+), 79 deletions(-) diff --git a/pb_model_training.py b/pb_model_training.py index 7df8fab..4fd8892 100644 --- a/pb_model_training.py +++ b/pb_model_training.py @@ -7,30 +7,34 @@ (``pb_recommendation``): a ``train_*`` function fits an estimator and returns an opaque model object, while the matching ``predict_by_*`` function in ``pb_recommendation`` consumes it to complete a single Target Voter's partial -ballot. Keeping them apart also lets the (heavy) ML dependencies be imported only -on the training side. +ballot. What each model trains on follows the paper (Section 3.1: predictions use the preferences of LV *and* the preferences in E_TV): * :py:func:`train_classification` is supervised, so it trains on the **Learning - Voters' full ballots only** (features = votes on the exposed projects). The - Target Voter's exposed votes enter later, as features, at prediction time - so - one trained classifier can be reused across many Target Voters. + Voters' full ballots** (features = votes on the exposed projects, one binary + classifier per project). The Target Voter's exposed votes enter later, as + features, at prediction time. * :py:func:`train_matrix_factorization` and :py:func:`train_factorization_machines` are collaborative filtering, so the Target Voter must be **inside** the fitted user-item matrix: they train on the Learning Voters' full ballots **and** that Target Voter's exposed set E_v - (passed in as the ``approved`` / ``disapproved`` project sets), and are - therefore (re)fitted per Target Voter. + (passed in as the ``approved`` / ``disapproved`` project sets). .. note:: - Per the pabutools maintainer (Simon Rey, issue thread): completing a partial - vote should live in a **separate module** and the ML libraries must **not be - a hard requirement** - exactly how pabutools treats Jinja for rule - explanations. So every ML dependency here is imported lazily inside the - ``train_*`` function that needs it, raising a friendly ``ImportError`` if it - is missing, and is declared only as an optional extra in ``pyproject.toml``. + The paper backs these with ``xgboost`` (classification), ``scikit-surprise`` + (matrix factorization) and ``lightfm`` (factorization machines). Following + the maintainer's advice, the ML libraries are **not a hard requirement**: + ``xgboost`` is imported lazily inside :py:func:`train_classification`. The + ``scikit-surprise`` / ``lightfm`` wheels could not be built in this + environment (Windows + NumPy 2), so the matrix-factorization and + factorization-machines models are computed with a small, self-contained + NumPy factorization instead - ``numpy`` is already a pabutools dependency. + +This module is deliberately *ballot-agnostic*: it takes plain project sets, not +the partial CardinalBallot. The caller (``pb_recommendation``) decodes the ballot +into approved / disapproved / exposed sets and passes those in. Programmer: Roei Yanku Date: 2026-06-20. @@ -38,17 +42,18 @@ from __future__ import annotations +import logging + +import numpy as np + from pabutools.election import ( Instance, Project, ApprovalProfile, + ApprovalBallot, ) -# This module is deliberately *ballot-agnostic*: it takes plain project sets, not -# the partial CardinalBallot. The caller (``pb_recommendation``) decodes the -# ballot into approved / disapproved / exposed sets and passes those in. That -# keeps the +1/-1/0 convention in one place (``pb_recommendation``) and the -# import one-directional (``pb_recommendation`` -> here), with no cycle. +logger = logging.getLogger(__name__) # --------------------------------------------------------------------------- @@ -58,14 +63,16 @@ def train_classification( instance: Instance, lv_profile: ApprovalProfile, exposed: set[Project], -) -> object: +) -> dict: """ Train the per-project binary classifiers of Section 2.1.1 on the LV ballots. For every project the voter might be asked to predict, a classifier is fitted whose features are the votes on the ``exposed`` projects and whose label is the vote on the target project. Backed by the external ``xgboost`` library - (:py:class:`xgboost.XGBClassifier`, class-weighted loss for the imbalanced - data), imported inside the implementation. + (:py:class:`xgboost.XGBClassifier`, class-weighted for the imbalanced data), + imported lazily. When there are no exposed features, or all LV voters agree on + the target (a single class), no classifier can be trained, so we fall back to + the constant majority label. Parameters ---------- @@ -78,36 +85,130 @@ def train_classification( Returns ------- - object - A fitted model (e.g. a mapping from each hidden project to its - trained classifier) to be passed to + dict + ``{"features": [...], "per_project": {project: ("const", 0/1) or + ("model", classifier)}}``, consumed by :py:func:`pb_recommendation.predict_by_classification`. + + Examples + -------- + With nothing exposed and unanimous LV approvals, every project falls back to + the constant majority label (1 for the approved projects, 0 otherwise). + + >>> p1, p2 = Project("p1", 1), Project("p2", 1) + >>> inst = Instance([p1, p2], budget_limit=2) + >>> lv = ApprovalProfile([ApprovalBallot([p1])] * 3) + >>> model = train_classification(inst, lv, exposed=set()) + >>> model["per_project"][p1], model["per_project"][p2] + (('const', 1), ('const', 0)) """ - return None # Empty implementation + features = sorted(exposed, key=str) + lv = list(lv_profile) + per_project: dict[Project, tuple] = {} + for target in instance: + labels = [1 if target in ballot else 0 for ballot in lv] + if not features or not lv or len(set(labels)) < 2: + # No features or a single class -> the majority label is the best we + # can do; a classifier cannot be trained. + approve = 1 if (labels and 2 * sum(labels) >= len(labels)) else 0 + per_project[target] = ("const", approve) + continue + try: + import xgboost + except ImportError: + raise ImportError( + "You need to install xgboost to train the classification " + "predictor (pip install pabutools[recommendation])." + ) + X = np.array([[1 if f in ballot else 0 for f in features] for ballot in lv]) + y = np.array(labels) + positives = int(y.sum()) + negatives = len(y) - positives + scale_pos_weight = (negatives / positives) if positives else 1.0 + classifier = xgboost.XGBClassifier( + n_estimators=50, max_depth=3, verbosity=0, + scale_pos_weight=scale_pos_weight, + ) + classifier.fit(X, y) + per_project[target] = ("model", classifier) + logger.info( + "train_classification: %d features, %d per-project classifiers", + len(features), len(per_project), + ) + return {"features": features, "per_project": per_project} # --------------------------------------------------------------------------- # Section 2.1.2 - Collaborative filtering via Matrix Factorization. # --------------------------------------------------------------------------- +def _factorized_tv_scores( + instance: Instance, + lv_profile: ApprovalProfile, + approved: set[Project], + disapproved: set[Project], + rank: int = 20, +) -> dict[Project, float]: + """ + Shared collaborative-filtering core (Section 2.1.2). Builds the user-item + rating matrix - one row per LV voter (approve=1, else 0) plus one row for the + Target Voter (``approved`` -> 1, ``disapproved`` -> 0, unknown -> the item's + LV mean) - centres it by the item means, and reconstructs it from a truncated + SVD of rank ``rank``. Returns the reconstructed score of every project in the + Target Voter's row, i.e. the predicted approval level in [0, 1] (roughly). + + Examples + -------- + >>> p1, p2, p3 = Project("p1", 1), Project("p2", 1), Project("p3", 1) + >>> inst = Instance([p1, p2, p3], budget_limit=3) + >>> lv = ApprovalProfile([ApprovalBallot([p1, p2])] * 3) + >>> scores = _factorized_tv_scores(inst, lv, set(), set()) + >>> scores[p1] >= 0.5 and scores[p2] >= 0.5 and scores[p3] < 0.5 + True + """ + items = sorted(instance, key=str) + lv = list(lv_profile) + if lv: + lv_matrix = np.array( + [[1.0 if item in ballot else 0.0 for item in items] for ballot in lv] + ) + item_means = lv_matrix.mean(axis=0) + else: + lv_matrix = np.zeros((0, len(items))) + item_means = np.zeros(len(items)) + # The Target Voter's row: known exposed votes, unknown cells seeded with the + # item mean so the matrix is complete before factorising. + tv_row = np.array([ + 1.0 if item in approved else (0.0 if item in disapproved else item_means[j]) + for j, item in enumerate(items) + ]) + matrix = np.vstack([lv_matrix, tv_row]) + centered = matrix - item_means + u, singular, vt = np.linalg.svd(centered, full_matrices=False) + kept = min(rank, len(singular)) + reconstruction = (u[:, :kept] * singular[:kept]) @ vt[:kept] + item_means + tv_scores = reconstruction[-1] + return {item: float(tv_scores[j]) for j, item in enumerate(items)} + + def train_matrix_factorization( instance: Instance, lv_profile: ApprovalProfile, approved: set[Project], disapproved: set[Project], -) -> object: +) -> dict[Project, float]: """ - Train the Matrix Factorization model of Section 2.1.2. Collaborative - filtering needs the Target Voter inside the matrix, so the model is fitted on - **both** the Learning Voters' full ballots **and** the Target Voter's exposed - set E_v (paper Section 3.1: predictions use the preferences of LV *and* the - preferences in E_TV): build the sparse user-item rating matrix with one row - per LV voter (full) plus one row for this Target Voter holding only her - exposed votes (``approved`` -> 1, ``disapproved`` -> 0; the hidden projects - are left empty), then factorise it. The reconstructed cells of the Target - Voter's row over the hidden projects are what gets read off at prediction - time. Without her exposed votes the model could only reproduce the LV - average. Backed by the external ``scikit-surprise`` library - (:py:class:`surprise.SVD`), imported inside the implementation. + Train the Matrix Factorization model of Section 2.1.2. Collaborative filtering + needs the Target Voter inside the matrix, so the model is fitted on **both** + the Learning Voters' full ballots **and** the Target Voter's exposed set E_v + (``approved`` -> 1, ``disapproved`` -> 0). The user-item matrix is factorised + (truncated SVD) and the Target Voter's row is reconstructed; the score of each + hidden project is what gets thresholded at 0.5 at prediction time. Without her + exposed votes the model could only reproduce the LV average. + + .. note:: + The paper uses ``scikit-surprise`` (:py:class:`surprise.SVD`). That wheel + could not be built here (Windows + NumPy 2), so the factorization is done + with NumPy via :py:func:`_factorized_tv_scores`. Parameters ---------- @@ -122,11 +223,22 @@ def train_matrix_factorization( Returns ------- - object - A fitted factorisation model to be passed to + dict[:py:class:`~pabutools.election.instance.Project`, float] + The reconstructed approval score of every project for this Target + Voter, consumed by :py:func:`pb_recommendation.predict_by_matrix_factorization`. + + Examples + -------- + >>> p1, p2, p3 = Project("p1", 1), Project("p2", 1), Project("p3", 1) + >>> inst = Instance([p1, p2, p3], budget_limit=3) + >>> lv = ApprovalProfile([ApprovalBallot([p1, p2])] * 3) + >>> scores = train_matrix_factorization(inst, lv, set(), set()) + >>> scores[p1] >= 0.5 and scores[p3] < 0.5 + True """ - return None # Empty implementation + logger.info("train_matrix_factorization: factorising the user-item matrix") + return _factorized_tv_scores(instance, lv_profile, approved, disapproved) # --------------------------------------------------------------------------- @@ -137,16 +249,21 @@ def train_factorization_machines( lv_profile: ApprovalProfile, approved: set[Project], disapproved: set[Project], -) -> object: +) -> dict[Project, float]: """ Train the Factorization Machines model of Section 2.1.2: like Matrix - Factorization but with a linear term plus pairwise latent interactions and - optional side features. Being collaborative filtering, it is fitted on - **both** the Learning Voters' full ballots **and** the Target Voter's exposed - set E_v (``approved`` -> 1, ``disapproved`` -> 0), exactly as in - :py:func:`train_matrix_factorization`; the hidden projects are what the fitted - model predicts. Backed by an external FM library (e.g. ``lightfm`` / - ``fastFM``), imported inside the implementation. + Factorization but with a linear (bias) term plus pairwise latent interactions. + It is fitted on **both** the Learning Voters' full ballots **and** the Target + Voter's exposed set E_v (``approved`` -> 1, ``disapproved`` -> 0). Centring the + matrix by the item means captures the linear per-item bias, and the truncated + SVD captures the pairwise latent interactions, so the same + :py:func:`_factorized_tv_scores` core is reused; the hidden projects are what + the fitted model predicts. + + .. note:: + The paper uses an external FM library (``lightfm`` / ``fastFM``). Those + wheels could not be built here (Windows + NumPy 2), so the model is + computed with the same NumPy factorization core. Parameters ---------- @@ -161,11 +278,22 @@ def train_factorization_machines( Returns ------- - object - A fitted FM model to be passed to + dict[:py:class:`~pabutools.election.instance.Project`, float] + The predicted approval score of every project for this Target Voter, + consumed by :py:func:`pb_recommendation.predict_by_factorization_machines`. + + Examples + -------- + >>> p1, p2, p3 = Project("p1", 1), Project("p2", 1), Project("p3", 1) + >>> inst = Instance([p1, p2, p3], budget_limit=3) + >>> lv = ApprovalProfile([ApprovalBallot([p1, p2])] * 3) + >>> scores = train_factorization_machines(inst, lv, set(), set()) + >>> scores[p1] >= 0.5 and scores[p3] < 0.5 + True """ - return None # Empty implementation + logger.info("train_factorization_machines: fitting the FM model") + return _factorized_tv_scores(instance, lv_profile, approved, disapproved) if __name__ == "__main__": diff --git a/pb_recommendation.py b/pb_recommendation.py index 4a4b275..4674f01 100644 --- a/pb_recommendation.py +++ b/pb_recommendation.py @@ -9,6 +9,8 @@ from __future__ import annotations +import logging +import random from collections.abc import Iterable from pabutools.election import ( @@ -17,13 +19,25 @@ ApprovalProfile, ApprovalBallot, CardinalBallot, + total_cost, ) from pabutools.rules import BudgetAllocation # Model training lives in a separate module, ``pb_model_training`` (the # professor's note b: separate files for training the model and using it). The -# learning-based predictors below will call its ``train_*`` functions once -# implemented; at this stage their bodies are empty, so nothing is imported yet. +# learning-based predictors below delegate the fitting to its ``train_*`` +# functions; ``pb_model_training`` is ballot-agnostic (it takes plain project +# sets), so the dependency is one-directional and there is no import cycle. +from pb_model_training import ( + train_classification, + train_matrix_factorization, + train_factorization_machines, +) + +# Log the steps of every algorithm at INFO/DEBUG level (see the assignment's +# logging requirement). Configure a handler in your own script to see them, e.g. +# ``logging.basicConfig(level=logging.INFO)``. +logger = logging.getLogger(__name__) # =========================================================================== @@ -45,10 +59,12 @@ # ``partial_ballot`` / ``reveal_ballot`` and read them back with the # ``*_projects`` accessors / ``as_approval_ballot``. -# Scores stored (to be defined when the module is implemented): -# approved project (A_v) -> +1 (any strictly positive score) -# disapproved project (D_v) -> -1 (any strictly negative score) -# hidden project (H_v) -> 0 (absent projects mean the same) +#: Score stored for an approved project (A_v). Any strictly positive score works. +APPROVAL = 1 +#: Score stored for a disapproved project (D_v). Any strictly negative score works. +DISAPPROVAL = -1 +#: Score stored for a hidden project (H_v); absent projects mean the same. +HIDDEN = 0 def partial_ballot( @@ -71,7 +87,14 @@ def partial_ballot( >>> exposed_projects(b) == {p1, p2} True """ - return CardinalBallot() # Empty implementation + scores: dict[Project, int] = {} + for project in approved: + scores[project] = APPROVAL + for project in disapproved: + scores[project] = DISAPPROVAL + for project in hidden: + scores[project] = HIDDEN + return CardinalBallot(scores) def reveal_ballot( @@ -113,22 +136,29 @@ def reveal_ballot( >>> hidden_projects(b, inst) == {p2, p4} True """ - return CardinalBallot() # Empty implementation + approved = exposed & full_ballot # A_v = E_v ∩ full ballot + disapproved = exposed - full_ballot # D_v = E_v \ full ballot + hidden = set(instance) - exposed # H_v = P \ E_v + logger.debug( + "reveal_ballot: |E_v|=%d -> |A_v|=%d, |D_v|=%d, |H_v|=%d", + len(exposed), len(approved), len(disapproved), len(hidden), + ) + return partial_ballot(approved=approved, disapproved=disapproved, hidden=hidden) def approved_projects(ballot: CardinalBallot) -> set[Project]: """The approval set A_v: the projects with a strictly positive score.""" - return set() # Empty implementation + return {project for project, score in ballot.items() if score > 0} def disapproved_projects(ballot: CardinalBallot) -> set[Project]: """The disapproval set D_v: the projects with a strictly negative score.""" - return set() # Empty implementation + return {project for project, score in ballot.items() if score < 0} def exposed_projects(ballot: CardinalBallot) -> set[Project]: """The exposed set E_v = A_v ∪ D_v: the projects with a non-zero score.""" - return set() # Empty implementation + return {project for project, score in ballot.items() if score != 0} def hidden_projects(ballot: CardinalBallot, instance: Instance) -> set[Project]: @@ -136,7 +166,7 @@ def hidden_projects(ballot: CardinalBallot, instance: Instance) -> set[Project]: The hidden set H_v = P \\ E_v: every project of the instance that the voter was not asked about (score 0 or absent from the ballot). """ - return set() # Empty implementation + return set(instance) - exposed_projects(ballot) def as_approval_ballot(ballot: CardinalBallot) -> ApprovalBallot: @@ -144,7 +174,7 @@ def as_approval_ballot(ballot: CardinalBallot) -> ApprovalBallot: The pabutools approval ballot made of the approved projects A_v, so that a (completed) partial ballot can be fed to approval-based voting rules. """ - return ApprovalBallot() # Empty implementation + return ApprovalBallot(approved_projects(ballot)) # --------------------------------------------------------------------------- @@ -189,7 +219,12 @@ def approval_scores( >>> [s[p] for p in (p1, p2, p3)] [2, 2, 1] """ - return {} # Empty implementation + # Reuse the library's counting (Definition 2.1), then fill 0 for the + # projects that nobody approved so the whole universe P is represented. + library_scores = profile.approval_scores() + scores = {project: library_scores.get(project, 0) for project in instance} + logger.debug("approval_scores: %s", scores) + return scores def consensus_levels( @@ -225,7 +260,57 @@ def consensus_levels( >>> [c[p] for p in (p1, p2, p3)] [4, 0, 4] """ - return {} # Empty implementation + # consensus(p) = |approvers(p) - disapprovers(p)|. With n voters and + # score(p) approvers, disapprovers = n - score(p), so the difference is + # |score - (n - score)| = |2*score - n|. + n = profile.num_ballots() + scores = approval_scores(instance, profile) + consensus = {project: abs(2 * scores[project] - n) for project in instance} + logger.debug("consensus_levels (n=%d): %s", n, consensus) + return consensus + + +def most_popular_projects( + instance: Instance, profile: ApprovalProfile +) -> list[Project]: + """ + The paper's ordering sigma (Section 2.2.3): the projects sorted by decreasing + approval score, ties broken lexicographically by project name. sigma_1 is the + most popular project, sigma_m the least. Helper used by the greedy rule and by + offline revealing-by-popularity. + + Examples + -------- + >>> p1, p2, p3 = Project("p1", 1), Project("p2", 1), Project("p3", 1) + >>> inst = Instance([p1, p2, p3], budget_limit=3) + >>> prof = ApprovalProfile([ApprovalBallot([p1, p3]), ApprovalBallot([p1])]) + >>> most_popular_projects(inst, prof) # scores p1=2, p3=1, p2=0 + [p1, p3, p2] + """ + scores = approval_scores(instance, profile) + return sorted(instance, key=lambda project: (-scores[project], str(project))) + + +def most_consensual_projects( + instance: Instance, profile: ApprovalProfile +) -> list[Project]: + """ + The paper's ordering gamma (Section 2.2.3): the projects sorted by decreasing + consensus level, ties broken lexicographically by project name. gamma_1 is the + project most in consensus, gamma_m the "most controversial". Helper used by + offline revealing-by-consensus and by-controversiality. + + Examples + -------- + >>> p1, p2, p3 = Project("p1", 1), Project("p2", 1), Project("p3", 1) + >>> inst = Instance([p1, p2, p3], budget_limit=3) + >>> prof = ApprovalProfile([ApprovalBallot([p1, p2]), ApprovalBallot([p1]), + ... ApprovalBallot([p1, p2]), ApprovalBallot([p1])]) + >>> most_consensual_projects(inst, prof) # consensus p1=4, p3=4, p2=0 + [p1, p3, p2] + """ + consensus = consensus_levels(instance, profile) + return sorted(instance, key=lambda project: (-consensus[project], str(project))) # --------------------------------------------------------------------------- @@ -271,7 +356,22 @@ def greedy_approval( >>> sorted(greedy_approval(inst, prof), key=str) [p1, p2] """ - return BudgetAllocation() # Empty implementation + # Consider the projects by decreasing approval score (sigma) and fund each + # one whose cost still fits the remaining budget (Section 2.2.5). + chosen = BudgetAllocation() + spent = 0 + for project in most_popular_projects(instance, profile): + if spent + project.cost <= instance.budget_limit: + chosen.append(project) + spent += project.cost + logger.debug("greedy_approval: fund %s (spent %s)", project, spent) + else: + logger.debug("greedy_approval: skip %s (would exceed budget)", project) + logger.info( + "greedy_approval: funded %d/%d projects, cost %s of %s", + len(chosen), len(instance), spent, instance.budget_limit, + ) + return chosen # --------------------------------------------------------------------------- @@ -314,7 +414,13 @@ def random_setup( >>> len(exposed) == 2 and exposed <= {p1, p2, p3, p4} True """ - return set() # Empty implementation + rng = random.Random(seed) + # Sort first so the sample is reproducible from the seed (a set has no + # deterministic iteration order). + population = sorted(instance, key=str) + exposed = set(rng.sample(population, k)) + logger.info("random_setup: exposed %d of %d projects at random", k, len(population)) + return exposed # --------------------------------------------------------------------------- @@ -358,7 +464,9 @@ def offline_popularity( >>> offline_popularity(inst, lv, k=1) == {p1} True """ - return set() # Empty implementation + exposed = set(most_popular_projects(instance, lv_profile)[:k]) + logger.info("offline_popularity: exposing top-%d popular projects", k) + return exposed def offline_consensus( @@ -396,7 +504,9 @@ def offline_consensus( >>> offline_consensus(inst, lv, k=1) == {p1} True """ - return set() # Empty implementation + exposed = set(most_consensual_projects(instance, lv_profile)[:k]) + logger.info("offline_consensus: exposing top-%d consensual projects", k) + return exposed def offline_controversiality( @@ -435,7 +545,12 @@ def offline_controversiality( >>> offline_controversiality(inst, lv, k=1) == {p2} True """ - return set() # Empty implementation + # The k *least* consensual projects are the last k of gamma, + # {gamma_{m-k+1}, ..., gamma_m}. Slicing from m-k keeps k=0 correct. + gamma = most_consensual_projects(instance, lv_profile) + exposed = set(gamma[len(gamma) - k:]) + logger.info("offline_controversiality: exposing bottom-%d consensual projects", k) + return exposed # --------------------------------------------------------------------------- @@ -486,7 +601,25 @@ def online_adaptive_controversial( >>> online_adaptive_controversial(inst, lv, {p1, p2}, k=2) == {p1, p2} True """ - return set() # Empty implementation + # The consensus of a project is a property of the electorate that has voted + # on it. A Target Voter has only answered the projects already asked, so her + # revealed answers never affect the consensus of the *remaining* projects - + # which is why, for a single voter, recomputing each round is equivalent to + # ranking once by the LV consensus. We still run the k rounds explicitly and + # simulate each answer, matching Algorithm 5's structure. + consensus = consensus_levels(instance, lv_profile) + exposed: set[Project] = set() + for round_index in range(1, k + 1): + remaining = [project for project in instance if project not in exposed] + # Most controversial = lowest consensus, ties broken by project name. + project = min(remaining, key=lambda p: (consensus[p], str(p))) + answered_yes = project in full_ballot # simulate the voter's answer + logger.info( + "online_adaptive_controversial: round %d asks %s -> voter %s", + round_index, project, "approves" if answered_yes else "disapproves", + ) + exposed.add(project) + return exposed # --------------------------------------------------------------------------- @@ -531,7 +664,19 @@ def predict_by_majority( >>> predict_by_majority(inst, lv, partial) == {p1, p2} True """ - return ApprovalBallot() # Empty implementation + n = lv_profile.num_ballots() + scores = approval_scores(instance, lv_profile) + result = set(approved_projects(ballot)) # keep the exposed approvals A_v + for project in hidden_projects(ballot, instance): + # LV approval rate >= 50% <=> score >= n/2 <=> 2*score >= n. + if 2 * scores[project] >= n: + result.add(project) + logger.debug( + "predict_by_majority: predict %s approved (LV %d/%d)", + project, scores[project], n, + ) + logger.info("predict_by_majority: completed ballot to %d approvals", len(result)) + return ApprovalBallot(result) def predict_by_classification( @@ -576,7 +721,45 @@ def predict_by_classification( >>> predict_by_classification(inst, lv, partial) == {p1, p2} True """ - return ApprovalBallot() # Empty implementation + model = train_classification(instance, lv_profile, exposed_projects(ballot)) + features = model["features"] + # The TV voter's feature vector: her vote on each exposed project. + approved = approved_projects(ballot) + tv_features = [1 if feature in approved else 0 for feature in features] + result = set(approved) # keep the exposed approvals A_v + for project in hidden_projects(ballot, instance): + kind, payload = model["per_project"][project] + if kind == "const": + approve = payload + else: + import numpy as np + approve = int(payload.predict(np.array([tv_features]))[0]) + if approve == 1: + result.add(project) + logger.info("predict_by_classification: completed ballot to %d approvals", len(result)) + return ApprovalBallot(result) + + +def _approve_hidden_by_score( + instance: Instance, ballot: CardinalBallot, scores: dict[Project, float] +) -> set[Project]: + """ + Complete a partial ballot from per-project scores: keep the exposed approvals + A_v and add each hidden project whose predicted score is >= 0.5 (the exposed + disapprovals D_v stay rejected). Shared by the matrix-factorization and + factorization-machines predictors. + + >>> p1, p2, p3 = Project("p1", 1), Project("p2", 1), Project("p3", 1) + >>> inst = Instance([p1, p2, p3], budget_limit=3) + >>> b = partial_ballot(approved={p1}, hidden={p2, p3}) + >>> _approve_hidden_by_score(inst, b, {p1: 1.0, p2: 0.8, p3: 0.2}) == {p1, p2} + True + """ + result = set(approved_projects(ballot)) + for project in hidden_projects(ballot, instance): + if scores[project] >= 0.5: + result.add(project) + return result def predict_by_matrix_factorization( @@ -622,7 +805,12 @@ def predict_by_matrix_factorization( >>> predict_by_matrix_factorization(inst, lv, partial) == {p1, p2} True """ - return ApprovalBallot() # Empty implementation + scores = train_matrix_factorization( + instance, lv_profile, approved_projects(ballot), disapproved_projects(ballot) + ) + result = _approve_hidden_by_score(instance, ballot, scores) + logger.info("predict_by_matrix_factorization: completed to %d approvals", len(result)) + return ApprovalBallot(result) def predict_by_factorization_machines( @@ -667,7 +855,12 @@ def predict_by_factorization_machines( >>> predict_by_factorization_machines(inst, lv, partial) == {p1, p2} True """ - return ApprovalBallot() # Empty implementation + scores = train_factorization_machines( + instance, lv_profile, approved_projects(ballot), disapproved_projects(ballot) + ) + result = _approve_hidden_by_score(instance, ballot, scores) + logger.info("predict_by_factorization_machines: completed to %d approvals", len(result)) + return ApprovalBallot(result) # --------------------------------------------------------------------------- @@ -714,7 +907,24 @@ def recommend( >>> sorted(recommend(inst, lv, {"v4": {p1, p2}}, k=1), key=str) [p1, p2] """ - return BudgetAllocation() # Empty implementation + # Sampling: the offline-popularity sampler exposes the same k projects to + # every Target Voter. + exposed = offline_popularity(instance, lv_profile, k) + # Prediction: complete each TV ballot; start the combined profile from the LV + # ballots (which are already full). + completed_ballots = list(lv_profile) + for voter_id, full_ballot in tv_ballots.items(): + partial = reveal_ballot(instance, full_ballot, exposed) + completed = predict_by_majority(instance, lv_profile, partial) + logger.debug("recommend: TV %s completed to %d approvals", voter_id, len(completed)) + completed_ballots.append(completed) + # Voting rule: greedy approval on the LV + completed TV ballots. + combined = ApprovalProfile(completed_ballots) + logger.info( + "recommend: predicting bundle from %d LV + %d TV ballots", + lv_profile.num_ballots(), len(tv_ballots), + ) + return greedy_approval(instance, combined) # --------------------------------------------------------------------------- @@ -753,7 +963,14 @@ def fractional_allocation_score( >>> fractional_allocation_score({p3}, {p1}, budget_limit=6) 0.0 """ - return 0.0 # Empty implementation + # lambda = total cost of the correctly-predicted projects (pb ∩ rb). + correctly_predicted = real_bundle & predicted_bundle + score = total_cost(correctly_predicted) / budget_limit + logger.info( + "fractional_allocation_score: %d/%d projects correct, FA=%.3f", + len(correctly_predicted), len(real_bundle | predicted_bundle), score, + ) + return float(score) if __name__ == "__main__": From 002f45b109f99e97a7796be013d3b0c6e787b8af Mon Sep 17 00:00:00 2001 From: Roei Date: Tue, 7 Jul 2026 17:59:18 +0300 Subject: [PATCH 07/16] Reuse pabutools instead of own functions where the library suffices - greedy_approval now delegates to greedy_utilitarian_welfare with Cost_Sat: marginal satisfaction under Cost_Sat is score(p)*cost(p), so dividing by the cost ranks by the raw approval score - exactly the paper's greedy approval (verified equivalent, incl. tie-breaking, on 30 random instances) - Delete approval_scores: Definition 2.1 is exactly the library's profile.approval_scores(); callers use it directly - Delete most_popular_projects: the sigma ordering is inlined into its only remaining caller, offline_popularity - Drop the TestApprovalScores class (the library tests its own method) 35 tests + doctests pass. --- pb_recommendation.py | 123 ++++++++++---------------------------- test_pb_recommendation.py | 35 +---------- 2 files changed, 33 insertions(+), 125 deletions(-) diff --git a/pb_recommendation.py b/pb_recommendation.py index 4674f01..e1878b2 100644 --- a/pb_recommendation.py +++ b/pb_recommendation.py @@ -21,7 +21,8 @@ CardinalBallot, total_cost, ) -from pabutools.rules import BudgetAllocation +from pabutools.election.satisfaction import Cost_Sat +from pabutools.rules import BudgetAllocation, greedy_utilitarian_welfare # Model training lives in a separate module, ``pb_model_training`` (the # professor's note b: separate files for training the model and using it). The @@ -180,53 +181,9 @@ def as_approval_ballot(ballot: CardinalBallot) -> ApprovalBallot: # --------------------------------------------------------------------------- # Section 2.2.3 - Popularity and consensus (primitives used by every module). # --------------------------------------------------------------------------- -def approval_scores( - instance: Instance, profile: ApprovalProfile -) -> dict[Project, int]: - """ - Definition 2.1 (Approval scores): the approval score of a project is the - number of voters that approve it. - - This is exactly pabutools' - :py:meth:`~pabutools.election.profile.approvalprofile.AbstractApprovalProfile.approval_scores`, - so we delegate to the library instead of re-counting the ballots ourselves. - The only addition is that projects approved by nobody (absent from the - library's dictionary) are filled in with a score of 0, so every project of - the instance is present in the result. - - Parameters - ---------- - instance : :py:class:`~pabutools.election.instance.Instance` - The PB instance (the set of projects and the budget limit). - profile : :py:class:`~pabutools.election.profile.approvalprofile.ApprovalProfile` - The approval profile, i.e. the ballots of the voters. - - Returns - ------- - dict[:py:class:`~pabutools.election.instance.Project`, int] - A mapping from each project to its approval score. - - Examples - -------- - Example 1 from the paper (P = {p1, p2, p3}, three full ballots): - - >>> p1, p2, p3 = Project("p1", 1), Project("p2", 1), Project("p3", 2) - >>> inst = Instance([p1, p2, p3], budget_limit=3) - >>> prof = ApprovalProfile([ApprovalBallot([p1, p2]), - ... ApprovalBallot([p1, p3]), - ... ApprovalBallot([p2])]) - >>> s = approval_scores(inst, prof) - >>> [s[p] for p in (p1, p2, p3)] - [2, 2, 1] - """ - # Reuse the library's counting (Definition 2.1), then fill 0 for the - # projects that nobody approved so the whole universe P is represented. - library_scores = profile.approval_scores() - scores = {project: library_scores.get(project, 0) for project in instance} - logger.debug("approval_scores: %s", scores) - return scores - - +# Definition 2.1 (Approval scores) needs no function of our own: it is exactly +# pabutools' ``profile.approval_scores()``. Callers below use the library method +# directly (``.get(p, 0)`` supplies the 0 for projects that nobody approved). def consensus_levels( instance: Instance, profile: ApprovalProfile ) -> dict[Project, int]: @@ -262,35 +219,17 @@ def consensus_levels( """ # consensus(p) = |approvers(p) - disapprovers(p)|. With n voters and # score(p) approvers, disapprovers = n - score(p), so the difference is - # |score - (n - score)| = |2*score - n|. + # |score - (n - score)| = |2*score - n|. Scores come from the library + # (Definition 2.1 = pabutools' approval_scores()). n = profile.num_ballots() - scores = approval_scores(instance, profile) - consensus = {project: abs(2 * scores[project] - n) for project in instance} + scores = profile.approval_scores() + consensus = { + project: abs(2 * scores.get(project, 0) - n) for project in instance + } logger.debug("consensus_levels (n=%d): %s", n, consensus) return consensus -def most_popular_projects( - instance: Instance, profile: ApprovalProfile -) -> list[Project]: - """ - The paper's ordering sigma (Section 2.2.3): the projects sorted by decreasing - approval score, ties broken lexicographically by project name. sigma_1 is the - most popular project, sigma_m the least. Helper used by the greedy rule and by - offline revealing-by-popularity. - - Examples - -------- - >>> p1, p2, p3 = Project("p1", 1), Project("p2", 1), Project("p3", 1) - >>> inst = Instance([p1, p2, p3], budget_limit=3) - >>> prof = ApprovalProfile([ApprovalBallot([p1, p3]), ApprovalBallot([p1])]) - >>> most_popular_projects(inst, prof) # scores p1=2, p3=1, p2=0 - [p1, p3, p2] - """ - scores = approval_scores(instance, profile) - return sorted(instance, key=lambda project: (-scores[project], str(project))) - - def most_consensual_projects( instance: Instance, profile: ApprovalProfile ) -> list[Project]: @@ -326,10 +265,14 @@ def greedy_approval( are broken lexicographically by project name. .. note:: - Implemented from scratch; it does *not* reuse pabutools' - :py:func:`~pabutools.rules.greedy_utilitarian_welfare`, which ranks by - satisfaction **divided by cost** (density). The paper ranks by the - **raw** approval score, so the two give different bundles. + Delegates to pabutools' + :py:func:`~pabutools.rules.greedy_utilitarian_welfare` with + :py:class:`~pabutools.election.satisfaction.additivesatisfaction.Cost_Sat`: + the rule ranks by marginal satisfaction divided by cost, and under + Cost_Sat a project's marginal satisfaction is score(p)*cost(p), so the + ranking is by the **raw** approval score - exactly the paper's greedy + approval. (With the default Cardinality_Sat the ranking would be + score/cost, which is a different rule.) Parameters ---------- @@ -356,20 +299,13 @@ def greedy_approval( >>> sorted(greedy_approval(inst, prof), key=str) [p1, p2] """ - # Consider the projects by decreasing approval score (sigma) and fund each - # one whose cost still fits the remaining budget (Section 2.2.5). - chosen = BudgetAllocation() - spent = 0 - for project in most_popular_projects(instance, profile): - if spent + project.cost <= instance.budget_limit: - chosen.append(project) - spent += project.cost - logger.debug("greedy_approval: fund %s (spent %s)", project, spent) - else: - logger.debug("greedy_approval: skip %s (would exceed budget)", project) + chosen = greedy_utilitarian_welfare( + instance, profile, sat_class=Cost_Sat, resoluteness=True + ) + assert isinstance(chosen, BudgetAllocation) # resolute mode: one allocation logger.info( "greedy_approval: funded %d/%d projects, cost %s of %s", - len(chosen), len(instance), spent, instance.budget_limit, + len(chosen), len(instance), total_cost(chosen), instance.budget_limit, ) return chosen @@ -464,7 +400,10 @@ def offline_popularity( >>> offline_popularity(inst, lv, k=1) == {p1} True """ - exposed = set(most_popular_projects(instance, lv_profile)[:k]) + # The paper's sigma ordering: decreasing library approval score, ties by name. + scores = lv_profile.approval_scores() + sigma = sorted(instance, key=lambda p: (-scores.get(p, 0), str(p))) + exposed = set(sigma[:k]) logger.info("offline_popularity: exposing top-%d popular projects", k) return exposed @@ -665,15 +604,15 @@ def predict_by_majority( True """ n = lv_profile.num_ballots() - scores = approval_scores(instance, lv_profile) + scores = lv_profile.approval_scores() # Definition 2.1, from the library result = set(approved_projects(ballot)) # keep the exposed approvals A_v for project in hidden_projects(ballot, instance): # LV approval rate >= 50% <=> score >= n/2 <=> 2*score >= n. - if 2 * scores[project] >= n: + if 2 * scores.get(project, 0) >= n: result.add(project) logger.debug( "predict_by_majority: predict %s approved (LV %d/%d)", - project, scores[project], n, + project, scores.get(project, 0), n, ) logger.info("predict_by_majority: completed ballot to %d approvals", len(result)) return ApprovalBallot(result) diff --git a/test_pb_recommendation.py b/test_pb_recommendation.py index c5a83f6..023d785 100644 --- a/test_pb_recommendation.py +++ b/test_pb_recommendation.py @@ -35,7 +35,6 @@ exposed_projects, hidden_projects, as_approval_ballot, - approval_scores, consensus_levels, greedy_approval, random_setup, @@ -135,40 +134,10 @@ def reference_greedy_approval(projects, profile, budget): return chosen -# --------------------------------------------------------------------------- -# approval_scores -# --------------------------------------------------------------------------- -class TestApprovalScores: - def test_example1(self, example1): - p, inst, prof = example1 - scores = approval_scores(inst, prof) - assert scores[p["p1"]] == 2 - assert scores[p["p2"]] == 2 - assert scores[p["p3"]] == 1 - - def test_empty_profile(self): - p = make_projects([("p1", 1), ("p2", 1)]) - inst = Instance(p.values(), budget_limit=2) - scores = approval_scores(inst, ApprovalProfile([])) - assert all(scores[proj] == 0 for proj in p.values()) - - def test_large_structured_all_approve_all(self): - # "Complete-approval" analogue: 200 projects, 500 voters, everyone - # approves everything -> every score is exactly 500. - projects = padded_projects(200, cost=1) - inst = Instance(projects, budget_limit=10) - prof = ApprovalProfile([ApprovalBallot(projects)] * 500) - scores = approval_scores(inst, prof) - assert all(scores[p] == 500 for p in projects) - - def test_matches_manual_count_random(self): - # Cross-check against an independent counting implementation. - projects, inst, prof = random_instance(40, 120, 100, seed=31) - assert approval_scores(inst, prof) == manual_scores(projects, prof) - - # --------------------------------------------------------------------------- # consensus_levels +# (Approval scores, Definition 2.1, have no function of our own - the code uses +# pabutools' profile.approval_scores() directly, which the library itself tests.) # --------------------------------------------------------------------------- class TestConsensusLevels: def test_example7(self, consensus_data): From 012851a79ce36da977d24a5925f48c2059b8bf25 Mon Sep 17 00:00:00 2001 From: Roei Date: Tue, 7 Jul 2026 18:06:45 +0300 Subject: [PATCH 08/16] Rewrite function bodies in set-builder style mirroring the paper's notation - partial_ballot: one dict-union expression instead of three loops - reveal_ballot: single partial_ballot call on E&full, E-full, P-E - predict_by_majority and _approve_hidden_by_score: the completion is now literally A_v | {p in H_v : condition(p)} as in the paper - predict_by_classification: per-project votes as a dict comprehension, numpy import hoisted to the module level - online_adaptive_controversial: min over set(instance)-exposed directly - recommend: the sampling->prediction step is one list comprehension Per-item DEBUG logs are folded into the per-function INFO summaries. 35 tests + doctests pass. --- pb_recommendation.py | 101 +++++++++++++++++++------------------------ 1 file changed, 44 insertions(+), 57 deletions(-) diff --git a/pb_recommendation.py b/pb_recommendation.py index e1878b2..9082f37 100644 --- a/pb_recommendation.py +++ b/pb_recommendation.py @@ -13,6 +13,8 @@ import random from collections.abc import Iterable +import numpy as np + from pabutools.election import ( Instance, Project, @@ -88,14 +90,11 @@ def partial_ballot( >>> exposed_projects(b) == {p1, p2} True """ - scores: dict[Project, int] = {} - for project in approved: - scores[project] = APPROVAL - for project in disapproved: - scores[project] = DISAPPROVAL - for project in hidden: - scores[project] = HIDDEN - return CardinalBallot(scores) + return CardinalBallot( + {p: APPROVAL for p in approved} + | {p: DISAPPROVAL for p in disapproved} + | {p: HIDDEN for p in hidden} + ) def reveal_ballot( @@ -137,14 +136,11 @@ def reveal_ballot( >>> hidden_projects(b, inst) == {p2, p4} True """ - approved = exposed & full_ballot # A_v = E_v ∩ full ballot - disapproved = exposed - full_ballot # D_v = E_v \ full ballot - hidden = set(instance) - exposed # H_v = P \ E_v - logger.debug( - "reveal_ballot: |E_v|=%d -> |A_v|=%d, |D_v|=%d, |H_v|=%d", - len(exposed), len(approved), len(disapproved), len(hidden), + return partial_ballot( + approved=exposed & full_ballot, # A_v = E_v ∩ full ballot + disapproved=exposed - full_ballot, # D_v = E_v \ full ballot + hidden=set(instance) - exposed, # H_v = P \ E_v ) - return partial_ballot(approved=approved, disapproved=disapproved, hidden=hidden) def approved_projects(ballot: CardinalBallot) -> set[Project]: @@ -549,13 +545,12 @@ def online_adaptive_controversial( consensus = consensus_levels(instance, lv_profile) exposed: set[Project] = set() for round_index in range(1, k + 1): - remaining = [project for project in instance if project not in exposed] - # Most controversial = lowest consensus, ties broken by project name. - project = min(remaining, key=lambda p: (consensus[p], str(p))) - answered_yes = project in full_ballot # simulate the voter's answer + # Most controversial not-yet-asked project: lowest consensus, ties by name. + project = min(set(instance) - exposed, key=lambda p: (consensus[p], str(p))) logger.info( "online_adaptive_controversial: round %d asks %s -> voter %s", - round_index, project, "approves" if answered_yes else "disapproves", + round_index, project, + "approves" if project in full_ballot else "disapproves", ) exposed.add(project) return exposed @@ -605,15 +600,10 @@ def predict_by_majority( """ n = lv_profile.num_ballots() scores = lv_profile.approval_scores() # Definition 2.1, from the library - result = set(approved_projects(ballot)) # keep the exposed approvals A_v - for project in hidden_projects(ballot, instance): - # LV approval rate >= 50% <=> score >= n/2 <=> 2*score >= n. - if 2 * scores.get(project, 0) >= n: - result.add(project) - logger.debug( - "predict_by_majority: predict %s approved (LV %d/%d)", - project, scores.get(project, 0), n, - ) + # predicted ballot = A_v u {p in H_v : LV approval rate of p >= 1/2} + result = approved_projects(ballot) | { + p for p in hidden_projects(ballot, instance) if 2 * scores.get(p, 0) >= n + } logger.info("predict_by_majority: completed ballot to %d approvals", len(result)) return ApprovalBallot(result) @@ -661,20 +651,17 @@ def predict_by_classification( True """ model = train_classification(instance, lv_profile, exposed_projects(ballot)) - features = model["features"] - # The TV voter's feature vector: her vote on each exposed project. approved = approved_projects(ballot) - tv_features = [1 if feature in approved else 0 for feature in features] - result = set(approved) # keep the exposed approvals A_v - for project in hidden_projects(ballot, instance): - kind, payload = model["per_project"][project] - if kind == "const": - approve = payload - else: - import numpy as np - approve = int(payload.predict(np.array([tv_features]))[0]) - if approve == 1: - result.add(project) + # The TV voter's feature row x: her vote (1/0) on each exposed feature project. + x = np.array([[1 if f in approved else 0 for f in model["features"]]]) + hidden = hidden_projects(ballot, instance) + votes = { + p: (payload if kind == "const" else int(payload.predict(x)[0])) + for p, (kind, payload) in model["per_project"].items() + if p in hidden + } + # predicted ballot = A_v u {p in H_v : classifier of p predicts approval} + result = approved | {p for p in hidden if votes[p] == 1} logger.info("predict_by_classification: completed ballot to %d approvals", len(result)) return ApprovalBallot(result) @@ -694,11 +681,10 @@ def _approve_hidden_by_score( >>> _approve_hidden_by_score(inst, b, {p1: 1.0, p2: 0.8, p3: 0.2}) == {p1, p2} True """ - result = set(approved_projects(ballot)) - for project in hidden_projects(ballot, instance): - if scores[project] >= 0.5: - result.add(project) - return result + # predicted ballot = A_v u {p in H_v : score(p) >= 1/2} + return approved_projects(ballot) | { + p for p in hidden_projects(ballot, instance) if scores[p] >= 0.5 + } def predict_by_matrix_factorization( @@ -848,21 +834,22 @@ def recommend( """ # Sampling: the offline-popularity sampler exposes the same k projects to # every Target Voter. + # Sampling -> prediction: complete every TV ballot from its k exposed answers. exposed = offline_popularity(instance, lv_profile, k) - # Prediction: complete each TV ballot; start the combined profile from the LV - # ballots (which are already full). - completed_ballots = list(lv_profile) - for voter_id, full_ballot in tv_ballots.items(): - partial = reveal_ballot(instance, full_ballot, exposed) - completed = predict_by_majority(instance, lv_profile, partial) - logger.debug("recommend: TV %s completed to %d approvals", voter_id, len(completed)) - completed_ballots.append(completed) - # Voting rule: greedy approval on the LV + completed TV ballots. - combined = ApprovalProfile(completed_ballots) + combined = ApprovalProfile( + list(lv_profile) + + [ + predict_by_majority( + instance, lv_profile, reveal_ballot(instance, full_ballot, exposed) + ) + for full_ballot in tv_ballots.values() + ] + ) logger.info( "recommend: predicting bundle from %d LV + %d TV ballots", lv_profile.num_ballots(), len(tv_ballots), ) + # Voting rule: greedy approval on the LV + completed TV ballots. return greedy_approval(instance, combined) From ac86e6782c727d590fd668177fbb0bb6cf428ac7 Mon Sep 17 00:00:00 2001 From: Roei Date: Mon, 13 Jul 2026 22:26:07 +0300 Subject: [PATCH 09/16] Add the paper's experiment layer: LV/TV split, pipeline, treatment matrix Full-implementation stage of "A Recommendation System for Participatory Budgeting" (Leibiker & Talmon 2023), experiment side (Sections 3, 5, 6): - split_lv_tv: Step 1, partition the ideal profile into Learning/Target Voters by the paper's sample-degree and LV-degree knobs (Section 3.0.1). - run_pipeline (renamed from recommend): one cell of the design matrix; setup and predict are now required keyword arguments. - exposed_sets: one interface over the five Section 3.1 setups. - run_all_experiments: the full Section 6 treatment matrix (setups x predictors x sample degrees x LV degrees, n_repeat random splits averaged), reporting FA and inline Symmetric Distance per cell. - classification_metrics: Section 5.1 precision/recall/F1 over the hidden projects, dependency-free. - Remove predict_by_majority (not one of the paper's predictors) and its tests; PREDICTORS is now exactly classification/MF/FM. - fractional_allocation_score accepts any Iterable[Project], so the BudgetAllocation from greedy_approval can be passed as is. - Fix the [recommendation] extra: add scikit-learn (XGBClassifier needs xgboost's sklearn API), drop unused scikit-surprise/lightfm whose wheels do not build on Windows + NumPy 2. - Add the paper link to every file header per the submission rules. Co-Authored-By: Claude Opus 4.8 --- pb_model_training.py | 5 +- pb_recommendation.py | 441 +++++++++++++++++++++++++++++++------- pyproject.toml | 7 +- test_pb_recommendation.py | 98 +++++---- 4 files changed, 426 insertions(+), 125 deletions(-) diff --git a/pb_model_training.py b/pb_model_training.py index 4fd8892..cb81690 100644 --- a/pb_model_training.py +++ b/pb_model_training.py @@ -1,7 +1,8 @@ """ Model *training* for the learning-based prediction modules of -"A Recommendation System for Participatory Budgeting" -(Leibiker & Talmon, 2023), Section 2.1. +"A Recommendation System for Participatory Budgeting", +by Gil Leibiker and Nimrod Talmon (2023), https://optlearnmas23.github.io/files/p17.pdf +(Section 2.1). This module is intentionally separate from the module that *uses* the models (``pb_recommendation``): a ``train_*`` function fits an estimator and returns an diff --git a/pb_recommendation.py b/pb_recommendation.py index 9082f37..75b6ce7 100644 --- a/pb_recommendation.py +++ b/pb_recommendation.py @@ -26,8 +26,7 @@ from pabutools.election.satisfaction import Cost_Sat from pabutools.rules import BudgetAllocation, greedy_utilitarian_welfare -# Model training lives in a separate module, ``pb_model_training`` (the -# professor's note b: separate files for training the model and using it). The +# Model training lives in a separate module, ``pb_model_training``. The # learning-based predictors below delegate the fitting to its ``train_*`` # functions; ``pb_model_training`` is ballot-agnostic (it takes plain project # sets), so the dependency is one-directional and there is no import cycle. @@ -37,8 +36,8 @@ train_factorization_machines, ) -# Log the steps of every algorithm at INFO/DEBUG level (see the assignment's -# logging requirement). Configure a handler in your own script to see them, e.g. +# Log the steps of every algorithm at INFO/DEBUG level. +# Configure a handler in your own script to see them, e.g. # ``logging.basicConfig(level=logging.INFO)``. logger = logging.getLogger(__name__) @@ -557,57 +556,8 @@ def online_adaptive_controversial( # --------------------------------------------------------------------------- -# Section 2.1 / examples - Prediction module (majority rule over LV). +# Section 2.1.1 - Binary classification (predicted with a trained model). # --------------------------------------------------------------------------- -def predict_by_majority( - instance: Instance, - lv_profile: ApprovalProfile, - ballot: CardinalBallot, -) -> ApprovalBallot: - """ - Prediction module (Section 2.1). Completes a single partial TV ballot into a - full approval ballot: every project in the hidden set H_v is predicted as - approved iff its approval rate among the LV voters is at least 50%. The - exposed approvals A_v are kept; the exposed disapprovals D_v stay rejected. - - Parameters - ---------- - instance : :py:class:`~pabutools.election.instance.Instance` - The PB instance. - lv_profile : :py:class:`~pabutools.election.profile.approvalprofile.ApprovalProfile` - The full ballots of the LV voters, used as the training data. - ballot : :py:class:`~pabutools.election.ballot.cardinalballot.CardinalBallot` - The Target Voter's partial ballot to complete (the +1/-1/0 partial - ballot built by ``partial_ballot`` / ``reveal_ballot``). - - Returns - ------- - :py:class:`~pabutools.election.ballot.approvalballot.ApprovalBallot` - The full predicted approval ballot of the TV voter. - - Examples - -------- - Example 11 from the paper: 2 LV voters both approve {p1, p2}. A TV voter with - nothing exposed (all three projects hidden) is predicted to approve p1 (100%) - and p2 (100%) but not p3 (0%). - - >>> p1, p2, p3 = Project("p1", 4), Project("p2", 4), Project("p3", 6) - >>> inst = Instance([p1, p2, p3], budget_limit=6) - >>> lv = ApprovalProfile([ApprovalBallot([p1, p2]), ApprovalBallot([p1, p2])]) - >>> partial = partial_ballot(hidden={p1, p2, p3}) - >>> predict_by_majority(inst, lv, partial) == {p1, p2} - True - """ - n = lv_profile.num_ballots() - scores = lv_profile.approval_scores() # Definition 2.1, from the library - # predicted ballot = A_v u {p in H_v : LV approval rate of p >= 1/2} - result = approved_projects(ballot) | { - p for p in hidden_projects(ballot, instance) if 2 * scores.get(p, 0) >= n - } - logger.info("predict_by_majority: completed ballot to %d approvals", len(result)) - return ApprovalBallot(result) - - def predict_by_classification( instance: Instance, lv_profile: ApprovalProfile, @@ -791,16 +741,152 @@ def predict_by_factorization_machines( # --------------------------------------------------------------------------- # Full pipeline (sampling -> prediction -> greedy approval). # --------------------------------------------------------------------------- -def recommend( +# The paper's design space is a matrix: one setup (Section 3.1) times one +# predictor (Section 2.1). These registries name every choice so a caller can +# pick a cell (``run_pipeline``) or sweep the whole matrix (``run_all_experiments``). +SETUPS = ( + "random", + "offline_popularity", + "offline_consensus", + "offline_controversiality", + "online_adaptive_controversial", +) + +PREDICTORS = { + "classification": predict_by_classification, + "matrix_factorization": predict_by_matrix_factorization, + "factorization_machines": predict_by_factorization_machines, +} + + +def exposed_sets( instance: Instance, lv_profile: ApprovalProfile, tv_ballots: dict[str, set[Project]], + setup: str, k: int, + seed: int | None = None, +) -> dict[str, set[Project]]: + """ + The exposed set E_v of every Target Voter under one of the five Section 3.1 + setups, keyed by voter id. Helper that hides the setups' differing + signatures behind one interface: the three offline samplers expose the + *same* k projects to everyone, while ``random`` and + ``online_adaptive_controversial`` are computed per voter (the latter needs + each voter's full ballot to answer its adaptive questions). + + Examples + -------- + Offline popularity exposes the single most popular LV project (p1) to every + Target Voter. + + >>> p1, p2, p3 = Project("p1", 1), Project("p2", 1), Project("p3", 1) + >>> inst = Instance([p1, p2, p3], budget_limit=3) + >>> lv = ApprovalProfile([ApprovalBallot([p1, p2]), ApprovalBallot([p1])]) + >>> exposed_sets(inst, lv, {"v3": {p1}}, "offline_popularity", k=1) + {'v3': {p1}} + """ + if setup == "random": + return {vid: random_setup(instance, k, seed) for vid in tv_ballots} + if setup == "online_adaptive_controversial": + return { + vid: online_adaptive_controversial(instance, lv_profile, full_ballot, k) + for vid, full_ballot in tv_ballots.items() + } + offline = { + "offline_popularity": offline_popularity, + "offline_consensus": offline_consensus, + "offline_controversiality": offline_controversiality, + }[setup] + shared = offline(instance, lv_profile, k) # same set for every TV voter + return {vid: shared for vid in tv_ballots} + + +def split_lv_tv( + profile: ApprovalProfile, + sample_degree: float, + lv_degree: float, + seed: int | None = None, +) -> tuple[ApprovalProfile, dict[str, set[Project]]]: + """ + Step 1 of the pipeline (Section 2.4 / 3.0.1): partition the voters of the + *ideal* instance into Learning Voters (LV, who keep their full ballots) and + Target Voters (TV, whose ballots start hidden and are later completed). + + The partition follows the paper's two knobs (Example 3.1), read here at the + voter level: + + * ``sample_degree`` - the fraction of voters that are *sampled* (participate). + The rest are dropped from the partial instance I1. + * ``lv_degree`` - among the sampled voters, the fraction that are LV; the rest + are TV. ``lv_degree == 1`` is the paper's naive "sampling" baseline (every + sampled voter gives a full ballot, no prediction needed). + + Parameters + ---------- + profile : :py:class:`~pabutools.election.profile.approvalprofile.ApprovalProfile` + The ideal instance's full ballots (all n voters). + sample_degree : float + Fraction of voters sampled, in [0, 1]. + lv_degree : float + Fraction of the sampled voters that are LV, in [0, 1]. + seed : int, optional + Seed for the random partition, for reproducibility. + + Returns + ------- + tuple[:py:class:`~pabutools.election.profile.approvalprofile.ApprovalProfile`, dict[str, set[:py:class:`~pabutools.election.instance.Project`]]] + The LV profile (full ballots) and the TV voters' full ballots keyed by + voter id (their known ground truth, used to simulate the k answers). + + Examples + -------- + Four voters. Sampling everyone with ``lv_degree == 1`` makes everyone an LV + and leaves no TV; a half sample split evenly gives one LV and one TV. + + >>> p1, p2 = Project("p1", 1), Project("p2", 1) + >>> prof = ApprovalProfile([ApprovalBallot([p1]), ApprovalBallot([p2]), + ... ApprovalBallot([p1, p2]), ApprovalBallot([])]) + >>> lv, tv = split_lv_tv(prof, sample_degree=1.0, lv_degree=1.0) + >>> lv.num_ballots(), len(tv) + (4, 0) + >>> lv, tv = split_lv_tv(prof, sample_degree=0.5, lv_degree=0.5, seed=0) + >>> lv.num_ballots(), len(tv) + (1, 1) + """ + voters = list(profile) + order = list(range(len(voters))) + random.Random(seed).shuffle(order) + n_sample = round(sample_degree * len(voters)) + sampled = order[:n_sample] + n_lv = round(lv_degree * n_sample) + lv_profile = ApprovalProfile([voters[i] for i in sampled[:n_lv]]) + tv_ballots = {f"v{i}": set(voters[i]) for i in sampled[n_lv:]} + logger.info( + "split_lv_tv: sample=%.2f lv=%.2f -> %d LV, %d TV (of %d voters)", + sample_degree, lv_degree, lv_profile.num_ballots(), len(tv_ballots), + len(voters), + ) + return lv_profile, tv_ballots + + +def run_pipeline( + instance: Instance, + lv_profile: ApprovalProfile, + tv_ballots: dict[str, set[Project]], + k: int, + *, + setup: str, + predict, + seed: int | None = None, ) -> BudgetAllocation: """ - The complete Section 3 pipeline with the offline-popularity sampler: expose - the k most popular LV projects to each TV voter, predict the hidden votes by - LV majority, then run greedy approval on the LV plus completed TV ballots. + The complete Section 3 pipeline for one (setup, predictor) choice: expose k + projects to each TV voter with ``setup``, complete the hidden votes with the + ``predict`` module, then run greedy approval on the LV plus completed TV + ballots. ``setup`` and ``predict`` are required - the pipeline runs exactly + one cell of the design matrix, so the caller must name both explicitly (the + sweep over every cell is :py:func:`run_all_experiments`). Parameters ---------- @@ -813,6 +899,13 @@ def recommend( the ideal instance - keyed by voter id. k : int The number of projects to expose per TV voter. + setup : str + The name of the Section 3.1 setup to use (one of :py:data:`SETUPS`). + predict : callable + The Section 2.1 prediction module (one of :py:data:`PREDICTORS`' + values). + seed : int, optional + Seed for the ``random`` setup, ignored by the others. Returns ------- @@ -829,35 +922,232 @@ def recommend( ... Project("p3", 4), Project("p4", 4)) >>> inst = Instance([p1, p2, p3, p4], budget_limit=6) >>> lv = ApprovalProfile([ApprovalBallot([p1, p2])] * 3) - >>> sorted(recommend(inst, lv, {"v4": {p1, p2}}, k=1), key=str) + >>> sorted(run_pipeline(inst, lv, {"v4": {p1, p2}}, k=1, + ... setup="offline_popularity", + ... predict=predict_by_matrix_factorization), key=str) [p1, p2] """ - # Sampling: the offline-popularity sampler exposes the same k projects to - # every Target Voter. - # Sampling -> prediction: complete every TV ballot from its k exposed answers. - exposed = offline_popularity(instance, lv_profile, k) + # Step 1 (the LV/TV split) is done by the caller / :py:func:`split_lv_tv`. + # Step 2 - sampling: pick the exposed set of every TV voter under the setup. + exposed = exposed_sets(instance, lv_profile, tv_ballots, setup, k, seed) + # Steps 3-4 - prediction + combine: complete every TV ballot from its k + # exposed answers and merge with the known LV ballots. combined = ApprovalProfile( list(lv_profile) + [ - predict_by_majority( - instance, lv_profile, reveal_ballot(instance, full_ballot, exposed) + predict( + instance, + lv_profile, + reveal_ballot(instance, tv_ballots[vid], exposed[vid]), ) - for full_ballot in tv_ballots.values() + for vid in tv_ballots ] ) logger.info( - "recommend: predicting bundle from %d LV + %d TV ballots", - lv_profile.num_ballots(), len(tv_ballots), + "recommend: setup=%s predict=%s, %d LV + %d TV ballots", + setup, predict.__name__, lv_profile.num_ballots(), len(tv_ballots), ) - # Voting rule: greedy approval on the LV + completed TV ballots. + # Steps 5-6 - voting rule: greedy approval on the LV + completed TV ballots. return greedy_approval(instance, combined) +#: The partiality grid swept in the paper's experiments (Section 6). +SAMPLE_DEGREES = (0.1, 0.15, 0.3, 0.5, 0.7, 0.9) +LV_DEGREES = (0.1, 0.2, 0.3, 0.5, 0.7, 0.9, 1.0) + + +def run_all_experiments( + instance: Instance, + profile: ApprovalProfile, + k: int, + *, + setups: Iterable[str] = SETUPS, + predictors: Iterable[str] = tuple(PREDICTORS), + sample_degrees: Iterable[float] = SAMPLE_DEGREES, + lv_degrees: Iterable[float] = LV_DEGREES, + n_repeat: int = 50, + seed: int | None = None, +) -> dict[tuple[float, float, str, str], dict[str, float]]: + """ + The paper's full treatment matrix (Section 6, Figure 4). For every cell - + setup x predictor x sample_degree x lv_degree - the ideal ``profile`` is + split into LV/TV with :py:func:`split_lv_tv`, the pipeline is run with + :py:func:`run_pipeline`, and the predicted bundle is compared to the *real* + bundle (greedy approval on the whole ideal profile). Because the split is + random, each cell is repeated ``n_repeat`` times and the two bundle metrics - + Fractional Allocation (:py:func:`fractional_allocation_score`) and the + Symmetric Distance ``|rb △ pb|`` (Section 5.2.1, computed inline) - are + averaged. + + .. note:: + The paper repeats the sampling module 20 times and the prediction module + 50 times; ``n_repeat`` collapses both into one knob (default 50). The full + grid is ``len(setups) * len(predictors) * len(sample_degrees) * + len(lv_degrees)`` cells, so shrink the iterables or ``n_repeat`` for a + quick run. ``classification`` needs ``scikit-learn`` installed. + + Parameters + ---------- + instance : :py:class:`~pabutools.election.instance.Instance` + The PB instance. + profile : :py:class:`~pabutools.election.profile.approvalprofile.ApprovalProfile` + The ideal instance's full ballots (all voters). + k : int + The number of projects to expose per TV voter. + setups : Iterable[str], optional + The setups to sweep (names from :py:data:`SETUPS`). + predictors : Iterable[str], optional + The predictors to sweep (names from :py:data:`PREDICTORS`). Defaults to + all three paper predictors (classification / MF / FM). + sample_degrees, lv_degrees : Iterable[float], optional + The partiality grid (Section 3.0.1). Default to the paper's ranges. + n_repeat : int, optional + Random splits averaged per cell (default 50). + seed : int, optional + Seed for the whole sweep, for reproducibility. + + Returns + ------- + dict[tuple[float, float, str, str], dict[str, float]] + Keyed by ``(sample_degree, lv_degree, setup, predictor)``, each value + is ``{"FA": mean fractional allocation, "SD": mean symmetric distance}``. + + Examples + -------- + Example 10 from the paper - the "perfect" case, swept over a tiny grid with + the two ML-free recommendation predictors. Every cell yields an FA in [0, 1] + and a non-negative SD. + + >>> p1, p2, p3, p4 = (Project("p1", 3), Project("p2", 3), + ... Project("p3", 4), Project("p4", 4)) + >>> inst = Instance([p1, p2, p3, p4], budget_limit=6) + >>> prof = ApprovalProfile([ApprovalBallot([p1, p2])] * 4) + >>> results = run_all_experiments( + ... inst, prof, k=1, + ... predictors=("matrix_factorization", "factorization_machines"), + ... sample_degrees=(0.5,), lv_degrees=(0.5,), n_repeat=2, seed=0) + >>> len(results) == 5 * 2 # 5 setups x 2 predictors x 1 x 1 cells + True + >>> all(0.0 <= c["FA"] <= 1.0 and c["SD"] >= 0 for c in results.values()) + True + """ + # The real bundle: greedy approval on the whole ideal profile (all voters). + real_bundle = set(greedy_approval(instance, profile)) + rng = random.Random(seed) + results: dict[tuple[float, float, str, str], dict[str, float]] = {} + for sample_degree in sample_degrees: + for lv_degree in lv_degrees: + for setup in setups: + for name in predictors: + predict = PREDICTORS[name] + fa_sum = sd_sum = 0.0 + for _ in range(n_repeat): + lv_profile, tv_ballots = split_lv_tv( + profile, sample_degree, lv_degree, + seed=rng.randrange(2**32), + ) + predicted = set( + run_pipeline( + instance, lv_profile, tv_ballots, k, + setup=setup, predict=predict, + seed=rng.randrange(2**32), + ) + ) + fa_sum += fractional_allocation_score( + real_bundle, predicted, instance.budget_limit + ) + # Symmetric Distance (Section 5.2.1): |rb △ pb|. + sd_sum += len(real_bundle ^ predicted) + cell = {"FA": fa_sum / n_repeat, "SD": sd_sum / n_repeat} + results[(sample_degree, lv_degree, setup, name)] = cell + logger.info( + "run_all_experiments: sample=%.2f lv=%.2f setup=%s " + "predict=%s -> FA=%.3f SD=%.2f", + sample_degree, lv_degree, setup, name, + cell["FA"], cell["SD"], + ) + return results + + +# --------------------------------------------------------------------------- +# Section 5.1 - Classification accuracy metrics. +# --------------------------------------------------------------------------- +def classification_metrics( + real_approved: set[Project], + predicted_approved: set[Project], + hidden: set[Project], +) -> dict[str, float]: + """ + Section 5.1 (Classification Accuracy Metrics): precision, recall and F1 of a + prediction module, measured over one Target Voter's *hidden* projects (the + test set - the exposed votes are known, not predicted, so they are excluded). + Approval is the positive class, so from the confusion matrix over the hidden + projects: + + * TP = hidden projects the voter really approves and we predicted approve, + * FP = hidden projects we predicted approve but she really rejects, + * FN = hidden projects she really approves but we predicted reject, + + then ``precision = TP / (TP + FP)``, ``recall = TP / (TP + FN)`` and + ``F1 = 2 * precision * recall / (precision + recall)``. A denominator of 0 + (no predicted or no real approvals among the hidden projects) yields 0.0 for + that metric, the usual convention for an undefined score. + + Parameters + ---------- + real_approved : set[:py:class:`~pabutools.election.instance.Project`] + The projects the voter really approves (her ideal ballot A_v). + predicted_approved : set[:py:class:`~pabutools.election.instance.Project`] + The projects the prediction module marked as approved. + hidden : set[:py:class:`~pabutools.election.instance.Project`] + The voter's hidden set H_v, i.e. the projects scored (from + :py:func:`hidden_projects`). + + Returns + ------- + dict[str, float] + ``{"precision": ..., "recall": ..., "f1": ...}``, each in [0, 1]. + + Examples + -------- + Four hidden projects, the voter really approves {p1, p2}, the model predicts + {p1, p3}: one hit (p1), one false alarm (p3), one miss (p2), so precision = + recall = F1 = 0.5. A perfect prediction scores 1.0 across the board. + + >>> p1, p2, p3, p4 = (Project("p1", 1), Project("p2", 1), + ... Project("p3", 1), Project("p4", 1)) + >>> hidden = {p1, p2, p3, p4} + >>> classification_metrics({p1, p2}, {p1, p3}, hidden) + {'precision': 0.5, 'recall': 0.5, 'f1': 0.5} + >>> classification_metrics({p1, p2}, {p1, p2}, hidden) + {'precision': 1.0, 'recall': 1.0, 'f1': 1.0} + """ + # Restrict everything to the hidden projects: the exposed votes are known. + real = real_approved & hidden + predicted = predicted_approved & hidden + tp = len(real & predicted) + fp = len(predicted - real) + fn = len(real - predicted) + precision = tp / (tp + fp) if tp + fp else 0.0 + recall = tp / (tp + fn) if tp + fn else 0.0 + f1 = 2 * precision * recall / (precision + recall) if precision + recall else 0.0 + logger.info( + "classification_metrics: TP=%d FP=%d FN=%d -> P=%.3f R=%.3f F1=%.3f", + tp, fp, fn, precision, recall, f1, + ) + return {"precision": precision, "recall": recall, "f1": f1} + + # --------------------------------------------------------------------------- # Section 5.2 - Bundle evaluation metrics. # --------------------------------------------------------------------------- +# The Symmetric Distance (Section 5.2.1), |rb △ pb|, is a one-liner +# ``len(real_bundle ^ predicted_bundle)`` computed inline where needed (see +# ``run_all_experiments``), so it gets no function of its own. def fractional_allocation_score( - real_bundle: set[Project], predicted_bundle: set[Project], budget_limit: int + real_bundle: Iterable[Project], + predicted_bundle: Iterable[Project], + budget_limit: int, ) -> float: """ Definition 5.1 (Fractional Allocation score): the total cost of the @@ -866,10 +1156,12 @@ def fractional_allocation_score( Parameters ---------- - real_bundle : set[:py:class:`~pabutools.election.instance.Project`] - The bundle obtained from the real (full) ballots. - predicted_bundle : set[:py:class:`~pabutools.election.instance.Project`] - The bundle obtained from the predicted ballots. + real_bundle : Iterable[:py:class:`~pabutools.election.instance.Project`] + The bundle obtained from the real (full) ballots - any iterable of + projects, e.g. the :py:class:`~pabutools.rules.budgetallocation.BudgetAllocation` + returned by :py:func:`greedy_approval`, as is. + predicted_bundle : Iterable[:py:class:`~pabutools.election.instance.Project`] + The bundle obtained from the predicted ballots (same, any iterable). budget_limit : int The budget limit B of the instance. @@ -889,6 +1181,7 @@ def fractional_allocation_score( >>> fractional_allocation_score({p3}, {p1}, budget_limit=6) 0.0 """ + real_bundle, predicted_bundle = set(real_bundle), set(predicted_bundle) # lambda = total cost of the correctly-predicted projects (pb ∩ rb). correctly_predicted = real_bundle & predicted_bundle score = total_cost(correctly_predicted) / budget_limit diff --git a/pyproject.toml b/pyproject.toml index 5035b63..877fde7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,9 +37,12 @@ dependencies = [ [project.optional-dependencies] recommendation = [ + # predict_by_classification: XGBClassifier needs xgboost's sklearn API. "xgboost", - "scikit-surprise", - "lightfm" + "scikit-learn", + # Matrix factorization / factorization machines run on numpy (already a + # core dependency): the scikit-surprise / lightfm wheels do not build on + # Windows + NumPy 2, so pb_model_training uses its own SVD core instead. ] dev = [ "coverage", diff --git a/test_pb_recommendation.py b/test_pb_recommendation.py index 023d785..5d10b8b 100644 --- a/test_pb_recommendation.py +++ b/test_pb_recommendation.py @@ -1,6 +1,7 @@ """ Unit tests for `pb_recommendation`, the implementation of the algorithms in -"A Recommendation System for Participatory Budgeting" (Leibiker & Talmon, 2023). +"A Recommendation System for Participatory Budgeting", +by Gil Leibiker and Nimrod Talmon (2023), https://optlearnmas23.github.io/files/p17.pdf Run with: pytest test_pb_recommendation.py -v @@ -42,11 +43,11 @@ offline_consensus, offline_controversiality, online_adaptive_controversial, - predict_by_majority, predict_by_classification, predict_by_matrix_factorization, predict_by_factorization_machines, - recommend, + run_pipeline, + classification_metrics, fractional_allocation_score, ) @@ -340,46 +341,9 @@ def test_reveal_example2_4(self): # --------------------------------------------------------------------------- -# predict_by_majority -# --------------------------------------------------------------------------- -class TestPredictByMajority: - def test_example11(self): - p = make_projects([("p1", 4), ("p2", 4), ("p3", 6)]) - inst = Instance(p.values(), budget_limit=6) - lv = ApprovalProfile( - [ApprovalBallot([p["p1"], p["p2"]]), ApprovalBallot([p["p1"], p["p2"]])] - ) - partial = partial_ballot(hidden=set(p.values())) - pred = predict_by_majority(inst, lv, partial) - assert set(pred) == {p["p1"], p["p2"]} - - def test_exposed_approvals_are_kept(self): - p = make_projects([("p1", 1), ("p2", 1), ("p3", 1)]) - inst = Instance(p.values(), budget_limit=3) - # LV would reject p3 (0%), but the voter explicitly approved it. - lv = ApprovalProfile([ApprovalBallot([p["p1"]]), ApprovalBallot([p["p1"]])]) - partial = partial_ballot( - approved={p["p3"]}, hidden={p["p1"], p["p2"]} - ) - pred = predict_by_majority(inst, lv, partial) - assert p["p3"] in pred - - def test_exposed_disapprovals_stay_rejected(self): - p = make_projects([("p1", 1), ("p2", 1)]) - inst = Instance(p.values(), budget_limit=2) - # LV approves both p1 and p2 unanimously, but the voter explicitly - # disapproved p1; p2 is hidden and should be predicted as approved. - lv = ApprovalProfile([ApprovalBallot([p["p1"], p["p2"]])] * 3) - partial = partial_ballot(disapproved={p["p1"]}, hidden={p["p2"]}) - pred = predict_by_majority(inst, lv, partial) - assert p["p1"] not in pred # the explicit disapproval is honoured - assert p["p2"] in pred # hidden + LV majority -> approved - - -# --------------------------------------------------------------------------- -# Library-backed predictors: classification (XGBoost), MF, FM (Section 2.1). -# They share the contract of predict_by_majority, so the same behavioural -# invariants are checked for each one. +# The three prediction modules of the paper: classification (XGBoost), MF, FM +# (Section 2.1). They share one contract - keep the exposed votes, predict the +# hidden ones - so the same behavioural invariants are checked for each. # --------------------------------------------------------------------------- LIBRARY_PREDICTORS = [ predict_by_classification, @@ -416,14 +380,16 @@ def test_exposed_votes_are_respected(self, predictor): # --------------------------------------------------------------------------- -# recommend (full pipeline) +# run_pipeline (full pipeline) # --------------------------------------------------------------------------- -class TestRecommend: +class TestRunPipeline: def test_example10_perfect(self): p = make_projects([("p1", 3), ("p2", 3), ("p3", 4), ("p4", 4)]) inst = Instance(p.values(), budget_limit=6) lv = ApprovalProfile([ApprovalBallot([p["p1"], p["p2"]])] * 3) - bundle = set(recommend(inst, lv, {"v4": {p["p1"], p["p2"]}}, k=1)) + bundle = set(run_pipeline(inst, lv, {"v4": {p["p1"], p["p2"]}}, k=1, + setup="offline_popularity", + predict=predict_by_matrix_factorization)) assert bundle == {p["p1"], p["p2"]} def test_pipeline_respects_budget_random(self): @@ -432,12 +398,50 @@ def test_pipeline_respects_budget_random(self): inst = Instance(p.values(), budget_limit=6) lv = ApprovalProfile([ApprovalBallot([p["p1"], p["p2"]])] * 4) tv = {"v1": {p["p1"], p["p2"]}, "v2": {p["p1"]}} - bundle = list(recommend(inst, lv, tv, k=2)) + bundle = list(run_pipeline(inst, lv, tv, k=2, + setup="offline_popularity", + predict=predict_by_matrix_factorization)) assert set(bundle) <= set(p.values()) assert sum(proj.cost for proj in bundle) <= inst.budget_limit assert len(bundle) > 0 +# --------------------------------------------------------------------------- +# classification_metrics (Section 5.1) +# --------------------------------------------------------------------------- +class TestClassificationMetrics: + def test_mixed_hit_miss_false_alarm(self): + p = make_projects([("p1", 1), ("p2", 1), ("p3", 1), ("p4", 1)]) + hidden = set(p.values()) + # really approves {p1, p2}, predicted {p1, p3}: hit p1, miss p2, false p3. + m = classification_metrics({p["p1"], p["p2"]}, {p["p1"], p["p3"]}, hidden) + assert m == {"precision": 0.5, "recall": 0.5, "f1": 0.5} + + def test_perfect_prediction(self): + p = make_projects([("p1", 1), ("p2", 1), ("p3", 1)]) + hidden = set(p.values()) + m = classification_metrics({p["p1"], p["p2"]}, {p["p1"], p["p2"]}, hidden) + assert m == {"precision": 1.0, "recall": 1.0, "f1": 1.0} + + def test_exposed_votes_excluded(self): + # A wrong prediction on an *exposed* project must not affect the metrics: + # only the hidden set is scored. Hidden = {p2}, predicted perfectly there. + p = make_projects([("p1", 1), ("p2", 1)]) + m = classification_metrics( + real_approved={p["p2"]}, # p1 exposed, p2 hidden+approved + predicted_approved={p["p1"], p["p2"]}, + hidden={p["p2"]}, + ) + assert m == {"precision": 1.0, "recall": 1.0, "f1": 1.0} + + def test_no_approvals_gives_zero(self): + # No real and no predicted approvals among the hidden projects -> the + # metrics are undefined and reported as 0.0 by convention. + p = make_projects([("p1", 1), ("p2", 1)]) + m = classification_metrics(set(), set(), hidden=set(p.values())) + assert m == {"precision": 0.0, "recall": 0.0, "f1": 0.0} + + # --------------------------------------------------------------------------- # fractional_allocation_score # --------------------------------------------------------------------------- From 42c34ff699e0f17a8d4e89973c3b3882688b69a1 Mon Sep 17 00:00:00 2001 From: Roei Date: Mon, 13 Jul 2026 22:51:07 +0300 Subject: [PATCH 10/16] Add end-to-end bundle-recovery test over all setup x predictor pairs A structured two-camp electorate (60% majority approving {p1,p2,p3}, 40% minority approving {p4,p5,p6}) fixes the real winning bundle; a third of the voters become Target Voters with only k=2 of 6 projects exposed. Every combination of the 5 sampling setups and 3 prediction modules must reconstruct the exact real bundle (FA = 1.0, SD = 0) - the paper's own success criterion (Section 2.4 / 5.2). 15 new cases. Co-Authored-By: Claude Opus 4.8 --- test_pb_recommendation.py | 44 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/test_pb_recommendation.py b/test_pb_recommendation.py index 5d10b8b..06ef766 100644 --- a/test_pb_recommendation.py +++ b/test_pb_recommendation.py @@ -405,6 +405,50 @@ def test_pipeline_respects_budget_random(self): assert sum(proj.cost for proj in bundle) <= inst.budget_limit assert len(bundle) > 0 + @pytest.mark.parametrize("setup", [ + "random", + "offline_popularity", + "offline_consensus", + "offline_controversiality", + "online_adaptive_controversial", + ]) + @pytest.mark.parametrize("predictor", LIBRARY_PREDICTORS) + def test_two_camp_electorate_recovers_real_bundle(self, setup, predictor): + """ + The end-to-end criterion of the paper (Section 2.4): the pipeline must + estimate the *ideal* outcome. A structured two-camp electorate - a 60% + majority camp approving {p1, p2, p3} and a 40% minority camp approving + {p4, p5, p6} - fixes the real winning bundle at {p1, p2, p3}. A third + of the voters become TV with only k=2 of the 6 projects exposed, and + every setup x predictor combination must still reconstruct the exact + real bundle (FA = 1.0, SD = 0). + """ + p = make_projects([(f"p{i}", 1) for i in range(1, 7)]) + camp_a = {p["p1"], p["p2"], p["p3"]} # 18 of 30 voters (60%) + camp_b = {p["p4"], p["p5"], p["p6"]} # 12 of 30 voters (40%) + inst = Instance(p.values(), budget_limit=3) + + # The real bundle, from all 30 full ballots: camp A's projects score + # 18 > 12, and the budget funds exactly three unit-cost projects. + full_profile = ApprovalProfile( + [ApprovalBallot(camp_a)] * 18 + [ApprovalBallot(camp_b)] * 12 + ) + real_bundle = set(greedy_approval(inst, full_profile)) + assert real_bundle == camp_a # sanity: the ground truth is as designed + + # LV/TV split preserving the 60/40 mix: 20 LV, 10 TV. + lv = ApprovalProfile([ApprovalBallot(camp_a)] * 12 + + [ApprovalBallot(camp_b)] * 8) + tv = {f"a{i}": set(camp_a) for i in range(6)} + tv.update({f"b{i}": set(camp_b) for i in range(4)}) + + predicted = set(run_pipeline(inst, lv, tv, k=2, + setup=setup, predict=predictor, seed=7)) + assert predicted == real_bundle + assert fractional_allocation_score( + real_bundle, predicted, inst.budget_limit) == 1.0 + assert len(real_bundle ^ predicted) == 0 # symmetric distance + # --------------------------------------------------------------------------- # classification_metrics (Section 5.1) From ac4d5cf635a034431d89b8633886646a58d76c00 Mon Sep 17 00:00:00 2001 From: Roei Date: Mon, 13 Jul 2026 23:18:31 +0300 Subject: [PATCH 11/16] Add logging demo and input validation with a failure demo Running "python pb_recommendation.py" now shows, like the course examples: the doctest summary, the full DEBUG/INFO log trail of one end-to-end pipeline run on a two-camp electorate (split -> sampling -> per-voter prediction -> greedy approval -> FA score), and a validation failure demo. exposed_sets validates its inputs: an unknown setup name or a k outside [0, m] raises a ValueError naming the problem (was a raw KeyError before). Covered by doctests and two pytest cases; also fix the stale "recommend:" log prefix left from the run_pipeline rename. Co-Authored-By: Claude Opus 4.8 --- pb_recommendation.py | 71 +++++++++++++++++++++++++++++++++++++-- test_pb_recommendation.py | 19 +++++++++++ 2 files changed, 88 insertions(+), 2 deletions(-) diff --git a/pb_recommendation.py b/pb_recommendation.py index 75b6ce7..30e7ae0 100644 --- a/pb_recommendation.py +++ b/pb_recommendation.py @@ -785,7 +785,27 @@ def exposed_sets( >>> lv = ApprovalProfile([ApprovalBallot([p1, p2]), ApprovalBallot([p1])]) >>> exposed_sets(inst, lv, {"v3": {p1}}, "offline_popularity", k=1) {'v3': {p1}} + + Invalid inputs raise a ValueError naming the problem: + + >>> exposed_sets(inst, lv, {"v3": {p1}}, "by_magic", k=1) + Traceback (most recent call last): + ... + ValueError: unknown setup 'by_magic'; expected one of: random, offline_popularity, offline_consensus, offline_controversiality, online_adaptive_controversial + >>> exposed_sets(inst, lv, {"v3": {p1}}, "random", k=99) + Traceback (most recent call last): + ... + ValueError: k=99 must be between 0 and the number of projects (3) """ + if setup not in SETUPS: + raise ValueError( + f"unknown setup {setup!r}; expected one of: " + ", ".join(SETUPS) + ) + if not 0 <= k <= len(instance): + raise ValueError( + f"k={k} must be between 0 and the number of projects ({len(instance)})" + ) + logger.debug("exposed_sets: validated setup=%s, k=%d", setup, k) if setup == "random": return {vid: random_setup(instance, k, seed) for vid in tv_ballots} if setup == "online_adaptive_controversial": @@ -944,7 +964,7 @@ def run_pipeline( ] ) logger.info( - "recommend: setup=%s predict=%s, %d LV + %d TV ballots", + "run_pipeline: setup=%s predict=%s, %d LV + %d TV ballots", setup, predict.__name__, lv_profile.num_ballots(), len(tv_ballots), ) # Steps 5-6 - voting rule: greedy approval on the LV + completed TV ballots. @@ -1194,5 +1214,52 @@ def fractional_allocation_score( if __name__ == "__main__": import doctest + import sys + + print(doctest.testmod()) + + # ------------------------------------------------------------------ + # Logging demo - a complex example. Run ``python pb_recommendation.py`` + # to see the full log trail of one end-to-end pipeline run. + # ------------------------------------------------------------------ + logging.basicConfig( + level=logging.DEBUG, format="%(levelname)s\t%(message)s", + stream=sys.stdout, # keep the log lines in order with the prints + ) - doctest.testmod(verbose=True) + print("\n--- Complex example: two-camp electorate, partial ballots ---") + projects = [Project(f"p{i}", 1) for i in range(1, 7)] + camp_a, camp_b = set(projects[:3]), set(projects[3:]) + demo_instance = Instance(projects, budget_limit=3) + # The ideal instance: 18 voters approve {p1,p2,p3}, 12 approve {p4,p5,p6}. + demo_profile = ApprovalProfile( + [ApprovalBallot(camp_a)] * 18 + [ApprovalBallot(camp_b)] * 12 + ) + real = set(greedy_approval(demo_instance, demo_profile)) + + # A third of the voters only answer k=2 questions; predict the rest. + demo_lv, demo_tv = split_lv_tv( + demo_profile, sample_degree=1.0, lv_degree=2 / 3, seed=1 + ) + predicted = set( + run_pipeline( + demo_instance, demo_lv, demo_tv, k=2, + setup="offline_popularity", predict=predict_by_matrix_factorization, + ) + ) + fa = fractional_allocation_score(real, predicted, demo_instance.budget_limit) + print(f"real bundle: {sorted(real, key=str)}") + print(f"predicted bundle: {sorted(predicted, key=str)} (FA={fa:.2f})") + + # ------------------------------------------------------------------ + # Validation failure demo - invalid inputs raise a clear ValueError. + # ------------------------------------------------------------------ + print("\n--- Validation failure demo ---") + for setup_name, k in [("by_magic", 2), ("random", 99)]: + try: + run_pipeline( + demo_instance, demo_lv, demo_tv, k=k, + setup=setup_name, predict=predict_by_matrix_factorization, + ) + except ValueError as error: + print(f"Caught ValueError: {error}") diff --git a/test_pb_recommendation.py b/test_pb_recommendation.py index 06ef766..345db75 100644 --- a/test_pb_recommendation.py +++ b/test_pb_recommendation.py @@ -405,6 +405,25 @@ def test_pipeline_respects_budget_random(self): assert sum(proj.cost for proj in bundle) <= inst.budget_limit assert len(bundle) > 0 + def test_unknown_setup_raises_value_error(self): + p = make_projects([("p1", 1), ("p2", 1)]) + inst = Instance(p.values(), budget_limit=2) + lv = ApprovalProfile([ApprovalBallot([p["p1"]])] * 2) + with pytest.raises(ValueError, match="unknown setup 'by_magic'"): + run_pipeline(inst, lv, {"v1": {p["p1"]}}, k=1, + setup="by_magic", + predict=predict_by_matrix_factorization) + + def test_k_out_of_range_raises_value_error(self): + p = make_projects([("p1", 1), ("p2", 1)]) + inst = Instance(p.values(), budget_limit=2) + lv = ApprovalProfile([ApprovalBallot([p["p1"]])] * 2) + for bad_k in (-1, 3): # below 0 and above the number of projects + with pytest.raises(ValueError, match="must be between 0 and"): + run_pipeline(inst, lv, {"v1": {p["p1"]}}, k=bad_k, + setup="random", + predict=predict_by_matrix_factorization) + @pytest.mark.parametrize("setup", [ "random", "offline_popularity", From 84c1b71d63b531d3ec3c616ddb3f943448fce783 Mon Sep 17 00:00:00 2001 From: Roei Date: Mon, 13 Jul 2026 23:20:59 +0300 Subject: [PATCH 12/16] Remove the __main__ logging demo, keep the input validation The module's __main__ goes back to running only the doctests. The ValueError validation in exposed_sets (unknown setup / k out of range) stays, with its doctests and pytest coverage. Co-Authored-By: Claude Opus 4.8 --- pb_recommendation.py | 49 +------------------------------------------- 1 file changed, 1 insertion(+), 48 deletions(-) diff --git a/pb_recommendation.py b/pb_recommendation.py index 30e7ae0..80a1345 100644 --- a/pb_recommendation.py +++ b/pb_recommendation.py @@ -1214,52 +1214,5 @@ def fractional_allocation_score( if __name__ == "__main__": import doctest - import sys - print(doctest.testmod()) - - # ------------------------------------------------------------------ - # Logging demo - a complex example. Run ``python pb_recommendation.py`` - # to see the full log trail of one end-to-end pipeline run. - # ------------------------------------------------------------------ - logging.basicConfig( - level=logging.DEBUG, format="%(levelname)s\t%(message)s", - stream=sys.stdout, # keep the log lines in order with the prints - ) - - print("\n--- Complex example: two-camp electorate, partial ballots ---") - projects = [Project(f"p{i}", 1) for i in range(1, 7)] - camp_a, camp_b = set(projects[:3]), set(projects[3:]) - demo_instance = Instance(projects, budget_limit=3) - # The ideal instance: 18 voters approve {p1,p2,p3}, 12 approve {p4,p5,p6}. - demo_profile = ApprovalProfile( - [ApprovalBallot(camp_a)] * 18 + [ApprovalBallot(camp_b)] * 12 - ) - real = set(greedy_approval(demo_instance, demo_profile)) - - # A third of the voters only answer k=2 questions; predict the rest. - demo_lv, demo_tv = split_lv_tv( - demo_profile, sample_degree=1.0, lv_degree=2 / 3, seed=1 - ) - predicted = set( - run_pipeline( - demo_instance, demo_lv, demo_tv, k=2, - setup="offline_popularity", predict=predict_by_matrix_factorization, - ) - ) - fa = fractional_allocation_score(real, predicted, demo_instance.budget_limit) - print(f"real bundle: {sorted(real, key=str)}") - print(f"predicted bundle: {sorted(predicted, key=str)} (FA={fa:.2f})") - - # ------------------------------------------------------------------ - # Validation failure demo - invalid inputs raise a clear ValueError. - # ------------------------------------------------------------------ - print("\n--- Validation failure demo ---") - for setup_name, k in [("by_magic", 2), ("random", 99)]: - try: - run_pipeline( - demo_instance, demo_lv, demo_tv, k=k, - setup=setup_name, predict=predict_by_matrix_factorization, - ) - except ValueError as error: - print(f"Caught ValueError: {error}") + doctest.testmod(verbose=True) From ff4172bc775130164006ad3f96c10ba5cd1c935a Mon Sep 17 00:00:00 2001 From: Roei Date: Mon, 13 Jul 2026 23:31:58 +0300 Subject: [PATCH 13/16] Narrate run_pipeline and run_all_experiments in the log run_pipeline now logs when it starts (setup, predictor, k, LV/TV counts) and when all TV ballots are completed, so the log trail reads chronologically. run_all_experiments announces the sweep dimensions and the ground-truth bundle up front, logs every repeat's FA/SD at DEBUG, and labels the per-cell INFO line as a mean over the repeats. The iterable parameters are materialised as tuples first, so passing generators no longer exhausts them before the sweep. Co-Authored-By: Claude Opus 4.8 --- pb_recommendation.py | 39 ++++++++++++++++++++++++++++++++------- 1 file changed, 32 insertions(+), 7 deletions(-) diff --git a/pb_recommendation.py b/pb_recommendation.py index 80a1345..a176aa2 100644 --- a/pb_recommendation.py +++ b/pb_recommendation.py @@ -947,6 +947,10 @@ def run_pipeline( ... predict=predict_by_matrix_factorization), key=str) [p1, p2] """ + logger.info( + "run_pipeline: starting with setup=%s predict=%s k=%d on %d LV + %d TV ballots", + setup, predict.__name__, k, lv_profile.num_ballots(), len(tv_ballots), + ) # Step 1 (the LV/TV split) is done by the caller / :py:func:`split_lv_tv`. # Step 2 - sampling: pick the exposed set of every TV voter under the setup. exposed = exposed_sets(instance, lv_profile, tv_ballots, setup, k, seed) @@ -964,8 +968,8 @@ def run_pipeline( ] ) logger.info( - "run_pipeline: setup=%s predict=%s, %d LV + %d TV ballots", - setup, predict.__name__, lv_profile.num_ballots(), len(tv_ballots), + "run_pipeline: all %d TV ballots completed, running the voting rule", + len(tv_ballots), ) # Steps 5-6 - voting rule: greedy approval on the LV + completed TV ballots. return greedy_approval(instance, combined) @@ -1051,8 +1055,21 @@ def run_all_experiments( >>> all(0.0 <= c["FA"] <= 1.0 and c["SD"] >= 0 for c in results.values()) True """ + setups, predictors = tuple(setups), tuple(predictors) + sample_degrees, lv_degrees = tuple(sample_degrees), tuple(lv_degrees) + logger.info( + "run_all_experiments: sweeping %d setups x %d predictors x " + "%d sample degrees x %d LV degrees = %d cells, %d repeats each", + len(setups), len(predictors), len(sample_degrees), len(lv_degrees), + len(setups) * len(predictors) * len(sample_degrees) * len(lv_degrees), + n_repeat, + ) # The real bundle: greedy approval on the whole ideal profile (all voters). real_bundle = set(greedy_approval(instance, profile)) + logger.info( + "run_all_experiments: real bundle has %d projects (the ground truth " + "every cell is scored against)", len(real_bundle), + ) rng = random.Random(seed) results: dict[tuple[float, float, str, str], dict[str, float]] = {} for sample_degree in sample_degrees: @@ -1061,7 +1078,7 @@ def run_all_experiments( for name in predictors: predict = PREDICTORS[name] fa_sum = sd_sum = 0.0 - for _ in range(n_repeat): + for repeat in range(1, n_repeat + 1): lv_profile, tv_ballots = split_lv_tv( profile, sample_degree, lv_degree, seed=rng.randrange(2**32), @@ -1073,18 +1090,26 @@ def run_all_experiments( seed=rng.randrange(2**32), ) ) - fa_sum += fractional_allocation_score( + fa = fractional_allocation_score( real_bundle, predicted, instance.budget_limit ) # Symmetric Distance (Section 5.2.1): |rb △ pb|. - sd_sum += len(real_bundle ^ predicted) + sd = len(real_bundle ^ predicted) + fa_sum += fa + sd_sum += sd + logger.debug( + "run_all_experiments: repeat %d/%d of " + "(sample=%.2f, lv=%.2f, %s, %s): FA=%.3f SD=%d", + repeat, n_repeat, sample_degree, lv_degree, + setup, name, fa, sd, + ) cell = {"FA": fa_sum / n_repeat, "SD": sd_sum / n_repeat} results[(sample_degree, lv_degree, setup, name)] = cell logger.info( "run_all_experiments: sample=%.2f lv=%.2f setup=%s " - "predict=%s -> FA=%.3f SD=%.2f", + "predict=%s -> mean FA=%.3f mean SD=%.2f over %d repeats", sample_degree, lv_degree, setup, name, - cell["FA"], cell["SD"], + cell["FA"], cell["SD"], n_repeat, ) return results From e55602655c5e6d1a57f265b704f87d61730bfa29 Mon Sep 17 00:00:00 2001 From: Roei <69538162+roeiyanku@users.noreply.github.com> Date: Tue, 14 Jul 2026 00:14:20 +0300 Subject: [PATCH 14/16] Refactor test cases and remove unused fixtures Updated test cases by removing example1 fixture and several tests related to greedy approval and other functionalities. Adjusted edge case descriptions and cleaned up unused code. --- test_pb_recommendation.py | 153 +------------------------------------- 1 file changed, 1 insertion(+), 152 deletions(-) diff --git a/test_pb_recommendation.py b/test_pb_recommendation.py index 345db75..95760b7 100644 --- a/test_pb_recommendation.py +++ b/test_pb_recommendation.py @@ -8,7 +8,7 @@ Each test is the specification its function must satisfy. They come in three kinds: * small, hand-checked instances (the examples from the paper); - * edge cases (empty profile, single project, full ballots, k bounds); + * edge cases (partial ballots and invalid setup/k values); * large inputs, checked against an independent reference or via invariants. Programmer: Roei Yanku @@ -35,7 +35,6 @@ disapproved_projects, exposed_projects, hidden_projects, - as_approval_ballot, consensus_levels, greedy_approval, random_setup, @@ -60,21 +59,6 @@ def make_projects(specs): return {name: Project(name, cost) for name, cost in specs} -@pytest.fixture -def example1(): - """P = {p1, p2, p3}, costs 1,1,2, B = 3; v1={p1,p2}, v2={p1,p3}, v3={p2}.""" - p = make_projects([("p1", 1), ("p2", 1), ("p3", 2)]) - inst = Instance(p.values(), budget_limit=3) - prof = ApprovalProfile( - [ - ApprovalBallot([p["p1"], p["p2"]]), - ApprovalBallot([p["p1"], p["p3"]]), - ApprovalBallot([p["p2"]]), - ] - ) - return p, inst, prof - - @pytest.fixture def consensus_data(): """4 LV voters: p1 approved 4/0, p2 split 2/2, p3 rejected 0/4.""" @@ -148,11 +132,6 @@ def test_example7(self, consensus_data): assert c[p["p2"]] == 0 assert c[p["p3"]] == 4 - def test_nonnegative_and_bounded_random(self): - projects, inst, prof = random_instance(15, 40, 80, seed=2) - c = consensus_levels(inst, prof) - assert all(0 <= c[p] <= len(prof) for p in projects) - def test_large_structured_even_split_is_zero(self): # Exactly half approve and half reject each project -> consensus 0. projects = padded_projects(20, cost=1) @@ -168,11 +147,6 @@ def test_large_structured_even_split_is_zero(self): # greedy_approval (voting rule) # --------------------------------------------------------------------------- class TestGreedyApproval: - def test_example1(self, example1): - p, inst, prof = example1 - bundle = set(greedy_approval(inst, prof)) - assert bundle == {p["p1"], p["p2"]} - def test_skips_unaffordable_but_funds_affordable(self): # A project that exceeds the budget is skipped, while a cheaper one that # fits is funded. @@ -181,15 +155,6 @@ def test_skips_unaffordable_but_funds_affordable(self): prof = ApprovalProfile([ApprovalBallot([p["cheap"], p["dear"]])]) assert set(greedy_approval(inst, prof)) == {p["cheap"]} - def test_large_structured_known_bundle(self): - # 100 unit-cost projects, all approved by everyone, budget 30 -> exactly - # the 30 name-first projects are funded (tie-break decides everything). - projects = padded_projects(100, cost=1) - inst = Instance(projects, budget_limit=30) - prof = ApprovalProfile([ApprovalBallot(projects)] * 20) - bundle = set(greedy_approval(inst, prof)) - assert bundle == set(projects[:30]) - def test_matches_reference_random(self): # Cross-check against the independent greedy reference, several seeds. for seed in range(5): @@ -210,20 +175,6 @@ def test_exposes_exactly_k(self): assert len(exposed) == 2 assert exposed <= set(p.values()) - def test_returns_a_set_subset_of_projects(self): - projects, inst, _ = random_instance(10, 1, 30, seed=5) - exposed = random_setup(inst, k=3, seed=7) - assert isinstance(exposed, set) - assert len(exposed) == 3 - assert exposed <= set(projects) - - def test_k_equals_all_exposes_everything(self): - # Edge case: exposing k = |P| projects reveals the whole instance. - p = make_projects([("p1", 1), ("p2", 1)]) - inst = Instance(p.values(), budget_limit=2) - assert random_setup(inst, k=2, seed=0) == set(p.values()) - - # --------------------------------------------------------------------------- # offline_popularity / consensus / controversiality # --------------------------------------------------------------------------- @@ -248,15 +199,6 @@ def test_controversiality_example8(self, consensus_data): p, inst, lv = consensus_data assert offline_controversiality(inst, lv, k=1) == {p["p2"]} - def test_popularity_matches_manual_topk_random(self): - # Cross-check the popularity sampler against an independent top-k. - projects, inst, lv = random_instance(20, 40, 50, seed=33) - k = 5 - counts = manual_scores(projects, lv) - expected = set(sorted(projects, key=lambda p: (-counts[p], str(p)))[:k]) - assert offline_popularity(inst, lv, k=k) == expected - - # --------------------------------------------------------------------------- # online_adaptive_controversial # --------------------------------------------------------------------------- @@ -276,14 +218,6 @@ def test_example9(self): inst, lv, {p["p1"], p["p2"]}, k=2 ) == {p["p1"], p["p2"]} - def test_returns_k_distinct(self): - projects, inst, lv = random_instance(15, 30, 60, seed=9) - result = online_adaptive_controversial(inst, lv, set(projects[:5]), k=4) - assert isinstance(result, set) - assert len(result) == 4 - assert result <= set(projects) - - # --------------------------------------------------------------------------- # Partial (three-state) ballots, Section 2.3. Represented as a CardinalBallot # under the convention: +1 approved, -1 disapproved, 0 (or absent) hidden. @@ -300,20 +234,6 @@ def test_scores_follow_convention(self): assert b[p["p2"]] == -1 assert b[p["p3"]] == 0 - def test_states_partition_projects(self): - p = make_projects([("p1", 1), ("p2", 1), ("p3", 1)]) - inst = Instance(p.values(), budget_limit=3) - b = partial_ballot( - approved={p["p1"]}, disapproved={p["p2"]}, hidden={p["p3"]} - ) - assert approved_projects(b) == {p["p1"]} - assert disapproved_projects(b) == {p["p2"]} - assert hidden_projects(b, inst) == {p["p3"]} - assert exposed_projects(b) == {p["p1"], p["p2"]} - assert ( - approved_projects(b) | disapproved_projects(b) | hidden_projects(b, inst) - ) == set(p.values()) - def test_absent_project_is_hidden(self): # A project never mentioned in the ballot is hidden, just like an # explicit 0 - both belong to H_v. @@ -322,13 +242,6 @@ def test_absent_project_is_hidden(self): b = partial_ballot(approved={p["p1"]}, disapproved={p["p2"]}) # p3 absent assert hidden_projects(b, inst) == {p["p3"]} - def test_as_approval_ballot_keeps_only_approvals(self): - p = make_projects([("p1", 1), ("p2", 1)]) - b = partial_ballot(approved={p["p1"]}, disapproved={p["p2"]}) - ab = as_approval_ballot(b) - assert isinstance(ab, ApprovalBallot) - assert set(ab) == {p["p1"]} - def test_reveal_example2_4(self): # Example 2.4: full ballot {p1,p2}, exposed set {p1,p3}. p = make_projects([("p1", 1), ("p2", 1), ("p3", 1), ("p4", 1)]) @@ -353,17 +266,6 @@ def test_reveal_example2_4(self): class TestLibraryPredictors: - @pytest.mark.parametrize("predictor", LIBRARY_PREDICTORS) - def test_strong_pattern_recovered(self, predictor): - # LV unanimously approve {p1, p2} and reject p3; a TV voter with nothing - # exposed must be completed to {p1, p2} by any correct predictor. - p = make_projects([("p1", 4), ("p2", 4), ("p3", 6)]) - inst = Instance(p.values(), budget_limit=6) - lv = ApprovalProfile([ApprovalBallot([p["p1"], p["p2"]])] * 4) - partial = partial_ballot(hidden=set(p.values())) - pred = predictor(inst, lv, partial) - assert set(pred) == {p["p1"], p["p2"]} - @pytest.mark.parametrize("predictor", LIBRARY_PREDICTORS) def test_exposed_votes_are_respected(self, predictor): # The exposed approval p3 is kept and the exposed disapproval p1 stays @@ -383,28 +285,6 @@ def test_exposed_votes_are_respected(self, predictor): # run_pipeline (full pipeline) # --------------------------------------------------------------------------- class TestRunPipeline: - def test_example10_perfect(self): - p = make_projects([("p1", 3), ("p2", 3), ("p3", 4), ("p4", 4)]) - inst = Instance(p.values(), budget_limit=6) - lv = ApprovalProfile([ApprovalBallot([p["p1"], p["p2"]])] * 3) - bundle = set(run_pipeline(inst, lv, {"v4": {p["p1"], p["p2"]}}, k=1, - setup="offline_popularity", - predict=predict_by_matrix_factorization)) - assert bundle == {p["p1"], p["p2"]} - - def test_pipeline_respects_budget_random(self): - # Cheap, widely-approved projects so the pipeline must fund something. - p = make_projects([("p1", 2), ("p2", 2), ("p3", 9), ("p4", 9)]) - inst = Instance(p.values(), budget_limit=6) - lv = ApprovalProfile([ApprovalBallot([p["p1"], p["p2"]])] * 4) - tv = {"v1": {p["p1"], p["p2"]}, "v2": {p["p1"]}} - bundle = list(run_pipeline(inst, lv, tv, k=2, - setup="offline_popularity", - predict=predict_by_matrix_factorization)) - assert set(bundle) <= set(p.values()) - assert sum(proj.cost for proj in bundle) <= inst.budget_limit - assert len(bundle) > 0 - def test_unknown_setup_raises_value_error(self): p = make_projects([("p1", 1), ("p2", 1)]) inst = Instance(p.values(), budget_limit=2) @@ -480,12 +360,6 @@ def test_mixed_hit_miss_false_alarm(self): m = classification_metrics({p["p1"], p["p2"]}, {p["p1"], p["p3"]}, hidden) assert m == {"precision": 0.5, "recall": 0.5, "f1": 0.5} - def test_perfect_prediction(self): - p = make_projects([("p1", 1), ("p2", 1), ("p3", 1)]) - hidden = set(p.values()) - m = classification_metrics({p["p1"], p["p2"]}, {p["p1"], p["p2"]}, hidden) - assert m == {"precision": 1.0, "recall": 1.0, "f1": 1.0} - def test_exposed_votes_excluded(self): # A wrong prediction on an *exposed* project must not affect the metrics: # only the hidden set is scored. Hidden = {p2}, predicted perfectly there. @@ -497,24 +371,10 @@ def test_exposed_votes_excluded(self): ) assert m == {"precision": 1.0, "recall": 1.0, "f1": 1.0} - def test_no_approvals_gives_zero(self): - # No real and no predicted approvals among the hidden projects -> the - # metrics are undefined and reported as 0.0 by convention. - p = make_projects([("p1", 1), ("p2", 1)]) - m = classification_metrics(set(), set(), hidden=set(p.values())) - assert m == {"precision": 0.0, "recall": 0.0, "f1": 0.0} - - # --------------------------------------------------------------------------- # fractional_allocation_score # --------------------------------------------------------------------------- class TestFractionalAllocation: - def test_example11_zero_score(self): - p = make_projects([("p1", 4), ("p3", 6)]) - # disjoint bundles -> 0.0; a full overlap of cost 6 over budget 6 -> 1.0. - assert fractional_allocation_score({p["p3"]}, {p["p1"]}, budget_limit=6) == 0.0 - assert fractional_allocation_score({p["p3"]}, {p["p3"]}, budget_limit=6) == 1.0 - def test_partial_overlap(self): p = make_projects([("p1", 2), ("p2", 3), ("p3", 5)]) # overlap is {p2}, cost 3, budget 10 -> 0.3 @@ -523,17 +383,6 @@ def test_partial_overlap(self): ) assert score == pytest.approx(0.3) - def test_value_matches_overlap_random(self): - projects, inst, _ = random_instance(12, 1, 40, seed=14) - rng = random.Random(15) - rb = set(rng.sample(projects, 6)) - pb = set(rng.sample(projects, 6)) - score = fractional_allocation_score(rb, pb, budget_limit=inst.budget_limit) - expected = sum(p.cost for p in (rb & pb)) / inst.budget_limit - assert 0.0 <= score <= 1.0 - assert score == pytest.approx(expected) - - if __name__ == "__main__": import sys From 7e907ac88651f8ee0b344c0f05f4b0a31cabe89d Mon Sep 17 00:00:00 2001 From: Roei Date: Mon, 27 Jul 2026 21:19:35 +0300 Subject: [PATCH 15/16] Add pabutools.recommendation: a recommendation system for participatory budgeting Implements "A Recommendation System for Participatory Budgeting" by Gil Leibiker and Nimrod Talmon (AAMAS 2023), https://optlearnmas23.github.io/files/p17.pdf The module tackles the paper's information overload problem: instead of asking every voter about every project, it queries each voter for a partial ballot, estimates the votes it did not ask for, and runs the voting rule on the completion. Contents: * the five sampling setups of Section 3.1, plus next_adaptive_question so the online adaptive setup can drive a live process rather than only a simulation * the three prediction modules of Section 2.1: binary classification with XGBoost, matrix factorization, and factorization machines * the LV/TV partiality arithmetic of Section 3.0.1, greedy approval, elect for running a real process, and the evaluation metrics of Section 5 * 43 tests in tests/test_recommendation.py, plus doctests throughout The machine-learning libraries are an optional "recommendation" extra and are imported lazily, so the module imports and every non-ML algorithm works without them; tests that need to fit a model skip rather than fail. Two points where the paper is under-specified or inconsistent are documented in the docstrings with the reading implemented: Section 2.1.1 never defines what a training row is, and the Section 5.2.1 example disagrees with its own definition of symmetric distance. --- docs-source/source/reference/index.rst | 1 + .../source/reference/recommendation/index.rst | 16 + pabutools/recommendation/__init__.py | 109 ++ pabutools/recommendation/model_training.py | 1196 +++++++++++++++++ .../recommendation/recommendation.py | 1031 +++++++++----- pb_model_training.py | 303 ----- pyproject.toml | 10 +- test_pb_recommendation.py | 389 ------ tests/test_recommendation.py | 665 +++++++++ 9 files changed, 2694 insertions(+), 1026 deletions(-) create mode 100644 docs-source/source/reference/recommendation/index.rst create mode 100644 pabutools/recommendation/__init__.py create mode 100644 pabutools/recommendation/model_training.py rename pb_recommendation.py => pabutools/recommendation/recommendation.py (53%) delete mode 100644 pb_model_training.py delete mode 100644 test_pb_recommendation.py create mode 100644 tests/test_recommendation.py diff --git a/docs-source/source/reference/index.rst b/docs-source/source/reference/index.rst index f4be966..e2decbe 100644 --- a/docs-source/source/reference/index.rst +++ b/docs-source/source/reference/index.rst @@ -8,6 +8,7 @@ Reference election/index rules/index analysis/index + recommendation/index visualisation/index tiebreaking fractions diff --git a/docs-source/source/reference/recommendation/index.rst b/docs-source/source/reference/recommendation/index.rst new file mode 100644 index 0000000..f77277f --- /dev/null +++ b/docs-source/source/reference/recommendation/index.rst @@ -0,0 +1,16 @@ +Recommendation module +===================== + +.. automodule:: pabutools.recommendation + +Sampling, pipeline and evaluation +--------------------------------- + +.. automodule:: pabutools.recommendation.recommendation + :members: + +Model training and prediction +----------------------------- + +.. automodule:: pabutools.recommendation.model_training + :members: diff --git a/pabutools/recommendation/__init__.py b/pabutools/recommendation/__init__.py new file mode 100644 index 0000000..0277da4 --- /dev/null +++ b/pabutools/recommendation/__init__.py @@ -0,0 +1,109 @@ +""" +Module implementing the algorithms of +"A Recommendation System for Participatory Budgeting", +by Gil Leibiker and Nimrod Talmon (2023), https://optlearnmas23.github.io/files/p17.pdf + +:py:mod:`~pabutools.recommendation.recommendation` holds the sampling setups, +the pipeline, the voting rule and the evaluation metrics; +:py:mod:`~pabutools.recommendation.model_training` holds the fitting and +prediction of the three learning-based modules (whose ML libraries are optional +dependencies, installed with ``pip install pabutools[recommendation]``). + +Programmer: Roei Yanku +""" + +from pabutools.recommendation.recommendation import ( + APPROVAL, + DISAPPROVAL, + HIDDEN, + partial_ballot, + reveal_ballot, + approved_projects, + disapproved_projects, + exposed_projects, + hidden_projects, + as_approval_ballot, + consensus_levels, + most_consensual_projects, + greedy_approval, + random_setup, + offline_popularity, + offline_consensus, + offline_controversiality, + online_adaptive_controversial, + next_adaptive_question, + SETUPS, + PREDICTORS, + exposed_sets, + plan_sampling, + split_lv_tv, + complete_ballots, + run_pipeline, + run_experiment, + Predictor, + as_predictor, + elect, + SAMPLE_DEGREES, + LV_DEGREES, + run_all_experiments, + classification_metrics, + fractional_allocation_score, +) +from pabutools.recommendation.model_training import ( + train_classification, + train_matrix_factorization, + train_factorization_machines, + predict_by_classification, + predict_by_matrix_factorization, + predict_by_factorization_machines, +) + +__all__ = [ + # Section 2.3 - partial ballots and the +1/-1/0 score convention. + "APPROVAL", + "DISAPPROVAL", + "HIDDEN", + "partial_ballot", + "reveal_ballot", + "approved_projects", + "disapproved_projects", + "exposed_projects", + "hidden_projects", + "as_approval_ballot", + # Section 2.2 - popularity, consensus and the voting rule. + "consensus_levels", + "most_consensual_projects", + "greedy_approval", + # Section 3.1 - the sampling setups. + "SETUPS", + "random_setup", + "offline_popularity", + "offline_consensus", + "offline_controversiality", + "online_adaptive_controversial", + "next_adaptive_question", + "exposed_sets", + # Section 2.1 - the prediction modules. + "PREDICTORS", + "Predictor", + "as_predictor", + "train_classification", + "train_matrix_factorization", + "train_factorization_machines", + "predict_by_classification", + "predict_by_matrix_factorization", + "predict_by_factorization_machines", + # Running a process: a real one, or the paper's experiments. + "plan_sampling", + "elect", + "split_lv_tv", + "complete_ballots", + "run_pipeline", + "run_experiment", + "run_all_experiments", + "SAMPLE_DEGREES", + "LV_DEGREES", + # Section 5 - evaluation. + "classification_metrics", + "fractional_allocation_score", +] diff --git a/pabutools/recommendation/model_training.py b/pabutools/recommendation/model_training.py new file mode 100644 index 0000000..721c803 --- /dev/null +++ b/pabutools/recommendation/model_training.py @@ -0,0 +1,1196 @@ +""" +Model *training* for the learning-based prediction modules of +"A Recommendation System for Participatory Budgeting", +by Gil Leibiker and Nimrod Talmon (2023), https://optlearnmas23.github.io/files/p17.pdf +(Section 2.1). + +This module contains both fitting and prediction for the three learning-based +modules. ``pabutools.recommendation`` is responsible only for sampling, orchestration, +and the voting rule. + +All three models train on the votes the process actually *collected*, which +Section 3.1.1 defines as the preferences of the LV voters **and** the exposed +preferences E_TV of all Target Voters. The Target Voters' hidden votes are the +paper's test set (Section 5) and are never read during training. + +* :py:func:`train_classification` is supervised: one binary classifier per + project, taking preferences on P \\ {p} as input and the preference on p as + its 0/1 output. Every LV voter trains every project's classifier; a Target + Voter trains project p's when p lies in her exposed set E_v, with the + projects she was not asked about left missing - the very gaps that the rows + to be predicted will carry. +* :py:func:`train_matrix_factorization` and + :py:func:`train_factorization_machines` are collaborative filtering: one + voter-project matrix holds the Learning Voters' full ballots **and every Target + Voter's exposed votes** (the paper's E_TV), factorised once for the whole TV + population. The FM hybrid additionally describes every project by Table 2's + attributes - cost, categories and target population segments - which is what + lets a voter's taste reach a project she was never asked about. + +Following Section 5, all three also weight the minority class against the ~10% +approval rate of Table 1, and choose their hyperparameters on a 15% validation +set scored by F1 (Section 5.1's "suitable measure for an imbalanced data"). + +.. note:: + Section 2.1.1 says only that binary classification is used and that XGBoost + is a good algorithm for it; it never states what one training row is, nor + which features it carries. The per-project reading implemented here - a + voter's opinions on the other projects as the features - is therefore one + defensible interpretation of an under-specified section, not something the + paper prescribes. + +.. note:: + The paper names one library only: ``xgboost``, for the classification module + (Section 2.1.1). It does not say what backs matrix factorization or + factorization machines, so ``scikit-surprise`` and ``lightfm`` are this + implementation's choices. Following the maintainer's advice, the ML libraries + are **not a hard requirement**: they are imported lazily by the corresponding + training function and are available through the ``recommendation`` optional + dependency. + +Programmer: Roei Yanku +Date: 2026-06-20. +""" + +from __future__ import annotations + +import logging + +import numpy as np + +from pabutools.election import ( + Instance, + Project, + ApprovalProfile, + ApprovalBallot, + CardinalBallot, +) + +logger = logging.getLogger(__name__) + +#: The exposed votes of every Target Voter, keyed by voter id: E_v split into +#: her approvals A_v and disapprovals D_v (Section 2.3). +TVExposed = dict[str, tuple[set[Project], set[Project]]] + + +# --------------------------------------------------------------------------- +# Section 2.1.1 - Binary classification (one classifier per project). +# --------------------------------------------------------------------------- +#: Hyperparameter settings scored against the validation set (Section 5). The +#: first entry is what gets fitted when tuning is switched off (``param_grid=()``). +#: +#: ``class_weight`` is deliberately *not* varied here; see :py:func:`_fit_per_project` +#: for why the minority-class correction of Section 5 is off by default. +DEFAULT_PARAM_GRID = ( + {"n_estimators": 100, "max_depth": 3, "learning_rate": 0.1}, + {"n_estimators": 200, "max_depth": 3, "learning_rate": 0.05}, + {"n_estimators": 200, "max_depth": 6, "learning_rate": 0.1}, +) + + +def _xgboost(): + """ + The lazily imported ``xgboost`` module (an optional dependency). + + Examples + -------- + >>> _xgboost().__name__ + 'xgboost' + """ + try: + import xgboost + except ImportError: + raise ImportError( + "You need to install xgboost to train the classification " + "predictor (pip install pabutools[recommendation])." + ) + return xgboost + + +def _masked_copies(lv_votes: np.ndarray, tv_votes: np.ndarray, seed: int): + """ + The LV rows to fit on: their full ballots, plus one blanked copy of each. + + At prediction time a classifier is asked about a project the voter was never + questioned on, from a row where nearly every input is missing. An LV row is + complete, and a Target Voter only supplies a row for the projects she *was* + asked about - so for the projects that actually need predicting, the + classifiers would never meet a missing value during training and would send + every blank one fixed way. That is what made the offline setups, where every + voter is asked the same projects, return one identical ballot for everybody. + + Each LV voter is therefore added a second time with a real Target Voter's + pattern of unanswered projects blanked out, so the classifiers train on rows + shaped like the ones they will be asked about. Her labels stay correct + because the label column is dropped from the features anyway, so blanking it + cannot leak anything. + + Returns ``(features, labels)``: the labels are always the true full ballots, + only the features are blanked. + + Examples + -------- + One LV voter who answered all three projects, and one Target Voter who was + asked only about the first. The LV row is kept as it is and added a second + time with the two projects she would not have been asked about blanked out; + her labels stay the true ballot both times. + + >>> lv = np.array([[1.0, 0.0, 1.0]]) + >>> tv = np.array([[1.0, np.nan, np.nan]]) + >>> features, labels = _masked_copies(lv, tv, seed=0) + >>> features.tolist() + [[1.0, 0.0, 1.0], [1.0, nan, nan]] + >>> labels.tolist() + [[1.0, 0.0, 1.0], [1.0, 0.0, 1.0]] + + With no Target Voters there is nothing to imitate, so the rows are returned + untouched. + + >>> features, labels = _masked_copies(lv, np.zeros((0, 3)), seed=0) + >>> features.tolist() + [[1.0, 0.0, 1.0]] + """ + if not len(lv_votes) or not len(tv_votes): + return lv_votes, lv_votes + unasked = np.isnan(tv_votes) # True where that Target Voter was not asked + chosen = np.random.default_rng(seed).integers(0, len(tv_votes), len(lv_votes)) + masked = lv_votes.copy() + masked[unasked[chosen]] = np.nan + return np.vstack([lv_votes, masked]), np.vstack([lv_votes, lv_votes]) + + +def _project_rows( + index: int, lv_features: np.ndarray, lv_labels: np.ndarray, + tv_votes: np.ndarray, +): + """ + The ``(X, y)`` rows for the classifier of the project in column ``index``: + every LV row, plus the Target Voters who were asked about that project + (a voter without a preference on it has no label to contribute). The target + project's own column is dropped from ``X`` - it is the label, not an input. + + Examples + -------- + Two projects and two LV voters, plus one Target Voter asked only about + project 0. Building project 0's classifier leaves project 1 as the sole + input, and all three voters supply a label for project 0. + + >>> lv = np.array([[1.0, 0.0], [0.0, 1.0]]) + >>> tv = np.array([[1.0, np.nan]]) + >>> X, y = _project_rows(0, lv, lv, tv) + >>> X.ravel().tolist() + [0.0, 1.0, nan] + >>> y.tolist() + [1.0, 0.0, 1.0] + + For project 1 the Target Voter has no preference to contribute, so only the + two LV voters appear. + + >>> X, y = _project_rows(1, lv, lv, tv) + >>> y.tolist() + [0.0, 1.0] + """ + answered = ~np.isnan(tv_votes[:, index]) + y = np.concatenate([lv_labels[:, index], tv_votes[answered, index]]) + X = np.vstack([ + np.delete(lv_features, index, axis=1), + np.delete(tv_votes[answered], index, axis=1), + ]) + return X, y + + +def _fit_per_project( + projects: list[Project], lv_features: np.ndarray, lv_labels: np.ndarray, + tv_votes: np.ndarray, params: dict, +) -> dict[Project, tuple]: + """ + Fit one classifier per project on the given collected votes. A project whose + collected votes all agree (or that nobody voted on) needs no classifier, so + its common/majority preference is stored directly. + + Examples + -------- + Two voters who agree exactly: everyone approves p1 and rejects p2. Neither + project needs a classifier, so each is stored as the constant both voters + gave, and ``xgboost`` is never even imported. + + >>> p1, p2 = Project("p1", 1), Project("p2", 1) + >>> votes = np.array([[1.0, 0.0], [1.0, 0.0]]) + >>> fitted = _fit_per_project([p1, p2], votes, votes, np.zeros((0, 2)), {}) + >>> fitted[p1], fitted[p2] + (('const', 1), ('const', 0)) + """ + per_project: dict[Project, tuple] = {} + for index, project in enumerate(projects): + X, y = _project_rows(index, lv_features, lv_labels, tv_votes) + # An empty y has fewer than 2 distinct labels too, so it is covered. + if X.shape[1] == 0 or len(set(y.tolist())) < 2: + per_project[project] = ( + "const", int(len(y) > 0 and 2 * int(y.sum()) >= len(y)) + ) + continue + settings = dict(params) + # Section 5 addresses the ~10% approval rate of Table 1 "by modifying + # the model loss function to give more weight to minority class + # samples", which for XGBoost is ``scale_pos_weight`` (the weighted-loss + # approach of the paper's reference [26]). ``class_weight`` is the + # exponent applied to the class ratio: 0 leaves the loss alone, 1 + # balances the two classes completely, 0.5 is halfway. + # + # It defaults to 0, i.e. no correction, which is a deliberate departure + # from Section 5. Measured on three real Warsaw districts, the full + # correction collapses the very metric the paper reports: + # + # Praga-Poludnie 2022, offline_popularity: FA 0.802 -> 0.007 + # Mokotow 2022, offline_popularity: FA 0.869 -> 0.114 + # Wola 2021, offline_popularity: FA 0.731 -> 0.082 + # + # The cause is an interaction with the offline setups, where every + # Target Voter is asked about the *same* k projects and so arrives with + # an identical pattern of missing inputs. A balanced loss then makes the + # classifiers approve most of the ballot for everyone alike, the + # approval scores flatten, and the ranking greedy approval needs is + # lost. (Under the ``random`` setup, where exposures differ per voter, + # the correction is harmless and occasionally helps.) Raise it if you + # care about per-voter recall rather than the winning bundle: recall + # goes 0.078 -> 0.773 across that same range, while precision stays + # near 0.08. + exponent = settings.pop("class_weight", 0.0) + positives = int(y.sum()) + negatives = len(y) - positives + ratio = (negatives / positives) if positives else 1.0 + classifier = _xgboost().XGBClassifier( + verbosity=0, + scale_pos_weight=ratio ** exponent, + **settings, + ) + classifier.fit(X, y) + per_project[project] = ("model", ([p for p in projects if p != project], + classifier)) + return per_project + + +def _pooled_f1( + per_project: dict[Project, tuple], projects: list[Project], + lv_features: np.ndarray, lv_labels: np.ndarray, tv_votes: np.ndarray, +) -> float: + """ + F1 of a fitted bundle over every held-out vote, pooled across projects. + Section 5.1 calls F1 the "suitable measure for an imbalanced data", so it is + the criterion the validation set selects hyperparameters on. + + Examples + -------- + One held-out voter who approves p1 and rejects p2, against a bundle that + predicts exactly that: no false positives and no misses, so F1 is 1. + + >>> p1, p2 = Project("p1", 1), Project("p2", 1) + >>> votes = np.array([[1.0, 0.0]]) + >>> bundle = {p1: ("const", 1), p2: ("const", 0)} + >>> _pooled_f1(bundle, [p1, p2], votes, votes, np.zeros((0, 2))) + 1.0 + + A bundle that approves both projects finds the one real approval but also + raises a false alarm on p2: precision 0.5, recall 1.0. + + >>> _pooled_f1({p1: ("const", 1), p2: ("const", 1)}, + ... [p1, p2], votes, votes, np.zeros((0, 2))) + 0.6666666666666666 + """ + tp = fp = fn = 0 + for index, project in enumerate(projects): + X, y = _project_rows(index, lv_features, lv_labels, tv_votes) + if len(y) == 0: + continue + kind, payload = per_project[project] + if kind == "const": + predicted = np.full(len(y), float(payload)) + else: + predicted = payload[1].predict(X) + tp += int(((y == 1) & (predicted == 1)).sum()) + fp += int(((y == 0) & (predicted == 1)).sum()) + fn += int(((y == 1) & (predicted == 0)).sum()) + return _f1(tp, fp, fn) + + +def _f1(tp: int, fp: int, fn: int) -> float: + """ + F1 from a confusion matrix (Section 5.1), 0.0 when it is undefined. + + Examples + -------- + >>> _f1(3, 1, 1) # precision 0.75, recall 0.75 + 0.75 + >>> _f1(0, 5, 5) # nothing found: precision and recall both 0 + 0.0 + >>> _f1(0, 0, 0) # undefined; 0.0 by the usual convention + 0.0 + """ + precision = tp / (tp + fp) if tp + fp else 0.0 + recall = tp / (tp + fn) if tp + fn else 0.0 + return 2 * precision * recall / (precision + recall) if precision + recall else 0.0 + + +def train_classification( + instance: Instance, + lv_profile: ApprovalProfile, + tv_ballots: dict[str, CardinalBallot] | None = None, + *, + validation_fraction: float = 0.15, + param_grid=DEFAULT_PARAM_GRID, + seed: int = 0, +) -> dict: + """ + Train the per-project binary classifiers of Section 2.1.1. For every project + p, one classifier uses preferences on P \\ {p} as input and the preference on + p as its 0/1 output. The resulting model bundle is trained once and reused + for every Target Voter; preferences the voter was not asked about are passed + to XGBoost as missing values. + + The training data is every vote the process *collected*, which Section 3.1.1 + defines as the preferences of the LV voters **and** the exposed preferences + E_TV of the Target Voters: each LV voter contributes a row to every + project's classifier, and a Target Voter contributes a row for project p + exactly when p lies in her exposed set E_v (otherwise there is no label to + learn from). Those rows carry NaN for the projects she was not asked about, + which is what teaches XGBoost a split direction for missing inputs - the + same missingness the rows to be predicted will carry. + + Following Section 5, the collected votes are split three ways rather than + used wholesale: ``validation_fraction`` of the *voters* are held back as the + validation set ("a predefined set of votes from closed set of voters"), each + setting in ``param_grid`` is fitted on the rest and scored on them by pooled + F1, and the winning setting is then refitted on all collected votes. (The + Target Voters' hidden votes are the paper's test set and are never touched + here.) One setting is chosen for the whole bundle, not per project, matching + the paper's singular "the hyperparameters of a model". + + Backed by the external ``xgboost`` library + (:py:class:`xgboost.XGBClassifier`, class-weighted for the imbalanced data), + imported lazily. When no collected vote covers a project, or they all agree + on it, no classifier is needed, so we use that common/majority preference + directly. + + Parameters + ---------- + instance : :py:class:`~pabutools.election.instance.Instance` + The PB instance. + lv_profile : :py:class:`~pabutools.election.profile.approvalprofile.ApprovalProfile` + The full ballots of the LV voters, used as the training data. + tv_ballots : dict[str, :py:class:`~pabutools.election.ballot.cardinalballot.CardinalBallot`], optional + The Target Voters' partial ballots, keyed by voter id. Their exposed + votes E_TV join the training data; their hidden votes are never + looked at. + validation_fraction : float, optional + Share of the voters held out to tune the hyperparameters on (default + 0.15, the paper's figure). Set to 0 to skip the search. + param_grid : Iterable[dict], optional + The settings to score, defaulting to :py:data:`DEFAULT_PARAM_GRID`. + Tuning multiplies the number of fits by ``len(param_grid)``, so pass + a single setting - or ``()`` - for large sweeps. + seed : int, optional + Seed for the validation split, for reproducibility. + Returns + ------- + dict + ``{"per_project": {project: ("const", 0/1) or + ("model", (input_projects, classifier))}, "params": the + hyperparameters used}``, consumed by + :py:func:`predict_by_classification`. + + Examples + -------- + With unanimous LV approvals, every project falls back to the majority + preference (1 for approval, 0 for disapproval). + + >>> p1, p2 = Project("p1", 1), Project("p2", 1) + >>> inst = Instance([p1, p2], budget_limit=2) + >>> lv = ApprovalProfile([ApprovalBallot([p1])] * 3) + >>> model = train_classification(inst, lv) + >>> model["per_project"][p1], model["per_project"][p2] + (('const', 1), ('const', 0)) + + A Target Voter's exposed votes join the training data. On the LV ballots + alone the lone voter's rejection of p2 is unanimous, so p2 needs no + classifier; once three Target Voters expose an approval of it the collected + votes disagree and a classifier is fitted after all. + + >>> lv = ApprovalProfile([ApprovalBallot([p1])]) + >>> train_classification(inst, lv)["per_project"][p2][0] + 'const' + >>> tv = {f"v{i}": CardinalBallot({p2: 1}) for i in range(3)} + >>> train_classification(inst, lv, tv)["per_project"][p2][0] + 'model' + """ + projects = sorted(instance, key=str) + lv = list(lv_profile) + tv = list((tv_ballots or {}).values()) + # The collected votes as two voter x project tables. An LV voter answered + # every project, so her row is dense; a Target Voter's row is NaN wherever + # she was not asked, and NaN is XGBoost's own missing-value marker. + def table(rows): # the reshape keeps the (0, m) shape when nobody is in it + return np.array(rows, dtype=float).reshape(len(rows), len(projects)) + + tv_sets = [_preference_sets(instance, ballot) for ballot in tv] + lv_votes = table([[float(p in ballot) for p in projects] for ballot in lv]) + tv_votes = table([ + [1.0 if p in approved else 0.0 if p in disapproved else np.nan + for p in projects] + for approved, disapproved, _ in tv_sets + ]) + + # Blanked copies of the LV ballots, so the classifiers see the sparse rows + # they will actually be asked about (see :py:func:`_masked_copies`). + lv_features, lv_labels = _masked_copies(lv_votes, tv_votes, seed) + copies = len(lv_features) // len(lv) if len(lv) else 1 + + param_grid = tuple(param_grid) + params = dict(param_grid[0]) if param_grid else dict(DEFAULT_PARAM_GRID[0]) + # Section 5's validation set: whole voters, so that no voter has some of her + # votes fitted on and others scored. Voters are numbered LV first, then TV. + n_voters = len(lv) + len(tv) + n_validation = int(round(validation_fraction * n_voters)) + if len(param_grid) > 1 and 0 < n_validation < n_voters: + held_out = np.random.default_rng(seed).permutation(n_voters)[:n_validation] + is_validation = np.zeros(n_voters, dtype=bool) + is_validation[held_out] = True + lv_held, tv_held = is_validation[:len(lv)], is_validation[len(lv):] + # Every LV voter now owns ``copies`` rows, in [full, blanked] order, so + # her copies must fall on the same side of the split as she does. + rows_held = np.tile(lv_held, copies) + best_f1 = -1.0 + for candidate in param_grid: + trained = _fit_per_project( + projects, lv_features[~rows_held], lv_labels[~rows_held], + tv_votes[~tv_held], candidate, + ) + score = _pooled_f1( + trained, projects, lv_features[rows_held], lv_labels[rows_held], + tv_votes[tv_held], + ) + logger.debug("train_classification: %s -> validation F1 %.4f", + candidate, score) + if score > best_f1: + params, best_f1 = dict(candidate), score + logger.info( + "train_classification: tuned on %d of %d held-out voters, " + "best pooled F1 %.4f with %s", + n_validation, n_voters, best_f1, params, + ) + per_project = _fit_per_project( + projects, lv_features, lv_labels, tv_votes, params + ) + logger.info( + "train_classification: trained %d per-project classifiers once on " + "%d LV ballots (x%d, blanked) and the exposed votes of %d TV voters", + len(per_project), len(lv), copies, len(tv), + ) + return {"per_project": per_project, "params": params} + + +# --------------------------------------------------------------------------- +# Section 2.1.2 - Collaborative filtering. +# --------------------------------------------------------------------------- +def _known_preferences( + instance: Instance, + lv_profile: ApprovalProfile, + tv_exposed: TVExposed, +) -> tuple[list[Project], list[tuple[tuple[str, object], Project, int]]]: + """ + Return the known preferences: full LV ballots plus exposed TV sets E_v. + + Examples + -------- + The LV voter contributes a vote on every project; the Target Voter only on + the one she was asked about. Voters are keyed ``("lv", index)`` and + ``("tv", voter id)`` so the two groups cannot collide. + + >>> p1, p2 = Project("p1", 1), Project("p2", 1) + >>> inst = Instance([p1, p2], budget_limit=2) + >>> lv = ApprovalProfile([ApprovalBallot([p1])]) + >>> projects, known = _known_preferences(inst, lv, {"v": ({p2}, set())}) + >>> known + [(('lv', 0), p1, 1), (('lv', 0), p2, 0), (('tv', 'v'), p2, 1)] + """ + projects = sorted(instance, key=str) + preferences = [ + (("lv", i), p, int(p in ballot)) + for i, ballot in enumerate(lv_profile) + for p in projects + ] + preferences += [ + (("tv", voter), p, int(p in approved)) + for voter, (approved, disapproved) in tv_exposed.items() + for p in sorted(approved | disapproved, key=str) + ] + return projects, preferences + + +#: Latent ranks scored against the validation set for matrix factorization. +DEFAULT_MF_RANKS = (5, 20, 50) +#: ``(rank, ridge)`` settings scored against the validation set for the hybrid. +DEFAULT_FM_SETTINGS = ((5, 0.0), (20, 0.0), (20, 1e-5), (50, 1e-5)) + + +def _minority_weight(preferences) -> int: + """ + How many times an approval must count for the two classes to weigh the same. + Section 5 addresses the ~10% approval rate of Table 1 "by modifying the model + loss function to give more weight to minority class samples"; with roughly + one approval per nine rejections this returns 9. + + Examples + -------- + >>> _minority_weight([("v", "p", 1)] + [("v", "p", 0)] * 9) + 9 + >>> _minority_weight([("v", "p", 1)] * 5 + [("v", "p", 0)] * 5) + 1 + >>> _minority_weight([]) # nothing collected + 1 + """ + positives = sum(label for *_, label in preferences) + negatives = len(preferences) - positives + return max(1, round(negatives / positives)) if positives else 1 + + +def _tune(preferences, settings, fit, validation_fraction: float, seed: int): + """ + Pick the setting with the best validation F1 (Sections 5 and 5.1). + + A share of the collected *votes* is held back, rather than a share of the + voters as in :py:func:`train_classification`: a collaborative-filtering model + cannot score a voter it never saw, so holding out whole voters would leave + nothing to validate on. ``fit(votes, setting)`` returns a scorer taking a + voter and a list of projects and returning one score in [0, 1] each. + + Examples + -------- + Twenty collected votes, half of them approvals, and two candidate settings + standing in for a real model: one that approves everything and one that + approves nothing. The first finds every approval, the second finds none, so + the F1 comparison picks the first. + + >>> votes = [("v", f"p{i}", i % 2) for i in range(20)] + >>> def fit(training, approve_everything): + ... return lambda voter, projects: [ + ... 1.0 if approve_everything else 0.0 for _ in projects + ... ] + >>> _tune(votes, (False, True), fit, validation_fraction=0.5, seed=0) + True + + With a single setting there is nothing to choose between, so it is returned + without fitting anything. + + >>> _tune(votes, (False,), fit, validation_fraction=0.5, seed=0) + False + """ + settings = tuple(settings) + n_validation = int(round(validation_fraction * len(preferences))) + if len(settings) < 2 or not 0 < n_validation < len(preferences): + return settings[0] + order = np.random.default_rng(seed).permutation(len(preferences)) + held_out = set(order[:n_validation].tolist()) + training = [pref for i, pref in enumerate(preferences) if i not in held_out] + validation: dict = {} + for index in sorted(held_out): + voter, project, label = preferences[index] + validation.setdefault(voter, []).append((project, label)) + + best, best_f1 = settings[0], -1.0 + for setting in settings: + score = fit(training, setting) + tp = fp = fn = 0 + for voter, votes in validation.items(): + values = score(voter, [project for project, _ in votes]) + for (_, label), value in zip(votes, values): + if value >= 0.5 and label: + tp += 1 + elif value >= 0.5: + fp += 1 + elif label: + fn += 1 + f1 = _f1(tp, fp, fn) + logger.debug("_tune: %s -> validation F1 %.4f", setting, f1) + if f1 > best_f1: + best, best_f1 = setting, f1 + logger.info( + "_tune: %d of %d collected votes held out, best F1 %.4f with %s", + n_validation, len(preferences), best_f1, best, + ) + return best + + +def _fit_mf(preferences, rank: int, weight: int): + """ + Fit one SVD and return ``score(voter, projects)``. + + Examples + -------- + One voter who approved p1 and rejected p2; the fitted model scores any + (voter, project) pair on the 0-1 scale the preferences were given on. + + >>> p1, p2 = Project("p1", 1), Project("p2", 1) + >>> score = _fit_mf([(("lv", 0), p1, 1), (("lv", 0), p2, 0)], 2, 1) + >>> values = score(("lv", 0), [p1, p2]) + >>> len(values) == 2 and all(0.0 <= v <= 1.0 for v in values) + True + """ + import pandas as pd + from surprise import Dataset, Reader, SVD + + # Surprise exposes no sample-weight API, so an approval is repeated + # ``weight`` times instead: that many more gradient steps is the same thing + # as that much more loss weight, and it lifts the reconstructed scores of + # the minority class towards the 0.5 threshold they are compared against. + rows = [pref for pref in preferences for _ in range(weight if pref[2] else 1)] + data = Dataset.load_from_df( + pd.DataFrame(rows, columns=["voter", "project", "preference"]), + Reader(rating_scale=(0, 1)), # Surprise's name for 0/1 preferences. + ) + model = SVD(n_factors=max(1, rank), n_epochs=50, random_state=0) + model.fit(data.build_full_trainset()) + return lambda voter, projects: [ + float(model.predict(voter, p).est) for p in projects + ] + + +def train_matrix_factorization( + instance: Instance, + lv_profile: ApprovalProfile, + tv_exposed: TVExposed, + rank: int | None = None, + *, + ranks=DEFAULT_MF_RANKS, + validation_fraction: float = 0.15, + seed: int = 0, +) -> dict[str, dict[Project, float]]: + """ + Fit ``R_vp ~= mu + b_v + b_p + q_p^T x_v`` (paper, Section 2.1.2). + + Approvals are up-weighted against the imbalance of Table 1, and the latent + rank is chosen by validation F1 over ``ranks`` (Section 5) unless an explicit + ``rank`` is given, in which case that one is used and nothing is tuned. + + Examples + -------- + Three LV voters who all approve p1 and reject p2, and one Target Voter whose + exposed votes agree with them. Every project gets a reconstructed score on + the 0-1 scale, and the pair the electorate likes scores above the pair it + does not. + + >>> p1, p2 = Project("p1", 1), Project("p2", 1) + >>> inst = Instance([p1, p2], budget_limit=2) + >>> lv = ApprovalProfile([ApprovalBallot([p1])] * 3) + >>> scores = train_matrix_factorization(inst, lv, {"v": ({p1}, {p2})}, rank=2) + >>> sorted(scores) == ["v"] and len(scores["v"]) == 2 + True + >>> scores["v"][p1] > scores["v"][p2] + True + """ + try: + import pandas # noqa: F401 (needed by _fit_mf) + import surprise # noqa: F401 + except ImportError: + raise ImportError( + "You need scikit-surprise and pandas to train the matrix-factorization " + "predictor (pip install pabutools[recommendation])." + ) + + projects, preferences = _known_preferences(instance, lv_profile, tv_exposed) + if not preferences: + return {v: {p: 0.0 for p in projects} for v in tv_exposed} + + weight = _minority_weight(preferences) + + def fit(votes, latent_rank): + """One candidate fit, in the shape :py:func:`_tune` expects.""" + return _fit_mf(votes, latent_rank, weight) + + if rank is None: + rank = _tune(preferences, ranks, fit, validation_fraction, seed) + logger.info( + "train_matrix_factorization: factorising %d collected votes over %d " + "projects at rank %d, approvals weighted x%d", + len(preferences), len(projects), rank, weight, + ) + score = fit(preferences, rank) + return { + v: dict(zip(projects, score(("tv", v), projects))) for v in tv_exposed + } + + +# --------------------------------------------------------------------------- +# Section 2.1.2 - Hybrid Factorization Machines. +# --------------------------------------------------------------------------- +def _fit_fm(projects, voters, preferences, rank: int, ridge: float, weight: int): + """ + Fit one LightFM hybrid and return ``score(voter, projects)``. + + Examples + -------- + Like :py:func:`_fit_mf`, but each project also carries Table 2's attributes. + The example is marked skipped so the doctest suite still passes without the + optional ``lightfm`` installed; the same path is covered by the unit tests, + which skip themselves when the library is absent. + + >>> p1, p2 = Project("p1", 1), Project("p2", 1) + >>> known = [(("lv", 0), p1, 1), (("lv", 0), p2, 0)] + >>> score = _fit_fm([p1, p2], [("lv", 0)], known, 2, 0.0, 1) # doctest: +SKIP + >>> [0.0 <= v <= 1.0 for v in score(("lv", 0), [p1, p2])] # doctest: +SKIP + [True, True] + """ + from lightfm import LightFM + from lightfm.data import Dataset + + # Table 2's project attributes. Every project also keeps its own identity + # feature (LightFM adds one by default), so these content features sit + # alongside the latent factor rather than replacing it - that combination is + # the hybrid of Section 2.1.2. + categories = sorted({c for p in projects for c in p.categories}) + targets = sorted({t for p in projects for t in p.targets}) + dataset = Dataset() + dataset.fit( + voters, + projects, + item_features=(["cost"] + + [f"category={c}" for c in categories] + + [f"target={t}" for t in targets]), + ) + preference_matrix, signs = dataset.build_interactions( + (v, p, 2 * preference - 1) for v, p, preference in preferences + ) + # Section 5's minority-class weighting. LightFM does honour sample weights + # under the logistic loss - only the k-OS loss ignores them - so an approval + # simply contributes ``weight`` times as much to the loss as a rejection. + sample_weight = signs.copy() + sample_weight.data = np.where( + signs.data > 0, float(weight), 1.0 + ).astype(np.float32) + preference_matrix.data = signs.data + costs = np.array([float(p.cost) for p in projects]) + costs = (costs - costs.mean()) / costs.std() if costs.std() else costs * 0 + project_attributes = dataset.build_item_features( + ( + (p, {"cost": cost} + | {f"category={c}": 1.0 for c in p.categories} + | {f"target={t}": 1.0 for t in p.targets}) + for p, cost in zip(projects, costs) + ), + normalize=False, + ) + model = LightFM( + no_components=max(1, rank), + loss="logistic", + user_alpha=ridge, + item_alpha=ridge, + random_state=0, + ) + model.fit( + preference_matrix, + item_features=project_attributes, + sample_weight=sample_weight, + epochs=30, + num_threads=1, + ) + user_ids, _, item_ids, _ = dataset.mapping() + + def score(voter, wanted): + """The fitted approval probability of each ``wanted`` project.""" + raw = model.predict( + user_ids[voter], + np.array([item_ids[p] for p in wanted]), + item_features=project_attributes, + ) + return 1 / (1 + np.exp(-raw)) + + return score + + +def train_factorization_machines( + instance: Instance, + lv_profile: ApprovalProfile, + tv_exposed: TVExposed, + rank: int | None = 20, + ridge: float | None = 0.0, + *, + settings=DEFAULT_FM_SETTINGS, + validation_fraction: float = 0.15, + seed: int = 0, +) -> dict[str, dict[Project, float]]: + """ + Fit the paper's hybrid latent model ``y(v,p)=FM(v, p, attributes(p))``. + + What makes this module *hybrid* (Section 2.1.2) rather than plain matrix + factorization is that projects are described by content features as well as + by their own latent factor, so a voter's taste can transfer to a project she + was never asked about. The features are Table 2's project attributes, all of + which ``parse_pabulib`` fills in: the numeric ``cost`` (standardised), plus + one multi-hot indicator per ``category`` and per ``target`` population + segment - multi-hot because Section 4 notes that "a certain proposal can + consist multiple topics and population segments". An instance whose projects + carry no categories or targets falls back to cost alone. + + Approvals are up-weighted against the imbalance of Table 1. ``rank`` and + ``ridge`` are *not* tuned by default, a deliberate departure from Section 5: + pass ``rank=None, ridge=None`` to search ``settings`` by validation F1 + instead. + + Measured on Praga-Poludnie 2022 with the offline-popularity setup, the + search costs accuracy rather than buying it - FA 0.787 at the fixed + ``rank=20, ridge=0.0`` against 0.590 for whatever the search selects - as + well as fitting the model four times over. The reason is that Section 5.1's + F1 is measured per vote while Section 5.2's FA depends on the *ranking* of + projects surviving, and the two do not agree; the same disagreement is why + :py:func:`_fit_per_project` leaves the minority-class correction off. Ridge + is no help either: sweeping it over ``0 .. 0.1`` moves offline FA around + erratically (0.787, 0.782, 0.346, 0.753, 0.007) without ever reaching the + 1.000 that plain matrix factorization gets on the same split. + + .. note:: + The content features are worth having under ``random`` sampling and + harmful under the offline setups, measured on that same district: + + ============================ ================== ======== + FM item features offline_popularity random + ============================ ================== ======== + none at all (i.e. plain MF) 1.000 0.927 + cost only 0.814 0.955 + cost + categories + targets 0.590 1.000 + ============================ ================== ======== + + The offline setups expose every voter to the *same*, deliberately + unrepresentative k projects (the most popular, or the most divisive), so + the shared content embeddings are fitted on that slice and then applied + to the projects nobody was asked about. Under ``random`` the exposed + projects are a fair sample and the features help. The paper treats the + sampling module and the prediction module as independent choices + (Section 3); this is evidence that they interact. + + Examples + -------- + The hybrid counterpart of :py:func:`train_matrix_factorization`: same + voter-project preferences, but every project also described by its cost, + categories and target segments. Skipped so the suite passes without the + optional ``lightfm``; the unit tests cover this path and skip themselves + when it is missing. + + >>> green = Project("green", 1, categories={"environment"}) + >>> road = Project("road", 2, categories={"transport"}) + >>> inst = Instance([green, road], budget_limit=3) + >>> lv = ApprovalProfile([ApprovalBallot([green])] * 3) + >>> scores = train_factorization_machines( # doctest: +SKIP + ... inst, lv, {"v": ({green}, {road})}) + >>> scores["v"][green] > scores["v"][road] # doctest: +SKIP + True + """ + try: + import lightfm # noqa: F401 (needed by _fit_fm) + except ImportError: + raise ImportError( + "You need lightfm to train the factorization-machines predictor " + "(pip install pabutools[recommendation])." + ) + + projects, preferences = _known_preferences(instance, lv_profile, tv_exposed) + voters = [("lv", i) for i in range(lv_profile.num_ballots())] + [ + ("tv", v) for v in tv_exposed + ] + if not projects or not preferences: + return {v: {p: 0.0 for p in projects} for v in tv_exposed} + + weight = _minority_weight(preferences) + + def fit(votes, setting): + """One candidate fit, in the shape :py:func:`_tune` expects.""" + return _fit_fm(projects, voters, votes, *setting, weight) + + if rank is None and ridge is None: + rank, ridge = _tune(preferences, settings, fit, validation_fraction, seed) + else: + rank, ridge = (20 if rank is None else rank, 0.0 if ridge is None else ridge) + logger.info( + "train_factorization_machines: fitting %d collected votes over %d " + "projects at rank %d, ridge %g, approvals weighted x%d, with %d " + "categories and %d target segments as content features", + len(preferences), len(projects), rank, ridge, weight, + len({c for p in projects for c in p.categories}), + len({t for p in projects for t in p.targets}), + ) + score = fit(preferences, (rank, ridge)) + return { + v: dict(zip(projects, score(("tv", v), projects))) for v in tv_exposed + } + + +# --------------------------------------------------------------------------- +# Complete the Target Voters' partial ballots with the fitted models. +# --------------------------------------------------------------------------- +def _preference_sets( + instance: Instance, ballot: CardinalBallot +) -> tuple[set[Project], set[Project], set[Project]]: + """ + Decode a partial ballot into A_v, D_v, and H_v. + + Examples + -------- + p1 is approved and p2 rejected; p3 was never scored, so it is hidden. + + >>> p1, p2, p3 = Project("p1", 1), Project("p2", 1), Project("p3", 1) + >>> inst = Instance([p1, p2, p3], budget_limit=3) + >>> a, d, h = _preference_sets(inst, CardinalBallot({p1: 1, p2: -1})) + >>> a == {p1}, d == {p2}, h == {p3} + (True, True, True) + """ + approved = {p for p, preference in ballot.items() if preference > 0} + disapproved = {p for p, preference in ballot.items() if preference < 0} + hidden = set(instance) - approved - disapproved + return approved, disapproved, hidden + + +def _complete_by_scores( + instance: Instance, lv_profile: ApprovalProfile, + tv_ballots: dict[str, CardinalBallot], train, +) -> dict[str, ApprovalBallot]: + """ + Complete every TV ballot from a collaborative-filtering score: ``train`` is + fitted on the LV ballots plus all the exposed sets E_TV, then a hidden + project is approved iff its score is >= 0.5, while A_v is kept and D_v stays + rejected. The MF and FM modules differ only in ``train``, so they share this. + + Examples + -------- + A stand-in model that scores p3 highly and everything else low, applied to a + voter who approved p1, rejected p2 and was never asked about p3. Her + approval of p1 survives, her rejection of p2 survives, and only the hidden + p3 is decided by the score. + + >>> p1, p2, p3 = Project("p1", 1), Project("p2", 1), Project("p3", 1) + >>> inst = Instance([p1, p2, p3], budget_limit=3) + >>> def train(instance, lv_profile, exposed): + ... return {v: {p: (0.9 if p == p3 else 0.1) for p in instance} + ... for v in exposed} + >>> done = _complete_by_scores( + ... inst, ApprovalProfile([]), {"v": CardinalBallot({p1: 1, p2: -1})}, train) + >>> done["v"] == ApprovalBallot({p1, p3}) + True + """ + sets = {v: _preference_sets(instance, b) for v, b in tv_ballots.items()} + scores = train(instance, lv_profile, + {v: (approved, disapproved) + for v, (approved, disapproved, _) in sets.items()}) + completed = { + v: ApprovalBallot(approved | {p for p in hidden if scores[v][p] >= 0.5}) + for v, (approved, _, hidden) in sets.items() + } + logger.info( + "%s: completed %d TV ballots, %d of %d hidden votes scored >= 0.5 " + "and became approvals", + getattr(train, "__name__", "collaborative filtering"), len(completed), + sum(len(completed[v]) - len(sets[v][0]) for v in sets), + sum(len(sets[v][2]) for v in sets), + ) + return completed + + +def predict_by_classification( + instance: Instance, + lv_profile: ApprovalProfile, + tv_ballots: dict[str, CardinalBallot], + **kwargs, +) -> dict[str, ApprovalBallot]: + """ + Prediction module - binary classification (Section 2.1.1): predict each + hidden project with a per-project binary classifier. One classifier is + trained for every project p, using preferences on P \\ {p} as input and the + preference on p as output. The collection is trained once by + :py:func:`train_classification` - on the LV ballots *and* the Target Voters' + exposed votes E_TV, per Section 3.1.1 - and applied to every TV voter; + preferences outside E_v are passed to XGBoost as missing values. Exposed + approvals A_v are kept and exposed disapprovals D_v stay rejected. + + Parameters + ---------- + instance : :py:class:`~pabutools.election.instance.Instance` + The PB instance. + lv_profile : :py:class:`~pabutools.election.profile.approvalprofile.ApprovalProfile` + The full ballots of the LV voters, used as the training data. + tv_ballots : dict[str, :py:class:`~pabutools.election.ballot.cardinalballot.CardinalBallot`] + Every Target Voter's partial ballot to complete, keyed by voter id. + Build these with ``pabutools.recommendation.partial_ballot`` / + ``reveal_ballot`` rather than by hand; the score convention is + +1 approved, -1 disapproved, 0 (or absent) hidden. + **kwargs + Forwarded to :py:func:`train_classification`. Pass ``param_grid=()`` + to skip the Section 5 hyperparameter search, which costs roughly ten + times a plain fit and is worth switching off for large sweeps. + + Returns + ------- + dict[str, :py:class:`~pabutools.election.ballot.approvalballot.ApprovalBallot`] + The full predicted approval ballot of every TV voter, keyed by + voter id. + + Examples + -------- + 2 LV voters both approve {p1, p2}. A TV voter with nothing exposed is + completed to {p1, p2} - on such an unambiguous pattern any correct + predictor (XGBoost included) agrees with the LV majority. + + >>> p1, p2, p3 = Project("p1", 4), Project("p2", 4), Project("p3", 6) + >>> inst = Instance([p1, p2, p3], budget_limit=6) + >>> lv = ApprovalProfile([ApprovalBallot([p1, p2]), ApprovalBallot([p1, p2])]) + >>> partial = CardinalBallot({p1: 0, p2: 0, p3: 0}) # nothing exposed + >>> predict_by_classification(inst, lv, {"v3": partial})["v3"] == {p1, p2} + True + """ + model = train_classification(instance, lv_profile, tv_ballots, **kwargs) + projects = sorted(instance, key=str) + voters = list(tv_ballots) + preferences = { + vid: _preference_sets(instance, tv_ballots[vid]) for vid in voters + } + # One row per Target Voter over every project, in the same column order the + # classifiers were fitted on: 1 approved, 0 disapproved, NaN not asked. + votes = np.array( + [ + [ + 1.0 if p in approved else 0.0 if p in disapproved else np.nan + for p in projects + ] + for approved, disapproved, _ in (preferences[v] for v in voters) + ], + dtype=float, + ).reshape(len(voters), len(projects)) + + # Classify per *project*, not per (voter, project): every voter with that + # project hidden goes through XGBoost in one call. On a Warsaw district this + # is ~100 calls rather than ~700 000. + predicted: dict[str, set[Project]] = {vid: set() for vid in voters} + for index, project in enumerate(projects): + rows = np.flatnonzero(np.isnan(votes[:, index])) # hidden for these + if not len(rows): + continue + kind, payload = model["per_project"][project] + if kind == "const": + labels = np.full(len(rows), payload) + else: + # Dropping this project's column reproduces the ``input_projects`` + # order the classifier was trained on. + labels = payload[1].predict(np.delete(votes[rows], index, axis=1)) + for row, label in zip(rows, labels): + if int(label) == 1: + predicted[voters[row]].add(project) + logger.info( + "predict_by_classification: completed %d TV ballots in %d batched calls", + len(voters), len(projects), + ) + return { + vid: ApprovalBallot(preferences[vid][0] | predicted[vid]) + for vid in voters + } + + +def predict_by_matrix_factorization( + instance: Instance, + lv_profile: ApprovalProfile, + tv_ballots: dict[str, CardinalBallot], +) -> dict[str, ApprovalBallot]: + """ + Prediction module - collaborative filtering via Matrix Factorization + (Section 2.1.2): one voter-project matrix holds the LV ballots and *every* TV + voter's exposed votes (the paper's E_TV; approve=1, disapprove=0), it is + factorised once by :py:func:`train_matrix_factorization`, and a hidden + project is predicted approved iff its reconstructed score is >= 0.5. + Approvals are up-weighted against the imbalance of Table 1 and the latent + rank is chosen on a 15% validation set (Section 5). Exposed approvals A_v + are kept and exposed disapprovals D_v stay rejected. + + Parameters + ---------- + instance : :py:class:`~pabutools.election.instance.Instance` + The PB instance. + lv_profile : :py:class:`~pabutools.election.profile.approvalprofile.ApprovalProfile` + The full ballots of the LV voters, used as the training data. + tv_ballots : dict[str, :py:class:`~pabutools.election.ballot.cardinalballot.CardinalBallot`] + Every Target Voter's partial ballot to complete, keyed by voter id. + + Returns + ------- + dict[str, :py:class:`~pabutools.election.ballot.approvalballot.ApprovalBallot`] + The full predicted approval ballot of every TV voter, keyed by + voter id. + + Examples + -------- + Three LV voters all support the cheap pair {p1, p2} and reject {p3, p4}. + The reconstructed matrix completes a TV voter with nothing exposed to + {p1, p2}. + + >>> p1, p2, p3, p4 = (Project("p1", 3), Project("p2", 3), + ... Project("p3", 4), Project("p4", 4)) + >>> inst = Instance([p1, p2, p3, p4], budget_limit=6) + >>> lv = ApprovalProfile([ApprovalBallot([p1, p2])] * 3) + >>> partial = CardinalBallot({p1: 0, p2: 0, p3: 0, p4: 0}) # nothing exposed + >>> predict_by_matrix_factorization(inst, lv, {"v4": partial})["v4"] == {p1, p2} + True + """ + return _complete_by_scores( + instance, lv_profile, tv_ballots, train_matrix_factorization + ) + + +def predict_by_factorization_machines( + instance: Instance, + lv_profile: ApprovalProfile, + tv_ballots: dict[str, CardinalBallot], +) -> dict[str, ApprovalBallot]: + """ + Prediction module - hybrid Factorization Machines (Section 2.1.2): fitted by + :py:func:`train_factorization_machines` on the same LV + E_TV voter-project + preferences as Matrix Factorization, but with each project also described by + Table 2's attributes - its cost, its categories and its target population + segments - which is what makes the module hybrid rather than plain MF. + Approvals are up-weighted against the imbalance of Table 1 and the latent + rank and regularisation are chosen on a 15% validation set (Section 5). A + hidden project is predicted approved iff the FM score is >= 0.5; exposed + approvals A_v are kept and exposed disapprovals D_v stay rejected. + + Parameters + ---------- + instance : :py:class:`~pabutools.election.instance.Instance` + The PB instance. + lv_profile : :py:class:`~pabutools.election.profile.approvalprofile.ApprovalProfile` + The full ballots of the LV voters, used as the training data. + tv_ballots : dict[str, :py:class:`~pabutools.election.ballot.cardinalballot.CardinalBallot`] + Every Target Voter's partial ballot to complete, keyed by voter id. + + Returns + ------- + dict[str, :py:class:`~pabutools.election.ballot.approvalballot.ApprovalBallot`] + The full predicted approval ballot of every TV voter, keyed by + voter id. + + Examples + -------- + 2 LV voters both approve {p1, p2}. A TV voter with nothing exposed is + completed to {p1, p2}. Skipped so the doctest suite passes without the + optional ``lightfm``; the unit tests cover this and skip themselves when the + library is absent. + + >>> p1, p2, p3 = Project("p1", 4), Project("p2", 4), Project("p3", 6) + >>> inst = Instance([p1, p2, p3], budget_limit=6) + >>> lv = ApprovalProfile([ApprovalBallot([p1, p2]), ApprovalBallot([p1, p2])]) + >>> partial = CardinalBallot({p1: 0, p2: 0, p3: 0}) # nothing exposed + >>> predict_by_factorization_machines( # doctest: +SKIP + ... inst, lv, {"v3": partial})["v3"] == {p1, p2} + True + """ + return _complete_by_scores( + instance, lv_profile, tv_ballots, train_factorization_machines + ) + + +if __name__ == "__main__": + import doctest + + doctest.testmod(verbose=True) diff --git a/pb_recommendation.py b/pabutools/recommendation/recommendation.py similarity index 53% rename from pb_recommendation.py rename to pabutools/recommendation/recommendation.py index a176aa2..bd0dabe 100644 --- a/pb_recommendation.py +++ b/pabutools/recommendation/recommendation.py @@ -11,9 +11,7 @@ import logging import random -from collections.abc import Iterable - -import numpy as np +from collections.abc import Callable, Iterable from pabutools.election import ( Instance, @@ -26,14 +24,13 @@ from pabutools.election.satisfaction import Cost_Sat from pabutools.rules import BudgetAllocation, greedy_utilitarian_welfare -# Model training lives in a separate module, ``pb_model_training``. The -# learning-based predictors below delegate the fitting to its ``train_*`` -# functions; ``pb_model_training`` is ballot-agnostic (it takes plain project -# sets), so the dependency is one-directional and there is no import cycle. -from pb_model_training import ( - train_classification, - train_matrix_factorization, - train_factorization_machines, +# All model fitting and prediction lives in ``model_training``; the three +# Section 2.1 prediction modules are used here as the ``predict`` argument of +# the pipeline and are listed in ``PREDICTORS``. +from pabutools.recommendation.model_training import ( + predict_by_classification, + predict_by_matrix_factorization, + predict_by_factorization_machines, ) # Log the steps of every algorithm at INFO/DEBUG level. @@ -143,17 +140,46 @@ def reveal_ballot( def approved_projects(ballot: CardinalBallot) -> set[Project]: - """The approval set A_v: the projects with a strictly positive score.""" + """ + The approval set A_v: the projects with a strictly positive score. + + Examples + -------- + >>> p1, p2, p3 = Project("p1", 1), Project("p2", 1), Project("p3", 1) + >>> b = partial_ballot(approved={p1}, disapproved={p2}, hidden={p3}) + >>> approved_projects(b) == {p1} + True + """ return {project for project, score in ballot.items() if score > 0} def disapproved_projects(ballot: CardinalBallot) -> set[Project]: - """The disapproval set D_v: the projects with a strictly negative score.""" + """ + The disapproval set D_v: the projects with a strictly negative score. + + Examples + -------- + >>> p1, p2, p3 = Project("p1", 1), Project("p2", 1), Project("p3", 1) + >>> b = partial_ballot(approved={p1}, disapproved={p2}, hidden={p3}) + >>> disapproved_projects(b) == {p2} + True + """ return {project for project, score in ballot.items() if score < 0} def exposed_projects(ballot: CardinalBallot) -> set[Project]: - """The exposed set E_v = A_v ∪ D_v: the projects with a non-zero score.""" + """ + The exposed set E_v = A_v ∪ D_v: the projects with a non-zero score. + + Examples + -------- + The hidden p3 is left out, whichever way the other two went. + + >>> p1, p2, p3 = Project("p1", 1), Project("p2", 1), Project("p3", 1) + >>> b = partial_ballot(approved={p1}, disapproved={p2}, hidden={p3}) + >>> exposed_projects(b) == {p1, p2} + True + """ return {project for project, score in ballot.items() if score != 0} @@ -161,6 +187,18 @@ def hidden_projects(ballot: CardinalBallot, instance: Instance) -> set[Project]: """ The hidden set H_v = P \\ E_v: every project of the instance that the voter was not asked about (score 0 or absent from the ballot). + + Examples + -------- + p3 scores 0 and p4 is absent from the ballot altogether; both count as + hidden, so the voter is treated as never having been asked about either. + + >>> p1, p2, p3, p4 = (Project("p1", 1), Project("p2", 1), + ... Project("p3", 1), Project("p4", 1)) + >>> inst = Instance([p1, p2, p3, p4], budget_limit=4) + >>> b = partial_ballot(approved={p1}, disapproved={p2}, hidden={p3}) + >>> hidden_projects(b, inst) == {p3, p4} + True """ return set(instance) - exposed_projects(ballot) @@ -169,6 +207,13 @@ def as_approval_ballot(ballot: CardinalBallot) -> ApprovalBallot: """ The pabutools approval ballot made of the approved projects A_v, so that a (completed) partial ballot can be fed to approval-based voting rules. + + Examples + -------- + >>> p1, p2, p3 = Project("p1", 1), Project("p2", 1), Project("p3", 1) + >>> b = partial_ballot(approved={p1}, disapproved={p2}, hidden={p3}) + >>> as_approval_ballot(b) == ApprovalBallot({p1}) + True """ return ApprovalBallot(approved_projects(ballot)) @@ -201,8 +246,8 @@ def consensus_levels( Examples -------- - Example 7 from the paper (4 LV voters): p1 approved 4/0, p2 split 2/2, - p3 rejected 0/4. Both p1 and p3 are in full consensus, p2 in none. + Four LV voters: p1 approved 4/0, p2 split 2/2, p3 rejected 0/4. Both p1 + and p3 are in full consensus, p2 in none. >>> p1, p2, p3 = Project("p1", 1), Project("p2", 1), Project("p3", 1) >>> inst = Instance([p1, p2, p3], budget_limit=3) @@ -244,7 +289,14 @@ def most_consensual_projects( [p1, p3, p2] """ consensus = consensus_levels(instance, profile) - return sorted(instance, key=lambda project: (-consensus[project], str(project))) + gamma = sorted(instance, key=lambda project: (-consensus[project], str(project))) + if gamma: + logger.debug( + "most_consensual_projects: gamma_1=%s (consensus %d), " + "gamma_m=%s (consensus %d, the most controversial)", + gamma[0], consensus[gamma[0]], gamma[-1], consensus[gamma[-1]], + ) + return gamma # --------------------------------------------------------------------------- @@ -283,7 +335,7 @@ def greedy_approval( Examples -------- - Example 1 from the paper: scores p1 = p2 = 2, p3 = 1, budget 3. p1 and p2 + Example 2.3 from the paper: scores p1 = p2 = 2, p3 = 1, budget 3. p1 and p2 are funded (cost 1 each); p3 (cost 2) no longer fits. >>> p1, p2, p3 = Project("p1", 1), Project("p2", 1), Project("p3", 2) @@ -314,10 +366,11 @@ def random_setup( seed: int | None = None, ) -> set[Project]: """ - Algorithm 1 - Random setup (Section 3.1.1): the exposed set E_v of a Target + Random setup (Sections 2.4 / 3.1.1): the exposed set E_v of one Target Voter, made of k projects chosen uniformly at random from P (the rest is - predicted later). Called once per TV voter, so a different draw can be drawn - for each; it needs no ballot, since the choice is random. + predicted later). Called once per TV voter with its own seed, so each + voter gets an independent draw ("possibly different E for each voter"); + it needs no ballot, since the choice is random. Parameters ---------- @@ -335,8 +388,8 @@ def random_setup( Examples -------- - Example 5 from the paper: k = 2. Whatever the draw, exactly two projects are - exposed and they are real projects of the instance. + With k = 2, whatever the draw, exactly two projects are exposed and they + are real projects of the instance. >>> p1, p2, p3, p4 = (Project("p1", 2), Project("p2", 2), ... Project("p3", 3), Project("p4", 3)) @@ -361,7 +414,7 @@ def offline_popularity( instance: Instance, lv_profile: ApprovalProfile, k: int ) -> set[Project]: """ - Algorithm 2 - Offline revealing by popularity (Section 3.1.2). The exposed + Offline revealing by popularity (Section 3.1.2). The exposed set E = {sigma_1, ..., sigma_k}: the k most approved projects among the LV voters. The same set is exposed to every Target Voter, so it depends only on the LV profile (the score ordering sigma is used only to pick the top k; the @@ -384,8 +437,7 @@ def offline_popularity( Examples -------- - Example 6 from the paper: LV scores p1 = 3, p2 = 1, p3 = 1, so the single - most popular project is p1. + LV scores p1 = 3, p2 = 1, p3 = 1, so the single most popular project is p1. >>> p1, p2, p3 = Project("p1", 1), Project("p2", 1), Project("p3", 1) >>> inst = Instance([p1, p2, p3], budget_limit=3) @@ -407,7 +459,7 @@ def offline_consensus( instance: Instance, lv_profile: ApprovalProfile, k: int ) -> set[Project]: """ - Algorithm 3 - Offline revealing by consensus (Section 3.1.2). The exposed set + Offline revealing by consensus (Section 3.1.2). The exposed set E = {gamma_1, ..., gamma_k}: the k projects with the highest consensus level among the LV voters. The same set is exposed to every Target Voter. @@ -428,8 +480,8 @@ def offline_consensus( Examples -------- - Example 7 from the paper: p1 and p3 both have consensus 4 (p1 unanimously - approved, p3 unanimously rejected); the tie is broken by name towards p1. + p1 and p3 both have consensus 4 (p1 unanimously approved, p3 unanimously + rejected); the tie is broken by name towards p1. >>> p1, p2, p3 = Project("p1", 1), Project("p2", 1), Project("p3", 1) >>> inst = Instance([p1, p2, p3], budget_limit=3) @@ -447,10 +499,12 @@ def offline_controversiality( instance: Instance, lv_profile: ApprovalProfile, k: int ) -> set[Project]: """ - Algorithm 4 - Offline revealing by controversiality (Section 3.1.2). The - exposed set E = {gamma_{m-k+1}, ..., gamma_m}: the k projects *least* in - consensus among the LV voters (the hardest to predict, so asked directly). - The same set is exposed to every Target Voter. + Offline revealing by controversiality (Section 3.1.2). The exposed set + E = {gamma_{m-k+1}, ..., gamma_m}: the k projects *least* in consensus + among the LV voters (the hardest to predict, so asked directly). The same + set is exposed to every Target Voter. (The paper writes + {gamma_{m-k}, ..., gamma_m}, which is k+1 projects - an off-by-one; its + prose says "the k projects least in consensus", implemented here.) Parameters ---------- @@ -469,8 +523,8 @@ def offline_controversiality( Examples -------- - Example 8 from the paper (same data as Example 7): p2 is split exactly in - half (consensus 0) and is therefore the single most controversial project. + Same data as the consensus example: p2 is split exactly in half + (consensus 0) and is therefore the single most controversial project. >>> p1, p2, p3 = Project("p1", 1), Project("p2", 1), Project("p3", 1) >>> inst = Instance([p1, p2, p3], budget_limit=3) @@ -493,16 +547,21 @@ def offline_controversiality( def online_adaptive_controversial( instance: Instance, lv_profile: ApprovalProfile, - full_ballot: set[Project], + tv_ballots: dict[str, set[Project]], k: int, -) -> set[Project]: +) -> dict[str, set[Project]]: """ - Algorithm 5 - Online adaptive-controversial setup (Section 3.1.3): the - exposed set E_v of one Target Voter, built in k iterations. In each iteration - the most controversial project is recomputed given the LV ballots and the - answers already revealed, then asked about (each answer affects the next - question, hence "adaptive"). The voter's full ballot is needed to simulate - those answers. + Online adaptive-controversial setup (Section 3.1.3): the exposed set E_v of + every Target Voter, keyed by voter id. The voters are queried one after the + other (in ``tv_ballots`` order), each in k iterations. In every iteration + the most controversial not-yet-asked project is recomputed over *everyone + who has voted on it so far* - the LV voters plus every TV answer already + collected - and the current voter's answer (simulated from her full ballot) + is folded into that tally. Each answer thus shifts which projects later + voters are asked about; this feedback is what makes the setup adaptive and + distinguishes it from offline revealing-by-controversiality, per the + paper's "choose project proposals based on all voters preferences + iteratively" (Section 6). Parameters ---------- @@ -510,232 +569,163 @@ def online_adaptive_controversial( The PB instance. lv_profile : :py:class:`~pabutools.election.profile.approvalprofile.ApprovalProfile` The full ballots of the Learning Voters (LV). - full_ballot : set[:py:class:`~pabutools.election.instance.Project`] - The full ballot of the Target Voter (TV) being queried - her approval - set A_v in the ideal instance - used to answer each adaptive question. + tv_ballots : dict[str, set[:py:class:`~pabutools.election.instance.Project`]] + The full ballot of every Target Voter (her approval set A_v in the + ideal instance), keyed by voter id - used to answer the adaptive + questions. Voters are queried in iteration order. k : int - The number of iterations / projects to expose. + The number of iterations / projects to expose per voter. Returns ------- - set[:py:class:`~pabutools.election.instance.Project`] - The exposed set E_v of the k projects that ended up being asked. + dict[str, set[:py:class:`~pabutools.election.instance.Project`]] + The exposed set E_v of each Target Voter, keyed by voter id. Examples -------- - Example 9 from the paper: 4 LV voters, one TV voter v5, k = 2. p1, p2, p3 - are all tied as most controversial; the first question (tie broken by name) - is p1, and after the voter's answer the next is p2, so E_v = {p1, p2}. + One voter, 4 LV voters, k = 2: p1, p2, p3 are tied as most controversial + (consensus 0); the first question (tie broken by name) is p1, the second + p2, so E_v = {p1, p2}. >>> p1, p2, p3, p4 = (Project("p1", 1), Project("p2", 1), ... Project("p3", 1), Project("p4", 1)) >>> inst = Instance([p1, p2, p3, p4], budget_limit=4) >>> lv = ApprovalProfile([ApprovalBallot([p1, p2]), ApprovalBallot([p1, p3]), ... ApprovalBallot([p2, p3]), ApprovalBallot([])]) - >>> online_adaptive_controversial(inst, lv, {p1, p2}, k=2) == {p1, p2} + >>> online_adaptive_controversial(inst, lv, {"v5": {p1, p2}}, k=2)["v5"] == {p1, p2} True - """ - # The consensus of a project is a property of the electorate that has voted - # on it. A Target Voter has only answered the projects already asked, so her - # revealed answers never affect the consensus of the *remaining* projects - - # which is why, for a single voter, recomputing each round is equivalent to - # ranking once by the LV consensus. We still run the k rounds explicitly and - # simulate each answer, matching Algorithm 5's structure. - consensus = consensus_levels(instance, lv_profile) - exposed: set[Project] = set() - for round_index in range(1, k + 1): - # Most controversial not-yet-asked project: lowest consensus, ties by name. - project = min(set(instance) - exposed, key=lambda p: (consensus[p], str(p))) - logger.info( - "online_adaptive_controversial: round %d asks %s -> voter %s", - round_index, project, - "approves" if project in full_ballot else "disapproves", - ) - exposed.add(project) - return exposed + The adaptive feedback: p1 and p2 start tied as most controversial, so the + first voter is asked p1. Her approval breaks the tie - p1 now leans + approved and is *less* controversial - so the second voter is asked p2. -# --------------------------------------------------------------------------- -# Section 2.1.1 - Binary classification (predicted with a trained model). -# --------------------------------------------------------------------------- -def predict_by_classification( - instance: Instance, - lv_profile: ApprovalProfile, - ballot: CardinalBallot, -) -> ApprovalBallot: + >>> q1, q2, q3 = Project("q1", 1), Project("q2", 1), Project("q3", 1) + >>> inst2 = Instance([q1, q2, q3], budget_limit=3) + >>> lv2 = ApprovalProfile([ApprovalBallot([q1]), ApprovalBallot([q2])]) + >>> online_adaptive_controversial(inst2, lv2, {"v3": {q1}, "v4": {q1}}, k=1) + {'v3': {q1}, 'v4': {q2}} """ - Prediction module - binary classification (Section 2.1.1): predict each - hidden project with a per-project binary classifier trained on the LV ballots - (features = votes on the exposed projects, label = vote on the target), then - applied to the TV voter. Exposed approvals A_v are kept and exposed - disapprovals D_v stay rejected. The classifiers are fitted by - :py:func:`pb_model_training.train_classification` (backed by ``xgboost``); - this function only *applies* the trained model. - - Parameters - ---------- - instance : :py:class:`~pabutools.election.instance.Instance` - The PB instance. - lv_profile : :py:class:`~pabutools.election.profile.approvalprofile.ApprovalProfile` - The full ballots of the LV voters, used as the training data. - ballot : :py:class:`~pabutools.election.ballot.cardinalballot.CardinalBallot` - The Target Voter's partial ballot to complete (the +1/-1/0 partial - ballot built by ``partial_ballot`` / ``reveal_ballot``). - - Returns - ------- - :py:class:`~pabutools.election.ballot.approvalballot.ApprovalBallot` - The full predicted approval ballot of the TV voter. - - Examples - -------- - Example 11 from the paper: 2 LV voters both approve {p1, p2}. A TV voter with - nothing exposed is completed to {p1, p2} - on such an unambiguous pattern any - correct predictor (XGBoost included) agrees with the LV majority. - - >>> p1, p2, p3 = Project("p1", 4), Project("p2", 4), Project("p3", 6) - >>> inst = Instance([p1, p2, p3], budget_limit=6) - >>> lv = ApprovalProfile([ApprovalBallot([p1, p2]), ApprovalBallot([p1, p2])]) - >>> partial = partial_ballot(hidden={p1, p2, p3}) - >>> predict_by_classification(inst, lv, partial) == {p1, p2} - True - """ - model = train_classification(instance, lv_profile, exposed_projects(ballot)) - approved = approved_projects(ballot) - # The TV voter's feature row x: her vote (1/0) on each exposed feature project. - x = np.array([[1 if f in approved else 0 for f in model["features"]]]) - hidden = hidden_projects(ballot, instance) - votes = { - p: (payload if kind == "const" else int(payload.predict(x)[0])) - for p, (kind, payload) in model["per_project"].items() - if p in hidden - } - # predicted ballot = A_v u {p in H_v : classifier of p predicts approval} - result = approved | {p for p in hidden if votes[p] == 1} - logger.info("predict_by_classification: completed ballot to %d approvals", len(result)) - return ApprovalBallot(result) - - -def _approve_hidden_by_score( - instance: Instance, ballot: CardinalBallot, scores: dict[Project, float] -) -> set[Project]: - """ - Complete a partial ballot from per-project scores: keep the exposed approvals - A_v and add each hidden project whose predicted score is >= 0.5 (the exposed - disapprovals D_v stay rejected). Shared by the matrix-factorization and - factorization-machines predictors. - - >>> p1, p2, p3 = Project("p1", 1), Project("p2", 1), Project("p3", 1) - >>> inst = Instance([p1, p2, p3], budget_limit=3) - >>> b = partial_ballot(approved={p1}, hidden={p2, p3}) - >>> _approve_hidden_by_score(inst, b, {p1: 1.0, p2: 0.8, p3: 0.2}) == {p1, p2} - True - """ - # predicted ballot = A_v u {p in H_v : score(p) >= 1/2} - return approved_projects(ballot) | { - p for p in hidden_projects(ballot, instance) if scores[p] >= 0.5 - } + # Definition 2.2 over everyone who has voted on p so far: the LV electorate + # first, then each collected TV answer. consensus(p) = |approvers - disapprovers|. + scores = lv_profile.approval_scores() + approvers = {p: scores.get(p, 0) for p in instance} + disapprovers = {p: lv_profile.num_ballots() - approvers[p] for p in instance} + exposed_by_voter: dict[str, set[Project]] = {} + for vid, ballot in tv_ballots.items(): + exposed: set[Project] = set() + for _ in range(k): + # gamma_m of the not-yet-asked projects: least consensus, ties by name. + asked = min( + set(instance) - exposed, + key=lambda p: (abs(approvers[p] - disapprovers[p]), str(p)), + ) + logger.info( + "online_adaptive_controversial: asks %s -> voter %s %s", asked, + vid, "approves" if asked in ballot else "disapproves", + ) + (approvers if asked in ballot else disapprovers)[asked] += 1 + exposed.add(asked) + exposed_by_voter[vid] = exposed + return exposed_by_voter -def predict_by_matrix_factorization( +def next_adaptive_question( instance: Instance, lv_profile: ApprovalProfile, - ballot: CardinalBallot, -) -> ApprovalBallot: + answers: CardinalBallot, + collected: Iterable[CardinalBallot] = (), +) -> Project: """ - Prediction module - collaborative filtering via Matrix Factorization - (Section 2.1.2): build the sparse user-item matrix from the LV ballots and - exposed TV votes (approve=1, disapprove=0), factorise it, and predict a - hidden project as approved iff its reconstructed score is >= 0.5. Exposed - approvals A_v are kept and exposed disapprovals D_v stay rejected. The model - is fitted by :py:func:`pb_model_training.train_matrix_factorization` (backed - by ``scikit-surprise``); this function only *applies* the trained model. + The single project to put to a voter next, under the online adaptive- + controversial setup (Section 3.1.3): the most controversial project she has + not been asked about yet, where controversiality is measured over everyone + who has voted on it so far. + + :py:func:`online_adaptive_controversial` runs this rule as a closed loop by + reading each answer off a known ballot, which a live process cannot do - the + answers are exactly what it does not have yet. This is that loop's single + step, exposed so a caller can drive it: ask the returned project, add the + reply to ``answers``, and call again until ``answers`` holds k votes. Feeding + the replies back is what makes the setup adaptive; it is the only setup of + the five that cannot be planned in advance with :py:func:`exposed_sets`. Parameters ---------- instance : :py:class:`~pabutools.election.instance.Instance` - The PB instance. + The PB instance (the universe of projects P). lv_profile : :py:class:`~pabutools.election.profile.approvalprofile.ApprovalProfile` - The full ballots of the LV voters, used as the training data. - ballot : :py:class:`~pabutools.election.ballot.cardinalballot.CardinalBallot` - The Target Voter's partial ballot to complete (the +1/-1/0 partial - ballot built by ``partial_ballot`` / ``reveal_ballot``). + The full ballots of the Learning Voters (LV). + answers : :py:class:`~pabutools.election.ballot.cardinalballot.CardinalBallot` + The replies this voter has given so far, as a partial ballot: her + answered projects are the ones excluded from the next question. + Pass an empty ``partial_ballot()`` for the first question. + collected : Iterable[:py:class:`~pabutools.election.ballot.cardinalballot.CardinalBallot`], optional + The partial ballots already gathered from *other* voters. Their votes + count towards controversiality but never towards what this voter is + asked, exactly as in the closed loop. Returns ------- - :py:class:`~pabutools.election.ballot.approvalballot.ApprovalBallot` - The full predicted approval ballot of the TV voter. + :py:class:`~pabutools.election.instance.Project` + The project to ask about, ties broken lexicographically by name. + + Raises + ------ + ValueError + If the voter has already been asked about every project. Examples -------- - Example 10 from the paper: three LV voters all support the cheap pair - {p1, p2} and reject {p3, p4}. The reconstructed matrix completes a TV voter - with nothing exposed to {p1, p2}. - - >>> p1, p2, p3, p4 = (Project("p1", 3), Project("p2", 3), - ... Project("p3", 4), Project("p4", 4)) - >>> inst = Instance([p1, p2, p3, p4], budget_limit=6) - >>> lv = ApprovalProfile([ApprovalBallot([p1, p2])] * 3) - >>> partial = partial_ballot(hidden={p1, p2, p3, p4}) - >>> predict_by_matrix_factorization(inst, lv, partial) == {p1, p2} - True + Two LV voters split over q1 and q2, nobody approving q3: q1 and q2 are tied + at consensus 0 and q3 is settled, so the first question is q1 (tie broken by + name). Her approval of q1 tips it towards approved, so the second question + is q2 - the same order the closed loop produces. + + >>> q1, q2, q3 = Project("q1", 1), Project("q2", 1), Project("q3", 1) + >>> inst = Instance([q1, q2, q3], budget_limit=3) + >>> lv = ApprovalProfile([ApprovalBallot([q1]), ApprovalBallot([q2])]) + >>> first = next_adaptive_question(inst, lv, partial_ballot()) + >>> first + q1 + >>> next_adaptive_question(inst, lv, partial_ballot(approved={first})) + q2 + + Once every project has been answered there is nothing left to ask: + + >>> answered = partial_ballot(approved={q1, q2}, disapproved={q3}) + >>> next_adaptive_question(inst, lv, answered) + Traceback (most recent call last): + ... + ValueError: this voter has already been asked about all 3 projects """ - scores = train_matrix_factorization( - instance, lv_profile, approved_projects(ballot), disapproved_projects(ballot) + remaining = set(instance) - exposed_projects(answers) + if not remaining: + raise ValueError( + f"this voter has already been asked about all {len(instance)} projects" + ) + # Definition 2.2 over everyone who has voted on p so far: the LV electorate, + # every answer collected from other voters, and this voter's own replies. + scores = lv_profile.approval_scores() + approvers = {p: scores.get(p, 0) for p in instance} + disapprovers = {p: lv_profile.num_ballots() - approvers[p] for p in instance} + for ballot in (*collected, answers): + for project, score in ballot.items(): + if score > 0: + approvers[project] += 1 + elif score < 0: + disapprovers[project] += 1 + asked = min( + remaining, + key=lambda p: (abs(approvers[p] - disapprovers[p]), str(p)), ) - result = _approve_hidden_by_score(instance, ballot, scores) - logger.info("predict_by_matrix_factorization: completed to %d approvals", len(result)) - return ApprovalBallot(result) - - -def predict_by_factorization_machines( - instance: Instance, - lv_profile: ApprovalProfile, - ballot: CardinalBallot, -) -> ApprovalBallot: - """ - Prediction module - hybrid Factorization Machines (Section 2.1.2): like MF - but with a linear term plus pairwise latent interactions and optional side - features, predicting a hidden project as approved iff the FM score is >= 0.5. - Exposed approvals A_v are kept and exposed disapprovals D_v stay rejected. - The model is fitted by - :py:func:`pb_model_training.train_factorization_machines` (backed by an - external FM library, e.g. ``lightfm`` / ``fastFM``); this function only - *applies* the trained model. - - Parameters - ---------- - instance : :py:class:`~pabutools.election.instance.Instance` - The PB instance. - lv_profile : :py:class:`~pabutools.election.profile.approvalprofile.ApprovalProfile` - The full ballots of the LV voters, used as the training data. - ballot : :py:class:`~pabutools.election.ballot.cardinalballot.CardinalBallot` - The Target Voter's partial ballot to complete (the +1/-1/0 partial - ballot built by ``partial_ballot`` / ``reveal_ballot``). - - Returns - ------- - :py:class:`~pabutools.election.ballot.approvalballot.ApprovalBallot` - The full predicted approval ballot of the TV voter. + logger.info( + "next_adaptive_question: asking %s (consensus %d of %d voters so far)", + asked, abs(approvers[asked] - disapprovers[asked]), + approvers[asked] + disapprovers[asked], + ) + return asked - Examples - -------- - Example 11 from the paper: 2 LV voters both approve {p1, p2}. A TV voter with - nothing exposed is completed to {p1, p2}. - >>> p1, p2, p3 = Project("p1", 4), Project("p2", 4), Project("p3", 6) - >>> inst = Instance([p1, p2, p3], budget_limit=6) - >>> lv = ApprovalProfile([ApprovalBallot([p1, p2]), ApprovalBallot([p1, p2])]) - >>> partial = partial_ballot(hidden={p1, p2, p3}) - >>> predict_by_factorization_machines(inst, lv, partial) == {p1, p2} - True - """ - scores = train_factorization_machines( - instance, lv_profile, approved_projects(ballot), disapproved_projects(ballot) - ) - result = _approve_hidden_by_score(instance, ballot, scores) - logger.info("predict_by_factorization_machines: completed to %d approvals", len(result)) - return ApprovalBallot(result) # --------------------------------------------------------------------------- @@ -771,9 +761,10 @@ def exposed_sets( The exposed set E_v of every Target Voter under one of the five Section 3.1 setups, keyed by voter id. Helper that hides the setups' differing signatures behind one interface: the three offline samplers expose the - *same* k projects to everyone, while ``random`` and - ``online_adaptive_controversial`` are computed per voter (the latter needs - each voter's full ballot to answer its adaptive questions). + *same* k projects to everyone, ``random`` draws independently per voter, + and ``online_adaptive_controversial`` queries the voters sequentially, + folding each answer into the controversiality tally that picks the later + voters' questions. Examples -------- @@ -807,12 +798,12 @@ def exposed_sets( ) logger.debug("exposed_sets: validated setup=%s, k=%d", setup, k) if setup == "random": - return {vid: random_setup(instance, k, seed) for vid in tv_ballots} + # "possibly different E for each voter" (Section 2.4): one independent + # uniform draw per voter, derived from the single seed. + rng = random.Random(seed) + return {vid: random_setup(instance, k, rng.randrange(2**32)) for vid in tv_ballots} if setup == "online_adaptive_controversial": - return { - vid: online_adaptive_controversial(instance, lv_profile, full_ballot, k) - for vid, full_ballot in tv_ballots.items() - } + return online_adaptive_controversial(instance, lv_profile, tv_ballots, k) offline = { "offline_popularity": offline_popularity, "offline_consensus": offline_consensus, @@ -822,72 +813,227 @@ def exposed_sets( return {vid: shared for vid in tv_ballots} +def plan_sampling( + n_voters: int, + n_projects: int, + sample_degree: float, + lv_degree: float, +) -> tuple[int, int]: + """ + Size a process from the two partiality knobs of Section 3.0.1, *before* any + ballot exists: how many voters must give a full ballot, and how many projects + each remaining voter is asked about. + + This is the arithmetic of Example 3.1 on its own. With n voters and m + projects, ``sample_degree`` (s) fixes the total number of collected votes at + s*n*m and ``lv_degree`` (l) is the share of those votes that come from full + ballots, so |LV| = round(s*l*n) and the remaining s*(1-l)*n*m votes are + divided equally among the s*n*m Target Voters. ``lv_degree == 1`` is the + paper's naive sampling baseline: the sampled voters answer in full and + nobody else is asked anything, so k = 0. + + Use it to plan a real election - "we have 5000 voters, 100 projects, and we + are willing to ask for 30% of the votes; how many questions each?" - + and :py:func:`split_lv_tv` to carve a simulated one out of a known profile. + + Parameters + ---------- + n_voters : int + The number n of voters in the electorate. + n_projects : int + The number m of projects on the ballot. + sample_degree : float + Fraction of all n*m votes that are collected, in [0, 1]. + lv_degree : float + Fraction of the collected votes coming from full ballots, in [0, 1]. + + Returns + ------- + tuple[int, int] + ``(number of Learning Voters, questions per Target Voter)``. + + Examples + -------- + Four voters and two projects. Collecting everything with ``lv_degree == 1`` + makes all four Learning Voters and leaves nobody to question; collecting + half the votes with half of them from full ballots gives one Learning Voter + and asks the other three about 0.5*0.5*4*2/3 ~ 1 project each. + + >>> plan_sampling(4, 2, sample_degree=1.0, lv_degree=1.0) + (4, 0) + >>> plan_sampling(4, 2, sample_degree=0.5, lv_degree=0.5) + (1, 1) + + A realistic district: 30% of the votes collected, a tenth of them as full + ballots, so 150 people fill in all 100 projects and the other 4850 are asked + about 28 each - instead of every one of the 5000 facing the whole ballot. + + >>> plan_sampling(5000, 100, sample_degree=0.3, lv_degree=0.1) + (150, 28) + + >>> plan_sampling(4, 2, sample_degree=1.5, lv_degree=0.5) + Traceback (most recent call last): + ... + ValueError: sample_degree=1.5 must be in [0, 1] + """ + for name, degree in (("sample_degree", sample_degree), ("lv_degree", lv_degree)): + if not 0 <= degree <= 1: + raise ValueError(f"{name}={degree} must be in [0, 1]") + n_lv = round(sample_degree * lv_degree * n_voters) + n_tv = n_voters - n_lv + if lv_degree == 1 or not n_tv: + logger.info( + "plan_sampling: sample=%.2f lv=%.2f -> all %d collected votes come " + "from %d full ballots, nobody else is asked anything (k=0)", + sample_degree, lv_degree, n_lv * n_projects, n_lv, + ) + return n_lv, 0 + votes_left = sample_degree * (1 - lv_degree) * n_voters * n_projects + k = round(votes_left / n_tv) + logger.info( + "plan_sampling: sample=%.2f lv=%.2f of %d voters x %d projects -> " + "%d full ballots, the other %d asked k=%d each " + "(%.0f of %d votes collected in total)", + sample_degree, lv_degree, n_voters, n_projects, n_lv, n_tv, k, + n_lv * n_projects + n_tv * k, n_voters * n_projects, + ) + return n_lv, k + + def split_lv_tv( + instance: Instance, profile: ApprovalProfile, sample_degree: float, lv_degree: float, seed: int | None = None, -) -> tuple[ApprovalProfile, dict[str, set[Project]]]: +) -> tuple[ApprovalProfile, dict[str, set[Project]], int]: """ - Step 1 of the pipeline (Section 2.4 / 3.0.1): partition the voters of the - *ideal* instance into Learning Voters (LV, who keep their full ballots) and - Target Voters (TV, whose ballots start hidden and are later completed). + Step 1 of the pipeline (Sections 2.4 / 3.0.1): partition the n voters of + the *ideal* instance into Learning Voters (LV, full ballots) and Target + Voters (TV, queried and completed), and derive k, the number of projects + each TV voter is asked about. - The partition follows the paper's two knobs (Example 3.1), read here at the - voter level: + The two knobs are *vote* budgets (Example 3.1 of the paper). With n voters + and m projects, ``sample_degree`` (s) fixes the total number of collected + votes at s*n*m and ``lv_degree`` (l) is the share of those votes that come + from full ballots: - * ``sample_degree`` - the fraction of voters that are *sampled* (participate). - The rest are dropped from the partial instance I1. - * ``lv_degree`` - among the sampled voters, the fraction that are LV; the rest - are TV. ``lv_degree == 1`` is the paper's naive "sampling" baseline (every - sampled voter gives a full ballot, no prediction needed). + * |LV| = round(s*l*n) voters, drawn at random, keep their full ballots; + * every *other* voter is a TV, and the remaining s*(1-l)*n*m votes are + divided equally among them: k = round(s*(1-l)*n*m / |TV|). + + ``lv_degree == 1`` is the paper's naive "sampling" baseline (Section 6): + the sampled voters answer in full and the rest of the community does not + vote at all - no TV voters, k = 0. Parameters ---------- + instance : :py:class:`~pabutools.election.instance.Instance` + The PB instance (m projects). profile : :py:class:`~pabutools.election.profile.approvalprofile.ApprovalProfile` The ideal instance's full ballots (all n voters). sample_degree : float - Fraction of voters sampled, in [0, 1]. + Fraction of all n*m votes that are collected, in [0, 1]. lv_degree : float - Fraction of the sampled voters that are LV, in [0, 1]. + Fraction of the collected votes coming from full ballots, in [0, 1]. seed : int, optional Seed for the random partition, for reproducibility. Returns ------- - tuple[:py:class:`~pabutools.election.profile.approvalprofile.ApprovalProfile`, dict[str, set[:py:class:`~pabutools.election.instance.Project`]]] - The LV profile (full ballots) and the TV voters' full ballots keyed by - voter id (their known ground truth, used to simulate the k answers). + tuple[:py:class:`~pabutools.election.profile.approvalprofile.ApprovalProfile`, dict[str, set[:py:class:`~pabutools.election.instance.Project`]], int] + The LV profile (full ballots), the TV voters' full ballots keyed by + voter id (their known ground truth, used to simulate the k answers), + and k. Examples -------- - Four voters. Sampling everyone with ``lv_degree == 1`` makes everyone an LV - and leaves no TV; a half sample split evenly gives one LV and one TV. + Four voters, two projects. Collecting everything with ``lv_degree == 1`` + makes everyone an LV and leaves no TV. Collecting half the votes with half + of them from full ballots gives one LV voter; the other three are all TV, + each asked k = 0.5*0.5*4*2/3 ~ 1 project. >>> p1, p2 = Project("p1", 1), Project("p2", 1) + >>> inst = Instance([p1, p2], budget_limit=2) >>> prof = ApprovalProfile([ApprovalBallot([p1]), ApprovalBallot([p2]), ... ApprovalBallot([p1, p2]), ApprovalBallot([])]) - >>> lv, tv = split_lv_tv(prof, sample_degree=1.0, lv_degree=1.0) - >>> lv.num_ballots(), len(tv) - (4, 0) - >>> lv, tv = split_lv_tv(prof, sample_degree=0.5, lv_degree=0.5, seed=0) - >>> lv.num_ballots(), len(tv) - (1, 1) + >>> lv, tv, k = split_lv_tv(inst, prof, sample_degree=1.0, lv_degree=1.0) + >>> lv.num_ballots(), len(tv), k + (4, 0, 0) + >>> lv, tv, k = split_lv_tv(inst, prof, sample_degree=0.5, lv_degree=0.5, seed=0) + >>> lv.num_ballots(), len(tv), k + (1, 3, 1) + >>> split_lv_tv(inst, prof, sample_degree=1.5, lv_degree=0.5) + Traceback (most recent call last): + ... + ValueError: sample_degree=1.5 must be in [0, 1] """ voters = list(profile) - order = list(range(len(voters))) - random.Random(seed).shuffle(order) - n_sample = round(sample_degree * len(voters)) - sampled = order[:n_sample] - n_lv = round(lv_degree * n_sample) - lv_profile = ApprovalProfile([voters[i] for i in sampled[:n_lv]]) - tv_ballots = {f"v{i}": set(voters[i]) for i in sampled[n_lv:]} + n, m = len(voters), len(instance) + n_lv, k = plan_sampling(n, m, sample_degree, lv_degree) + order = random.Random(seed).sample(range(n), n) # a random permutation + lv_profile = ApprovalProfile([voters[i] for i in order[:n_lv]]) + # lv_degree == 1 is the naive sampling baseline: nobody else votes at all. + tv_ballots = ( + {} if lv_degree == 1 + else {f"v{i}": set(voters[i]) for i in order[n_lv:]} + ) logger.info( - "split_lv_tv: sample=%.2f lv=%.2f -> %d LV, %d TV (of %d voters)", - sample_degree, lv_degree, lv_profile.num_ballots(), len(tv_ballots), - len(voters), + "split_lv_tv: sample=%.2f lv=%.2f -> %d LV, %d TV asked k=%d each (of %d voters)", + sample_degree, lv_degree, lv_profile.num_ballots(), len(tv_ballots), k, n, ) - return lv_profile, tv_ballots + return lv_profile, tv_ballots, k + + +def complete_ballots( + instance: Instance, + lv_profile: ApprovalProfile, + tv_ballots: dict[str, set[Project]], + k: int, + *, + setup: str, + predict, + seed: int | None = None, +) -> tuple[dict[str, ApprovalBallot], dict[str, set[Project]]]: + """ + Steps 2 and 3 of the pipeline: choose every Target Voter's exposed set under + ``setup``, then complete her ballot with the ``predict`` module. + + Returns the completed ballots *and* the exposed sets, both keyed by voter + id. The exposed sets come back because a caller needs them to tell which + votes were predicted rather than answered - Section 5.1 scores only the + hidden ones. Shared by :py:func:`run_pipeline`, which goes on to apply the + voting rule, and :py:func:`run_all_experiments`, which also scores the + ballots themselves. + + Examples + -------- + Three LV voters approve {p1, p2}; the single TV voter is shown p1 only, and + her hidden vote on p2 is filled in from theirs. + + >>> p1, p2, p3 = Project("p1", 3), Project("p2", 3), Project("p3", 4) + >>> inst = Instance([p1, p2, p3], budget_limit=6) + >>> lv = ApprovalProfile([ApprovalBallot([p1, p2])] * 3) + >>> done, shown = complete_ballots(inst, lv, {"v": {p1, p2}}, k=1, + ... setup="offline_popularity", + ... predict=predict_by_classification) + >>> shown["v"] == {p1}, done["v"] == {p1, p2} + (True, True) + """ + exposed = exposed_sets(instance, lv_profile, tv_ballots, setup, k, seed) + partial = { + vid: reveal_ballot(instance, tv_ballots[vid], exposed[vid]) + for vid in tv_ballots + } + hidden = sum(len(instance) - len(exposed[vid]) for vid in tv_ballots) + logger.info( + "complete_ballots: setup=%s exposed %d of %d votes across %d TV " + "voters, asking %s to predict the remaining %d", + setup, sum(len(e) for e in exposed.values()), + len(tv_ballots) * len(instance), len(tv_ballots), + getattr(predict, "__name__", predict), hidden, + ) + return predict(instance, lv_profile, partial), exposed def run_pipeline( @@ -923,7 +1069,8 @@ def run_pipeline( The name of the Section 3.1 setup to use (one of :py:data:`SETUPS`). predict : callable The Section 2.1 prediction module (one of :py:data:`PREDICTORS`' - values). + values): it takes the TV voters' partial ballots, keyed by voter + id, and returns their completed approval ballots. seed : int, optional Seed for the ``random`` setup, ignored by the others. @@ -934,9 +1081,8 @@ def run_pipeline( Examples -------- - Example 10 from the paper - the "perfect" case. Three LV voters and one TV - voter all support the two cheap projects {p1, p2}; the predicted bundle - coincides with the real one. + The "perfect" case: three LV voters and one TV voter all support the two + cheap projects {p1, p2}; the predicted bundle coincides with the real one. >>> p1, p2, p3, p4 = (Project("p1", 3), Project("p2", 3), ... Project("p3", 4), Project("p4", 4)) @@ -952,21 +1098,13 @@ def run_pipeline( setup, predict.__name__, k, lv_profile.num_ballots(), len(tv_ballots), ) # Step 1 (the LV/TV split) is done by the caller / :py:func:`split_lv_tv`. - # Step 2 - sampling: pick the exposed set of every TV voter under the setup. - exposed = exposed_sets(instance, lv_profile, tv_ballots, setup, k, seed) - # Steps 3-4 - prediction + combine: complete every TV ballot from its k - # exposed answers and merge with the known LV ballots. - combined = ApprovalProfile( - list(lv_profile) - + [ - predict( - instance, - lv_profile, - reveal_ballot(instance, tv_ballots[vid], exposed[vid]), - ) - for vid in tv_ballots - ] + # Steps 2-3 - sampling + prediction. + completed, _ = complete_ballots( + instance, lv_profile, tv_ballots, k, + setup=setup, predict=predict, seed=seed, ) + # Step 4 - combine with the known LV ballots. + combined = ApprovalProfile(list(lv_profile) + [completed[vid] for vid in tv_ballots]) logger.info( "run_pipeline: all %d TV ballots completed, running the voting rule", len(tv_ballots), @@ -975,6 +1113,196 @@ def run_pipeline( return greedy_approval(instance, combined) +def run_experiment( + instance: Instance, + profile: ApprovalProfile, + sample_degree: float, + lv_degree: float, + *, + setup: str, + predictor="classification", + seed: int | None = None, +) -> BudgetAllocation: + """ + One experiment end to end, straight from the two partiality knobs: split the + ideal ``profile`` into LV and TV with :py:func:`split_lv_tv` (which derives k + from the knobs) and run :py:func:`run_pipeline` on the result. A convenience + composition of the two, for when you want a single cell of the design matrix + without holding the split yourself. + + Like :py:func:`run_pipeline` this is a *simulation*: it needs the ideal + profile, because the Target Voters' answers are read off their true ballots. + To run a process whose answers you actually collected, use :py:func:`elect`. + + Parameters + ---------- + instance : :py:class:`~pabutools.election.instance.Instance` + The PB instance. + profile : :py:class:`~pabutools.election.profile.approvalprofile.ApprovalProfile` + The ideal instance's full ballots (all n voters). + sample_degree, lv_degree : float + The partiality knobs of Section 3.0.1, both in [0, 1]. + setup : str + The Section 3.1 setup to sample with (one of :py:data:`SETUPS`). + predictor : str or callable, optional + A name from :py:data:`PREDICTORS` or a prediction module itself. + Defaults to ``"classification"``, the paper's best performer. + seed : int, optional + Seed for the split and the sampling, for reproducibility. + + Returns + ------- + :py:class:`~pabutools.rules.budgetallocation.BudgetAllocation` + The predicted winning bundle. + + Examples + -------- + Four voters who all approve the two cheap projects; collecting half the votes + still reproduces the real bundle. + + >>> p1, p2, p3 = Project("p1", 3), Project("p2", 3), Project("p3", 4) + >>> inst = Instance([p1, p2, p3], budget_limit=6) + >>> prof = ApprovalProfile([ApprovalBallot([p1, p2])] * 4) + >>> sorted(run_experiment(inst, prof, 0.5, 0.5, setup="offline_popularity", + ... seed=0), key=str) + [p1, p2] + """ + logger.info( + "run_experiment: simulating sample_degree=%.2f lv_degree=%.2f with " + "setup=%s and the %s predictor", + sample_degree, lv_degree, setup, predictor, + ) + lv_profile, tv_ballots, k = split_lv_tv( + instance, profile, sample_degree, lv_degree, seed=seed + ) + return run_pipeline( + instance, lv_profile, tv_ballots, k, + setup=setup, predict=as_predictor(predictor), seed=seed, + ) + + +# --------------------------------------------------------------------------- +# Running a real process, as opposed to reproducing the paper's experiments. +# --------------------------------------------------------------------------- +#: A Section 2.1 prediction module: it completes the Target Voters' partial +#: ballots, keyed by voter id. Named in :py:data:`PREDICTORS`. +Predictor = Callable[ + [Instance, ApprovalProfile, dict[str, CardinalBallot]], + dict[str, ApprovalBallot], +] + + +def as_predictor(predictor: str | Predictor) -> Predictor: + """ + The prediction module named by ``predictor``, which may be a key of + :py:data:`PREDICTORS` or the function itself, so callers can say + ``"classification"`` instead of importing it. + + Examples + -------- + >>> as_predictor("classification") is predict_by_classification + True + >>> as_predictor(predict_by_classification) is predict_by_classification + True + >>> as_predictor("by_magic") + Traceback (most recent call last): + ... + ValueError: unknown predictor 'by_magic'; expected one of: classification, matrix_factorization, factorization_machines + """ + if callable(predictor): + return predictor + if predictor not in PREDICTORS: + raise ValueError( + f"unknown predictor {predictor!r}; expected one of: " + + ", ".join(PREDICTORS) + ) + return PREDICTORS[predictor] + + +def elect( + instance: Instance, + lv_profile: ApprovalProfile, + tv_ballots: dict[str, CardinalBallot], + predictor: str | Predictor = "classification", +) -> BudgetAllocation: + """ + Decide a real PB process from the ballots that were actually collected. + + This is the deployment counterpart of :py:func:`run_pipeline`. The pipeline + is a simulation: it takes the Target Voters' *true* ballots and reads their + answers off them, which you cannot do in a live process because those + answers are exactly what you do not have. ``elect`` instead takes the + answers themselves - each Target Voter's partial ballot, saying which of the + projects she was asked about she approved and which she rejected - completes + the gaps with a Section 2.1 prediction module, and applies greedy approval + to the LV ballots plus the completed TV ballots. + + A live process therefore runs in three steps: :py:func:`plan_sampling` to + size it, :py:func:`exposed_sets` to choose which projects to put in front of + each voter, and ``elect`` once the answers are in. + + .. note:: + Of the five setups, ``random`` and the three ``offline_*`` ones need + nothing but the voter ids to choose their questions, so they work in a + live process as they are. ``online_adaptive_controversial`` picks each + question from the answers gathered so far, so deploying it needs an + interactive loop rather than a single call. + + Parameters + ---------- + instance : :py:class:`~pabutools.election.instance.Instance` + The PB instance. + lv_profile : :py:class:`~pabutools.election.profile.approvalprofile.ApprovalProfile` + The full ballots that were submitted in full (the Learning Voters). + May be empty if every voter answered only part of the ballot. + tv_ballots : dict[str, :py:class:`~pabutools.election.ballot.cardinalballot.CardinalBallot`] + The partial ballots that came back, keyed by voter id: build each one + with :py:func:`partial_ballot` from the projects that voter approved + and rejected. Everything else is treated as unanswered and predicted. + predictor : str or callable, optional + A name from :py:data:`PREDICTORS` or a prediction module itself. + Defaults to ``"classification"``, the paper's best performer. + + Returns + ------- + :py:class:`~pabutools.rules.budgetallocation.BudgetAllocation` + The winning bundle, respecting the instance's budget limit. + + Examples + -------- + Three voters filled in the whole ballot and approved {p1, p2}. A fourth was + asked about only p1 and p3: she approved p1 and rejected p3, and was never + asked about p2. Her opinion on p2 is predicted from the others, and the + budget of 6 funds the two projects costing 3. + + >>> p1, p2, p3 = Project("p1", 3), Project("p2", 3), Project("p3", 4) + >>> inst = Instance([p1, p2, p3], budget_limit=6) + >>> lv = ApprovalProfile([ApprovalBallot([p1, p2])] * 3) + >>> answers = {"v4": partial_ballot(approved={p1}, disapproved={p3})} + >>> sorted(elect(inst, lv, answers), key=str) + [p1, p2] + + An election where nobody filled in a full ballot still works, as long as the + answers between them cover the projects. + + >>> answers = { + ... "a": partial_ballot(approved={p1}, disapproved={p3}), + ... "b": partial_ballot(approved={p2}, disapproved={p3}), + ... } + >>> sorted(elect(inst, ApprovalProfile([]), answers), key=str) + [p1, p2] + """ + completed = as_predictor(predictor)(instance, lv_profile, tv_ballots) + combined = ApprovalProfile( + list(lv_profile) + [completed[vid] for vid in tv_ballots] + ) + logger.info( + "elect: %d full ballots + %d partial ballots completed, running the rule", + lv_profile.num_ballots(), len(tv_ballots), + ) + return greedy_approval(instance, combined) + + #: The partiality grid swept in the paper's experiments (Section 6). SAMPLE_DEGREES = (0.1, 0.15, 0.3, 0.5, 0.7, 0.9) LV_DEGREES = (0.1, 0.2, 0.3, 0.5, 0.7, 0.9, 1.0) @@ -983,7 +1311,6 @@ def run_pipeline( def run_all_experiments( instance: Instance, profile: ApprovalProfile, - k: int, *, setups: Iterable[str] = SETUPS, predictors: Iterable[str] = tuple(PREDICTORS), @@ -995,20 +1322,36 @@ def run_all_experiments( """ The paper's full treatment matrix (Section 6, Figure 4). For every cell - setup x predictor x sample_degree x lv_degree - the ideal ``profile`` is - split into LV/TV with :py:func:`split_lv_tv`, the pipeline is run with - :py:func:`run_pipeline`, and the predicted bundle is compared to the *real* - bundle (greedy approval on the whole ideal profile). Because the split is - random, each cell is repeated ``n_repeat`` times and the two bundle metrics - - Fractional Allocation (:py:func:`fractional_allocation_score`) and the - Symmetric Distance ``|rb △ pb|`` (Section 5.2.1, computed inline) - are - averaged. + split into LV/TV with :py:func:`split_lv_tv` (which also derives k, the + number of questions per TV voter, from the two degrees - Section 3.0.1), + the pipeline is run with :py:func:`run_pipeline`, and the predicted bundle + is compared to the *real* bundle (greedy approval on the whole ideal + profile). Because the split is random, each cell is repeated ``n_repeat`` + times and the two bundle metrics - Fractional Allocation + (:py:func:`fractional_allocation_score`) and the Symmetric Distance + ``|rb △ pb|`` (Section 5.2.1, computed inline) - are averaged. + + .. warning:: + **The defaults reproduce the paper's grid and will not finish.** They + describe 6 sample degrees x 7 LV degrees x 5 setups x 3 predictors = 630 + cells, each repeated 50 times, i.e. 31 500 pipeline runs. Measured on a + real Warsaw district (Praga-Poludnie 2022, 10 424 voters, 96 projects), + one run takes 30-165 seconds depending on the predictor - so the full + default sweep is of the order of a year of compute. + + Scale it down deliberately. Shrinking ``sample_degrees``, + ``lv_degrees``, ``setups``, ``predictors`` and ``n_repeat`` all help + proportionally; passing a one-element ``param_grid`` through to + :py:func:`pabutools.recommendation.model_training.train_classification` removes the + Section 5 hyperparameter search, which costs roughly 10x a plain fit. + A few hundred voters, one sample degree and ``n_repeat=3`` runs in + minutes and is enough to see the trends. .. note:: The paper repeats the sampling module 20 times and the prediction module - 50 times; ``n_repeat`` collapses both into one knob (default 50). The full - grid is ``len(setups) * len(predictors) * len(sample_degrees) * - len(lv_degrees)`` cells, so shrink the iterables or ``n_repeat`` for a - quick run. ``classification`` needs ``scikit-learn`` installed. + 50 times; ``n_repeat`` collapses both into one knob (default 50). + ``classification`` needs ``xgboost`` installed, matrix factorization + needs ``scikit-surprise``, and the hybrid needs ``lightfm``. Parameters ---------- @@ -1016,8 +1359,6 @@ def run_all_experiments( The PB instance. profile : :py:class:`~pabutools.election.profile.approvalprofile.ApprovalProfile` The ideal instance's full ballots (all voters). - k : int - The number of projects to expose per TV voter. setups : Iterable[str], optional The setups to sweep (names from :py:data:`SETUPS`). predictors : Iterable[str], optional @@ -1033,27 +1374,37 @@ def run_all_experiments( Returns ------- dict[tuple[float, float, str, str], dict[str, float]] - Keyed by ``(sample_degree, lv_degree, setup, predictor)``, each value - is ``{"FA": mean fractional allocation, "SD": mean symmetric distance}``. + Keyed by ``(sample_degree, lv_degree, setup, predictor)``. Each value + holds the two bundle metrics of Section 5.2 - ``"FA"`` and ``"SD"`` - + and the three classification metrics of Section 5.1 - + ``"precision"``, ``"recall"`` and ``"f1"``, averaged over the Target + Voters and scored only on the votes that were predicted rather than + answered. All five are means over ``n_repeat`` random splits. Examples -------- - Example 10 from the paper - the "perfect" case, swept over a tiny grid with - the two ML-free recommendation predictors. Every cell yields an FA in [0, 1] - and a non-negative SD. + The "perfect" case, swept over a tiny grid. Every cell yields an FA in + [0, 1] and a non-negative SD. Only matrix factorization is swept here so + that the example runs wherever ``scikit-surprise`` is installed, without + also needing ``lightfm``. >>> p1, p2, p3, p4 = (Project("p1", 3), Project("p2", 3), ... Project("p3", 4), Project("p4", 4)) >>> inst = Instance([p1, p2, p3, p4], budget_limit=6) >>> prof = ApprovalProfile([ApprovalBallot([p1, p2])] * 4) >>> results = run_all_experiments( - ... inst, prof, k=1, - ... predictors=("matrix_factorization", "factorization_machines"), + ... inst, prof, predictors=("matrix_factorization",), ... sample_degrees=(0.5,), lv_degrees=(0.5,), n_repeat=2, seed=0) - >>> len(results) == 5 * 2 # 5 setups x 2 predictors x 1 x 1 cells + >>> len(results) == 5 * 1 # 5 setups x 1 predictor x 1 x 1 cells True >>> all(0.0 <= c["FA"] <= 1.0 and c["SD"] >= 0 for c in results.values()) True + + Every cell carries Section 5.1's ballot metrics as well as Section 5.2's + bundle ones. + + >>> sorted(results[(0.5, 0.5, "random", "matrix_factorization")]) + ['FA', 'SD', 'f1', 'precision', 'recall'] """ setups, predictors = tuple(setups), tuple(predictors) sample_degrees, lv_degrees = tuple(sample_degrees), tuple(lv_degrees) @@ -1077,33 +1428,50 @@ def run_all_experiments( for setup in setups: for name in predictors: predict = PREDICTORS[name] - fa_sum = sd_sum = 0.0 + totals = dict.fromkeys( + ("FA", "SD", "precision", "recall", "f1"), 0.0 + ) for repeat in range(1, n_repeat + 1): - lv_profile, tv_ballots = split_lv_tv( - profile, sample_degree, lv_degree, + # The split derives k from the two degrees (Section 3.0.1). + lv_profile, tv_ballots, k = split_lv_tv( + instance, profile, sample_degree, lv_degree, seed=rng.randrange(2**32), ) - predicted = set( - run_pipeline( - instance, lv_profile, tv_ballots, k, - setup=setup, predict=predict, - seed=rng.randrange(2**32), - ) + completed, exposed = complete_ballots( + instance, lv_profile, tv_ballots, k, + setup=setup, predict=predict, + seed=rng.randrange(2**32), ) + combined = ApprovalProfile( + list(lv_profile) + + [completed[vid] for vid in tv_ballots] + ) + predicted = set(greedy_approval(instance, combined)) fa = fractional_allocation_score( real_bundle, predicted, instance.budget_limit ) # Symmetric Distance (Section 5.2.1): |rb △ pb|. sd = len(real_bundle ^ predicted) - fa_sum += fa - sd_sum += sd + totals["FA"] += fa + totals["SD"] += sd + # Section 5.1, over the votes that were predicted rather + # than answered, averaged across the Target Voters (a + # voter with no hidden approvals scores 0, the usual + # convention for an undefined precision or recall). + for vid in tv_ballots: + scores = classification_metrics( + set(tv_ballots[vid]), set(completed[vid]), + set(instance) - exposed[vid], + ) + for metric, value in scores.items(): + totals[metric] += value / max(len(tv_ballots), 1) logger.debug( "run_all_experiments: repeat %d/%d of " "(sample=%.2f, lv=%.2f, %s, %s): FA=%.3f SD=%d", repeat, n_repeat, sample_degree, lv_degree, setup, name, fa, sd, ) - cell = {"FA": fa_sum / n_repeat, "SD": sd_sum / n_repeat} + cell = {k2: v / n_repeat for k2, v in totals.items()} results[(sample_degree, lv_degree, setup, name)] = cell logger.info( "run_all_experiments: sample=%.2f lv=%.2f setup=%s " @@ -1188,7 +1556,10 @@ def classification_metrics( # --------------------------------------------------------------------------- # The Symmetric Distance (Section 5.2.1), |rb △ pb|, is a one-liner # ``len(real_bundle ^ predicted_bundle)`` computed inline where needed (see -# ``run_all_experiments``), so it gets no function of its own. +# ``run_all_experiments``), so it gets no function of its own. (The paper's +# second toy example says SD({1,2,3}, {1,3,4}) = 1; by its own definition the +# symmetric difference is {2, 4}, i.e. 2 - its first and third examples do +# match |rb △ pb|.) def fractional_allocation_score( real_bundle: Iterable[Project], predicted_bundle: Iterable[Project], @@ -1217,8 +1588,8 @@ def fractional_allocation_score( Examples -------- - Example 10: pb = rb = {p1, p2}, costs 3 and 3, budget 6, so FA = 6/6 = 1.0. - Example 11: disjoint bundles, so FA = 0/6 = 0.0. + pb = rb = {p1, p2}, costs 3 and 3, budget 6, so FA = 6/6 = 1.0. Disjoint + bundles give FA = 0/6 = 0.0. >>> p1, p2, p3 = Project("p1", 3), Project("p2", 3), Project("p3", 6) >>> fractional_allocation_score({p1, p2}, {p1, p2}, budget_limit=6) diff --git a/pb_model_training.py b/pb_model_training.py deleted file mode 100644 index cb81690..0000000 --- a/pb_model_training.py +++ /dev/null @@ -1,303 +0,0 @@ -""" -Model *training* for the learning-based prediction modules of -"A Recommendation System for Participatory Budgeting", -by Gil Leibiker and Nimrod Talmon (2023), https://optlearnmas23.github.io/files/p17.pdf -(Section 2.1). - -This module is intentionally separate from the module that *uses* the models -(``pb_recommendation``): a ``train_*`` function fits an estimator and returns an -opaque model object, while the matching ``predict_by_*`` function in -``pb_recommendation`` consumes it to complete a single Target Voter's partial -ballot. - -What each model trains on follows the paper (Section 3.1: predictions use the -preferences of LV *and* the preferences in E_TV): - -* :py:func:`train_classification` is supervised, so it trains on the **Learning - Voters' full ballots** (features = votes on the exposed projects, one binary - classifier per project). The Target Voter's exposed votes enter later, as - features, at prediction time. -* :py:func:`train_matrix_factorization` and - :py:func:`train_factorization_machines` are collaborative filtering, so the - Target Voter must be **inside** the fitted user-item matrix: they train on the - Learning Voters' full ballots **and** that Target Voter's exposed set E_v - (passed in as the ``approved`` / ``disapproved`` project sets). - -.. note:: - The paper backs these with ``xgboost`` (classification), ``scikit-surprise`` - (matrix factorization) and ``lightfm`` (factorization machines). Following - the maintainer's advice, the ML libraries are **not a hard requirement**: - ``xgboost`` is imported lazily inside :py:func:`train_classification`. The - ``scikit-surprise`` / ``lightfm`` wheels could not be built in this - environment (Windows + NumPy 2), so the matrix-factorization and - factorization-machines models are computed with a small, self-contained - NumPy factorization instead - ``numpy`` is already a pabutools dependency. - -This module is deliberately *ballot-agnostic*: it takes plain project sets, not -the partial CardinalBallot. The caller (``pb_recommendation``) decodes the ballot -into approved / disapproved / exposed sets and passes those in. - -Programmer: Roei Yanku -Date: 2026-06-20. -""" - -from __future__ import annotations - -import logging - -import numpy as np - -from pabutools.election import ( - Instance, - Project, - ApprovalProfile, - ApprovalBallot, -) - -logger = logging.getLogger(__name__) - - -# --------------------------------------------------------------------------- -# Section 2.1.1 - Binary classification (one classifier per project). -# --------------------------------------------------------------------------- -def train_classification( - instance: Instance, - lv_profile: ApprovalProfile, - exposed: set[Project], -) -> dict: - """ - Train the per-project binary classifiers of Section 2.1.1 on the LV ballots. - For every project the voter might be asked to predict, a classifier is fitted - whose features are the votes on the ``exposed`` projects and whose label is - the vote on the target project. Backed by the external ``xgboost`` library - (:py:class:`xgboost.XGBClassifier`, class-weighted for the imbalanced data), - imported lazily. When there are no exposed features, or all LV voters agree on - the target (a single class), no classifier can be trained, so we fall back to - the constant majority label. - - Parameters - ---------- - instance : :py:class:`~pabutools.election.instance.Instance` - The PB instance. - lv_profile : :py:class:`~pabutools.election.profile.approvalprofile.ApprovalProfile` - The full ballots of the LV voters, used as the training data. - exposed : set[:py:class:`~pabutools.election.instance.Project`] - The projects used as features (those exposed to the TV voters). - - Returns - ------- - dict - ``{"features": [...], "per_project": {project: ("const", 0/1) or - ("model", classifier)}}``, consumed by - :py:func:`pb_recommendation.predict_by_classification`. - - Examples - -------- - With nothing exposed and unanimous LV approvals, every project falls back to - the constant majority label (1 for the approved projects, 0 otherwise). - - >>> p1, p2 = Project("p1", 1), Project("p2", 1) - >>> inst = Instance([p1, p2], budget_limit=2) - >>> lv = ApprovalProfile([ApprovalBallot([p1])] * 3) - >>> model = train_classification(inst, lv, exposed=set()) - >>> model["per_project"][p1], model["per_project"][p2] - (('const', 1), ('const', 0)) - """ - features = sorted(exposed, key=str) - lv = list(lv_profile) - per_project: dict[Project, tuple] = {} - for target in instance: - labels = [1 if target in ballot else 0 for ballot in lv] - if not features or not lv or len(set(labels)) < 2: - # No features or a single class -> the majority label is the best we - # can do; a classifier cannot be trained. - approve = 1 if (labels and 2 * sum(labels) >= len(labels)) else 0 - per_project[target] = ("const", approve) - continue - try: - import xgboost - except ImportError: - raise ImportError( - "You need to install xgboost to train the classification " - "predictor (pip install pabutools[recommendation])." - ) - X = np.array([[1 if f in ballot else 0 for f in features] for ballot in lv]) - y = np.array(labels) - positives = int(y.sum()) - negatives = len(y) - positives - scale_pos_weight = (negatives / positives) if positives else 1.0 - classifier = xgboost.XGBClassifier( - n_estimators=50, max_depth=3, verbosity=0, - scale_pos_weight=scale_pos_weight, - ) - classifier.fit(X, y) - per_project[target] = ("model", classifier) - logger.info( - "train_classification: %d features, %d per-project classifiers", - len(features), len(per_project), - ) - return {"features": features, "per_project": per_project} - - -# --------------------------------------------------------------------------- -# Section 2.1.2 - Collaborative filtering via Matrix Factorization. -# --------------------------------------------------------------------------- -def _factorized_tv_scores( - instance: Instance, - lv_profile: ApprovalProfile, - approved: set[Project], - disapproved: set[Project], - rank: int = 20, -) -> dict[Project, float]: - """ - Shared collaborative-filtering core (Section 2.1.2). Builds the user-item - rating matrix - one row per LV voter (approve=1, else 0) plus one row for the - Target Voter (``approved`` -> 1, ``disapproved`` -> 0, unknown -> the item's - LV mean) - centres it by the item means, and reconstructs it from a truncated - SVD of rank ``rank``. Returns the reconstructed score of every project in the - Target Voter's row, i.e. the predicted approval level in [0, 1] (roughly). - - Examples - -------- - >>> p1, p2, p3 = Project("p1", 1), Project("p2", 1), Project("p3", 1) - >>> inst = Instance([p1, p2, p3], budget_limit=3) - >>> lv = ApprovalProfile([ApprovalBallot([p1, p2])] * 3) - >>> scores = _factorized_tv_scores(inst, lv, set(), set()) - >>> scores[p1] >= 0.5 and scores[p2] >= 0.5 and scores[p3] < 0.5 - True - """ - items = sorted(instance, key=str) - lv = list(lv_profile) - if lv: - lv_matrix = np.array( - [[1.0 if item in ballot else 0.0 for item in items] for ballot in lv] - ) - item_means = lv_matrix.mean(axis=0) - else: - lv_matrix = np.zeros((0, len(items))) - item_means = np.zeros(len(items)) - # The Target Voter's row: known exposed votes, unknown cells seeded with the - # item mean so the matrix is complete before factorising. - tv_row = np.array([ - 1.0 if item in approved else (0.0 if item in disapproved else item_means[j]) - for j, item in enumerate(items) - ]) - matrix = np.vstack([lv_matrix, tv_row]) - centered = matrix - item_means - u, singular, vt = np.linalg.svd(centered, full_matrices=False) - kept = min(rank, len(singular)) - reconstruction = (u[:, :kept] * singular[:kept]) @ vt[:kept] + item_means - tv_scores = reconstruction[-1] - return {item: float(tv_scores[j]) for j, item in enumerate(items)} - - -def train_matrix_factorization( - instance: Instance, - lv_profile: ApprovalProfile, - approved: set[Project], - disapproved: set[Project], -) -> dict[Project, float]: - """ - Train the Matrix Factorization model of Section 2.1.2. Collaborative filtering - needs the Target Voter inside the matrix, so the model is fitted on **both** - the Learning Voters' full ballots **and** the Target Voter's exposed set E_v - (``approved`` -> 1, ``disapproved`` -> 0). The user-item matrix is factorised - (truncated SVD) and the Target Voter's row is reconstructed; the score of each - hidden project is what gets thresholded at 0.5 at prediction time. Without her - exposed votes the model could only reproduce the LV average. - - .. note:: - The paper uses ``scikit-surprise`` (:py:class:`surprise.SVD`). That wheel - could not be built here (Windows + NumPy 2), so the factorization is done - with NumPy via :py:func:`_factorized_tv_scores`. - - Parameters - ---------- - instance : :py:class:`~pabutools.election.instance.Instance` - The PB instance. - lv_profile : :py:class:`~pabutools.election.profile.approvalprofile.ApprovalProfile` - The full ballots of the Learning Voters (LV). - approved : set[:py:class:`~pabutools.election.instance.Project`] - The Target Voter's exposed approvals A_v (known cells set to 1). - disapproved : set[:py:class:`~pabutools.election.instance.Project`] - The Target Voter's exposed disapprovals D_v (known cells set to 0). - - Returns - ------- - dict[:py:class:`~pabutools.election.instance.Project`, float] - The reconstructed approval score of every project for this Target - Voter, consumed by - :py:func:`pb_recommendation.predict_by_matrix_factorization`. - - Examples - -------- - >>> p1, p2, p3 = Project("p1", 1), Project("p2", 1), Project("p3", 1) - >>> inst = Instance([p1, p2, p3], budget_limit=3) - >>> lv = ApprovalProfile([ApprovalBallot([p1, p2])] * 3) - >>> scores = train_matrix_factorization(inst, lv, set(), set()) - >>> scores[p1] >= 0.5 and scores[p3] < 0.5 - True - """ - logger.info("train_matrix_factorization: factorising the user-item matrix") - return _factorized_tv_scores(instance, lv_profile, approved, disapproved) - - -# --------------------------------------------------------------------------- -# Section 2.1.2 - Hybrid Factorization Machines. -# --------------------------------------------------------------------------- -def train_factorization_machines( - instance: Instance, - lv_profile: ApprovalProfile, - approved: set[Project], - disapproved: set[Project], -) -> dict[Project, float]: - """ - Train the Factorization Machines model of Section 2.1.2: like Matrix - Factorization but with a linear (bias) term plus pairwise latent interactions. - It is fitted on **both** the Learning Voters' full ballots **and** the Target - Voter's exposed set E_v (``approved`` -> 1, ``disapproved`` -> 0). Centring the - matrix by the item means captures the linear per-item bias, and the truncated - SVD captures the pairwise latent interactions, so the same - :py:func:`_factorized_tv_scores` core is reused; the hidden projects are what - the fitted model predicts. - - .. note:: - The paper uses an external FM library (``lightfm`` / ``fastFM``). Those - wheels could not be built here (Windows + NumPy 2), so the model is - computed with the same NumPy factorization core. - - Parameters - ---------- - instance : :py:class:`~pabutools.election.instance.Instance` - The PB instance. - lv_profile : :py:class:`~pabutools.election.profile.approvalprofile.ApprovalProfile` - The full ballots of the Learning Voters (LV). - approved : set[:py:class:`~pabutools.election.instance.Project`] - The Target Voter's exposed approvals A_v (known cells set to 1). - disapproved : set[:py:class:`~pabutools.election.instance.Project`] - The Target Voter's exposed disapprovals D_v (known cells set to 0). - - Returns - ------- - dict[:py:class:`~pabutools.election.instance.Project`, float] - The predicted approval score of every project for this Target Voter, - consumed by - :py:func:`pb_recommendation.predict_by_factorization_machines`. - - Examples - -------- - >>> p1, p2, p3 = Project("p1", 1), Project("p2", 1), Project("p3", 1) - >>> inst = Instance([p1, p2, p3], budget_limit=3) - >>> lv = ApprovalProfile([ApprovalBallot([p1, p2])] * 3) - >>> scores = train_factorization_machines(inst, lv, set(), set()) - >>> scores[p1] >= 0.5 and scores[p3] < 0.5 - True - """ - logger.info("train_factorization_machines: fitting the FM model") - return _factorized_tv_scores(instance, lv_profile, approved, disapproved) - - -if __name__ == "__main__": - import doctest - - doctest.testmod(verbose=True) diff --git a/pyproject.toml b/pyproject.toml index 877fde7..d4942a7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,9 +40,11 @@ recommendation = [ # predict_by_classification: XGBClassifier needs xgboost's sklearn API. "xgboost", "scikit-learn", - # Matrix factorization / factorization machines run on numpy (already a - # core dependency): the scikit-surprise / lightfm wheels do not build on - # Windows + NumPy 2, so pb_model_training uses its own SVD core instead. + "pandas", + "scikit-surprise", + # lightfm-next is a drop-in, Python 3.12-compatible fork of LightFM. + "lightfm-next; python_version >= '3.12' and sys_platform != 'win32'", + "lightfm; python_version < '3.12'", ] dev = [ "coverage", @@ -59,4 +61,4 @@ dev = [ [project.urls] "Homepage" = "https://github.com/comsoc-community/pabutools" -"Bug Tracker" = "https://github.com/comsoc-community/pabutools/issues" \ No newline at end of file +"Bug Tracker" = "https://github.com/comsoc-community/pabutools/issues" diff --git a/test_pb_recommendation.py b/test_pb_recommendation.py deleted file mode 100644 index 95760b7..0000000 --- a/test_pb_recommendation.py +++ /dev/null @@ -1,389 +0,0 @@ -""" -Unit tests for `pb_recommendation`, the implementation of the algorithms in -"A Recommendation System for Participatory Budgeting", -by Gil Leibiker and Nimrod Talmon (2023), https://optlearnmas23.github.io/files/p17.pdf - -Run with: pytest test_pb_recommendation.py -v - -Each test is the specification its function must satisfy. They come in three -kinds: - * small, hand-checked instances (the examples from the paper); - * edge cases (partial ballots and invalid setup/k values); - * large inputs, checked against an independent reference or via invariants. - -Programmer: Roei Yanku -Date: 2026-06-20. -""" - -import random -from collections import Counter - -import pytest - -from pabutools.election import ( - Instance, - Project, - ApprovalProfile, - ApprovalBallot, -) - -from pb_recommendation import ( - # Partial-ballot helpers + the +1/-1/0 convention constants. - partial_ballot, - reveal_ballot, - approved_projects, - disapproved_projects, - exposed_projects, - hidden_projects, - consensus_levels, - greedy_approval, - random_setup, - offline_popularity, - offline_consensus, - offline_controversiality, - online_adaptive_controversial, - predict_by_classification, - predict_by_matrix_factorization, - predict_by_factorization_machines, - run_pipeline, - classification_metrics, - fractional_allocation_score, -) - - -# --------------------------------------------------------------------------- -# Helpers / fixtures -# --------------------------------------------------------------------------- -def make_projects(specs): - """specs: list of (name, cost) -> dict name -> Project.""" - return {name: Project(name, cost) for name, cost in specs} - - -@pytest.fixture -def consensus_data(): - """4 LV voters: p1 approved 4/0, p2 split 2/2, p3 rejected 0/4.""" - p = make_projects([("p1", 1), ("p2", 1), ("p3", 1)]) - inst = Instance(p.values(), budget_limit=3) - lv = ApprovalProfile( - [ - ApprovalBallot([p["p1"], p["p2"]]), - ApprovalBallot([p["p1"]]), - ApprovalBallot([p["p1"], p["p2"]]), - ApprovalBallot([p["p1"]]), - ] - ) - return p, inst, lv - - -def random_instance(n_projects, n_voters, budget, seed): - """Build a random instance + approval profile for property testing.""" - rng = random.Random(seed) - projects = [Project(f"p{i}", rng.randint(1, 10)) for i in range(n_projects)] - inst = Instance(projects, budget_limit=budget) - ballots = [] - for _ in range(n_voters): - approved = [p for p in projects if rng.random() < 0.3] - ballots.append(ApprovalBallot(approved)) - return projects, inst, ApprovalProfile(ballots) - - -def padded_projects(m, cost): - """m unit-named projects p000, p001, ... so str-order == index-order.""" - width = len(str(m - 1)) - return [Project(f"p{i:0{width}d}", cost) for i in range(m)] - - -def manual_scores(projects, profile): - """Independent (Counter-based) reimplementation of the approval scores.""" - counts = Counter() - for ballot in profile: - for proj in ballot: - counts[proj] += 1 - return {proj: counts[proj] for proj in projects} - - -def reference_greedy_approval(projects, profile, budget): - """ - Independent reference implementation of greedy approval, used to cross-check - `greedy_approval` on random inputs (the "compare to another algorithm" - strategy). Projects are taken in decreasing score, ties broken by name, and - funded while they fit the budget. - """ - counts = manual_scores(projects, profile) - ordered = sorted(projects, key=lambda p: (-counts[p], str(p))) - chosen, spent = [], 0 - for p in ordered: - if spent + p.cost <= budget: - chosen.append(p) - spent += p.cost - return chosen - - -# --------------------------------------------------------------------------- -# consensus_levels -# (Approval scores, Definition 2.1, have no function of our own - the code uses -# pabutools' profile.approval_scores() directly, which the library itself tests.) -# --------------------------------------------------------------------------- -class TestConsensusLevels: - def test_example7(self, consensus_data): - p, inst, lv = consensus_data - c = consensus_levels(inst, lv) - assert c[p["p1"]] == 4 - assert c[p["p2"]] == 0 - assert c[p["p3"]] == 4 - - def test_large_structured_even_split_is_zero(self): - # Exactly half approve and half reject each project -> consensus 0. - projects = padded_projects(20, cost=1) - inst = Instance(projects, budget_limit=10) - prof = ApprovalProfile( - [ApprovalBallot(projects)] * 50 + [ApprovalBallot([])] * 50 - ) - c = consensus_levels(inst, prof) - assert all(c[p] == 0 for p in projects) - - -# --------------------------------------------------------------------------- -# greedy_approval (voting rule) -# --------------------------------------------------------------------------- -class TestGreedyApproval: - def test_skips_unaffordable_but_funds_affordable(self): - # A project that exceeds the budget is skipped, while a cheaper one that - # fits is funded. - p = make_projects([("cheap", 10), ("dear", 100)]) - inst = Instance(p.values(), budget_limit=10) - prof = ApprovalProfile([ApprovalBallot([p["cheap"], p["dear"]])]) - assert set(greedy_approval(inst, prof)) == {p["cheap"]} - - def test_matches_reference_random(self): - # Cross-check against the independent greedy reference, several seeds. - for seed in range(5): - projects, inst, prof = random_instance(30, 50, 40, seed=200 + seed) - got = set(greedy_approval(inst, prof)) - expected = set(reference_greedy_approval(projects, prof, inst.budget_limit)) - assert got == expected - - -# --------------------------------------------------------------------------- -# random_setup -# --------------------------------------------------------------------------- -class TestRandomSetup: - def test_exposes_exactly_k(self): - p = make_projects([("p1", 2), ("p2", 2), ("p3", 3), ("p4", 3)]) - inst = Instance(p.values(), budget_limit=6) - exposed = random_setup(inst, k=2, seed=0) - assert len(exposed) == 2 - assert exposed <= set(p.values()) - -# --------------------------------------------------------------------------- -# offline_popularity / consensus / controversiality -# --------------------------------------------------------------------------- -class TestOfflineSamplers: - def test_popularity_example6(self): - p = make_projects([("p1", 1), ("p2", 1), ("p3", 1)]) - inst = Instance(p.values(), budget_limit=3) - lv = ApprovalProfile( - [ - ApprovalBallot([p["p1"], p["p3"]]), - ApprovalBallot([p["p1"], p["p2"]]), - ApprovalBallot([p["p1"]]), - ] - ) - assert offline_popularity(inst, lv, k=1) == {p["p1"]} - - def test_consensus_example7(self, consensus_data): - p, inst, lv = consensus_data - assert offline_consensus(inst, lv, k=1) == {p["p1"]} - - def test_controversiality_example8(self, consensus_data): - p, inst, lv = consensus_data - assert offline_controversiality(inst, lv, k=1) == {p["p2"]} - -# --------------------------------------------------------------------------- -# online_adaptive_controversial -# --------------------------------------------------------------------------- -class TestOnlineAdaptive: - def test_example9(self): - p = make_projects([("p1", 1), ("p2", 1), ("p3", 1), ("p4", 1)]) - inst = Instance(p.values(), budget_limit=4) - lv = ApprovalProfile( - [ - ApprovalBallot([p["p1"], p["p2"]]), - ApprovalBallot([p["p1"], p["p3"]]), - ApprovalBallot([p["p2"], p["p3"]]), - ApprovalBallot([]), - ] - ) - assert online_adaptive_controversial( - inst, lv, {p["p1"], p["p2"]}, k=2 - ) == {p["p1"], p["p2"]} - -# --------------------------------------------------------------------------- -# Partial (three-state) ballots, Section 2.3. Represented as a CardinalBallot -# under the convention: +1 approved, -1 disapproved, 0 (or absent) hidden. -# --------------------------------------------------------------------------- -class TestPartialBallot: - def test_scores_follow_convention(self): - # The whole point of the encoding: +1 = approve, -1 = disapprove, - # 0 = unknown. Pin it down so the convention can't silently drift. - p = make_projects([("p1", 1), ("p2", 1), ("p3", 1)]) - b = partial_ballot( - approved={p["p1"]}, disapproved={p["p2"]}, hidden={p["p3"]} - ) - assert b[p["p1"]] == 1 - assert b[p["p2"]] == -1 - assert b[p["p3"]] == 0 - - def test_absent_project_is_hidden(self): - # A project never mentioned in the ballot is hidden, just like an - # explicit 0 - both belong to H_v. - p = make_projects([("p1", 1), ("p2", 1), ("p3", 1)]) - inst = Instance(p.values(), budget_limit=3) - b = partial_ballot(approved={p["p1"]}, disapproved={p["p2"]}) # p3 absent - assert hidden_projects(b, inst) == {p["p3"]} - - def test_reveal_example2_4(self): - # Example 2.4: full ballot {p1,p2}, exposed set {p1,p3}. - p = make_projects([("p1", 1), ("p2", 1), ("p3", 1), ("p4", 1)]) - inst = Instance(p.values(), budget_limit=4) - b = reveal_ballot(inst, {p["p1"], p["p2"]}, {p["p1"], p["p3"]}) - assert approved_projects(b) == {p["p1"]} - assert disapproved_projects(b) == {p["p3"]} - assert hidden_projects(b, inst) == {p["p2"], p["p4"]} - assert exposed_projects(b) == {p["p1"], p["p3"]} - - -# --------------------------------------------------------------------------- -# The three prediction modules of the paper: classification (XGBoost), MF, FM -# (Section 2.1). They share one contract - keep the exposed votes, predict the -# hidden ones - so the same behavioural invariants are checked for each. -# --------------------------------------------------------------------------- -LIBRARY_PREDICTORS = [ - predict_by_classification, - predict_by_matrix_factorization, - predict_by_factorization_machines, -] - - -class TestLibraryPredictors: - @pytest.mark.parametrize("predictor", LIBRARY_PREDICTORS) - def test_exposed_votes_are_respected(self, predictor): - # The exposed approval p3 is kept and the exposed disapproval p1 stays - # rejected, regardless of what the model would otherwise predict. - p = make_projects([("p1", 1), ("p2", 1), ("p3", 1)]) - inst = Instance(p.values(), budget_limit=3) - lv = ApprovalProfile([ApprovalBallot([p["p1"]])] * 3) - partial = partial_ballot( - approved={p["p3"]}, disapproved={p["p1"]}, hidden={p["p2"]} - ) - pred = predictor(inst, lv, partial) - assert p["p3"] in pred # exposed approval kept - assert p["p1"] not in pred # exposed disapproval rejected - - -# --------------------------------------------------------------------------- -# run_pipeline (full pipeline) -# --------------------------------------------------------------------------- -class TestRunPipeline: - def test_unknown_setup_raises_value_error(self): - p = make_projects([("p1", 1), ("p2", 1)]) - inst = Instance(p.values(), budget_limit=2) - lv = ApprovalProfile([ApprovalBallot([p["p1"]])] * 2) - with pytest.raises(ValueError, match="unknown setup 'by_magic'"): - run_pipeline(inst, lv, {"v1": {p["p1"]}}, k=1, - setup="by_magic", - predict=predict_by_matrix_factorization) - - def test_k_out_of_range_raises_value_error(self): - p = make_projects([("p1", 1), ("p2", 1)]) - inst = Instance(p.values(), budget_limit=2) - lv = ApprovalProfile([ApprovalBallot([p["p1"]])] * 2) - for bad_k in (-1, 3): # below 0 and above the number of projects - with pytest.raises(ValueError, match="must be between 0 and"): - run_pipeline(inst, lv, {"v1": {p["p1"]}}, k=bad_k, - setup="random", - predict=predict_by_matrix_factorization) - - @pytest.mark.parametrize("setup", [ - "random", - "offline_popularity", - "offline_consensus", - "offline_controversiality", - "online_adaptive_controversial", - ]) - @pytest.mark.parametrize("predictor", LIBRARY_PREDICTORS) - def test_two_camp_electorate_recovers_real_bundle(self, setup, predictor): - """ - The end-to-end criterion of the paper (Section 2.4): the pipeline must - estimate the *ideal* outcome. A structured two-camp electorate - a 60% - majority camp approving {p1, p2, p3} and a 40% minority camp approving - {p4, p5, p6} - fixes the real winning bundle at {p1, p2, p3}. A third - of the voters become TV with only k=2 of the 6 projects exposed, and - every setup x predictor combination must still reconstruct the exact - real bundle (FA = 1.0, SD = 0). - """ - p = make_projects([(f"p{i}", 1) for i in range(1, 7)]) - camp_a = {p["p1"], p["p2"], p["p3"]} # 18 of 30 voters (60%) - camp_b = {p["p4"], p["p5"], p["p6"]} # 12 of 30 voters (40%) - inst = Instance(p.values(), budget_limit=3) - - # The real bundle, from all 30 full ballots: camp A's projects score - # 18 > 12, and the budget funds exactly three unit-cost projects. - full_profile = ApprovalProfile( - [ApprovalBallot(camp_a)] * 18 + [ApprovalBallot(camp_b)] * 12 - ) - real_bundle = set(greedy_approval(inst, full_profile)) - assert real_bundle == camp_a # sanity: the ground truth is as designed - - # LV/TV split preserving the 60/40 mix: 20 LV, 10 TV. - lv = ApprovalProfile([ApprovalBallot(camp_a)] * 12 - + [ApprovalBallot(camp_b)] * 8) - tv = {f"a{i}": set(camp_a) for i in range(6)} - tv.update({f"b{i}": set(camp_b) for i in range(4)}) - - predicted = set(run_pipeline(inst, lv, tv, k=2, - setup=setup, predict=predictor, seed=7)) - assert predicted == real_bundle - assert fractional_allocation_score( - real_bundle, predicted, inst.budget_limit) == 1.0 - assert len(real_bundle ^ predicted) == 0 # symmetric distance - - -# --------------------------------------------------------------------------- -# classification_metrics (Section 5.1) -# --------------------------------------------------------------------------- -class TestClassificationMetrics: - def test_mixed_hit_miss_false_alarm(self): - p = make_projects([("p1", 1), ("p2", 1), ("p3", 1), ("p4", 1)]) - hidden = set(p.values()) - # really approves {p1, p2}, predicted {p1, p3}: hit p1, miss p2, false p3. - m = classification_metrics({p["p1"], p["p2"]}, {p["p1"], p["p3"]}, hidden) - assert m == {"precision": 0.5, "recall": 0.5, "f1": 0.5} - - def test_exposed_votes_excluded(self): - # A wrong prediction on an *exposed* project must not affect the metrics: - # only the hidden set is scored. Hidden = {p2}, predicted perfectly there. - p = make_projects([("p1", 1), ("p2", 1)]) - m = classification_metrics( - real_approved={p["p2"]}, # p1 exposed, p2 hidden+approved - predicted_approved={p["p1"], p["p2"]}, - hidden={p["p2"]}, - ) - assert m == {"precision": 1.0, "recall": 1.0, "f1": 1.0} - -# --------------------------------------------------------------------------- -# fractional_allocation_score -# --------------------------------------------------------------------------- -class TestFractionalAllocation: - def test_partial_overlap(self): - p = make_projects([("p1", 2), ("p2", 3), ("p3", 5)]) - # overlap is {p2}, cost 3, budget 10 -> 0.3 - score = fractional_allocation_score( - {p["p1"], p["p2"]}, {p["p2"], p["p3"]}, budget_limit=10 - ) - assert score == pytest.approx(0.3) - -if __name__ == "__main__": - import sys - - sys.exit(pytest.main([__file__, "-v"])) diff --git a/tests/test_recommendation.py b/tests/test_recommendation.py new file mode 100644 index 0000000..52f322f --- /dev/null +++ b/tests/test_recommendation.py @@ -0,0 +1,665 @@ +""" +Unit tests for `pabutools.recommendation`, the implementation of the algorithms in +"A Recommendation System for Participatory Budgeting", +by Gil Leibiker and Nimrod Talmon (2023), https://optlearnmas23.github.io/files/p17.pdf + +Run with: python -m unittest tests.test_recommendation -v + +Each test is the specification its function must satisfy. They come in three +kinds: + * small, hand-checked instances (the examples from the paper); + * edge cases (partial ballots and invalid setup/k values); + * large inputs, checked against an independent reference or via invariants. + +Programmer: Roei Yanku +Date: 2026-06-20. +""" + +import random +from collections import Counter +from importlib.util import find_spec +from unittest import SkipTest, TestCase, skipUnless + +from parameterized import parameterized + +from pabutools.election import ( + Instance, + Project, + ApprovalProfile, + ApprovalBallot, + total_cost, +) + +from pabutools.recommendation import ( + # Partial-ballot helpers + the +1/-1/0 convention constants. + partial_ballot, + reveal_ballot, + approved_projects, + disapproved_projects, + exposed_projects, + hidden_projects, + consensus_levels, + greedy_approval, + random_setup, + offline_popularity, + offline_consensus, + offline_controversiality, + online_adaptive_controversial, + next_adaptive_question, + exposed_sets, + predict_by_classification, + predict_by_matrix_factorization, + predict_by_factorization_machines, + run_pipeline, + run_experiment, + split_lv_tv, + plan_sampling, + elect, + classification_metrics, + fractional_allocation_score, +) +from pabutools.recommendation.model_training import ( + train_factorization_machines, + train_matrix_factorization, +) + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- +def make_projects(specs): + """specs: list of (name, cost) -> dict name -> Project.""" + return {name: Project(name, cost) for name, cost in specs} + + +# The three prediction modules are backed by the optional ``recommendation`` +# extra, which the library's own CI does not install, so a test that fits a +# model reports "skipped" there instead of failing. Everything that does not +# need a model - the samplers, the ballot algebra, the pipeline's validation, +# the metrics and the voting rule - runs unconditionally. +def _installed(module): + try: + return find_spec(module) is not None + except (ImportError, ValueError): + return False + + +PREDICTOR_LIBRARY = { + "predict_by_classification": "xgboost", + "predict_by_matrix_factorization": "surprise", + "predict_by_factorization_machines": "lightfm", +} + + +def requires(*modules): + """Skip the decorated test unless every named library is installed.""" + missing = [module for module in modules if not _installed(module)] + return skipUnless( + not missing, + f"needs {', '.join(missing)} (pip install pabutools[recommendation])", + ) + + +def require_predictor(predictor): + """Skip the running test when this predictor's library is absent. + + The per-predictor counterpart of :py:func:`requires`, for the tests + ``parameterized`` expands over :py:data:`LIBRARY_PREDICTORS`: which + library a case needs is only known once the case is running. + """ + module = PREDICTOR_LIBRARY[predictor.__name__] + if not _installed(module): + raise SkipTest( + f"needs {module} (pip install pabutools[recommendation])" + ) + + +def consensus_data(): + """4 LV voters: p1 approved 4/0, p2 split 2/2, p3 rejected 0/4.""" + p = make_projects([("p1", 1), ("p2", 1), ("p3", 1)]) + inst = Instance(p.values(), budget_limit=3) + lv = ApprovalProfile( + [ + ApprovalBallot([p["p1"], p["p2"]]), + ApprovalBallot([p["p1"]]), + ApprovalBallot([p["p1"], p["p2"]]), + ApprovalBallot([p["p1"]]), + ] + ) + return p, inst, lv + + +def random_instance(n_projects, n_voters, budget, seed): + """Build a random instance + approval profile for property testing.""" + rng = random.Random(seed) + projects = [Project(f"p{i}", rng.randint(1, 10)) for i in range(n_projects)] + inst = Instance(projects, budget_limit=budget) + ballots = [] + for _ in range(n_voters): + approved = [p for p in projects if rng.random() < 0.3] + ballots.append(ApprovalBallot(approved)) + return projects, inst, ApprovalProfile(ballots) + + +def padded_projects(m, cost): + """m unit-named projects p000, p001, ... so str-order == index-order.""" + width = len(str(m - 1)) + return [Project(f"p{i:0{width}d}", cost) for i in range(m)] + + +def manual_scores(projects, profile): + """Independent (Counter-based) reimplementation of the approval scores.""" + counts = Counter() + for ballot in profile: + for proj in ballot: + counts[proj] += 1 + return {proj: counts[proj] for proj in projects} + + +def reference_greedy_approval(projects, profile, budget): + """ + Independent reference implementation of greedy approval, used to cross-check + `greedy_approval` on random inputs (the "compare to another algorithm" + strategy). Projects are taken in decreasing score, ties broken by name, and + funded while they fit the budget. + """ + counts = manual_scores(projects, profile) + ordered = sorted(projects, key=lambda p: (-counts[p], str(p))) + chosen, spent = [], 0 + for p in ordered: + if spent + p.cost <= budget: + chosen.append(p) + spent += p.cost + return chosen + + +# --------------------------------------------------------------------------- +# consensus_levels +# (Approval scores, Definition 2.1, have no function of our own - the code uses +# pabutools' profile.approval_scores() directly, which the library itself tests.) +# --------------------------------------------------------------------------- +class TestConsensusLevels(TestCase): + def test_hand_checked_split(self): + p, inst, lv = consensus_data() + c = consensus_levels(inst, lv) + assert c[p["p1"]] == 4 + assert c[p["p2"]] == 0 + assert c[p["p3"]] == 4 + + def test_large_structured_even_split_is_zero(self): + # Exactly half approve and half reject each project -> consensus 0. + projects = padded_projects(20, cost=1) + inst = Instance(projects, budget_limit=10) + prof = ApprovalProfile( + [ApprovalBallot(projects)] * 50 + [ApprovalBallot([])] * 50 + ) + c = consensus_levels(inst, prof) + assert all(c[p] == 0 for p in projects) + + +# --------------------------------------------------------------------------- +# greedy_approval (voting rule) +# --------------------------------------------------------------------------- +class TestGreedyApproval(TestCase): + def test_skips_unaffordable_but_funds_affordable(self): + # A project that exceeds the budget is skipped, while a cheaper one that + # fits is funded. + p = make_projects([("cheap", 10), ("dear", 100)]) + inst = Instance(p.values(), budget_limit=10) + prof = ApprovalProfile([ApprovalBallot([p["cheap"], p["dear"]])]) + assert set(greedy_approval(inst, prof)) == {p["cheap"]} + + def test_matches_reference_random(self): + # Cross-check against the independent greedy reference, several seeds. + for seed in range(5): + projects, inst, prof = random_instance(30, 50, 40, seed=200 + seed) + got = set(greedy_approval(inst, prof)) + expected = set(reference_greedy_approval(projects, prof, inst.budget_limit)) + assert got == expected + + +# --------------------------------------------------------------------------- +# random_setup +# --------------------------------------------------------------------------- +class TestRandomSetup(TestCase): + def test_exposes_k_independently_per_voter(self): + # One voter's draw: exactly k real projects. + p = make_projects([("p1", 2), ("p2", 2), ("p3", 3), ("p4", 3)]) + inst = Instance(p.values(), budget_limit=6) + exposed = random_setup(inst, k=2, seed=0) + assert len(exposed) == 2 and exposed <= set(p.values()) + + # Section 2.4: "possibly different E for each voter" - one seed must + # not collapse the whole population onto a single shared random set, + # yet the same seed must reproduce the same draws. + projects = padded_projects(8, cost=1) + inst = Instance(projects, budget_limit=8) + lv = ApprovalProfile([ApprovalBallot(projects[:2])]) + tv = {f"v{i}": set() for i in range(12)} + exposed = exposed_sets(inst, lv, tv, "random", k=2, seed=123) + assert all(len(e) == 2 for e in exposed.values()) + assert len({frozenset(e) for e in exposed.values()}) > 1 + assert exposed == exposed_sets(inst, lv, tv, "random", k=2, seed=123) + +# --------------------------------------------------------------------------- +# offline_popularity / consensus / controversiality +# --------------------------------------------------------------------------- +class TestOfflineSamplers(TestCase): + def test_popularity_top_k(self): + p = make_projects([("p1", 1), ("p2", 1), ("p3", 1)]) + inst = Instance(p.values(), budget_limit=3) + lv = ApprovalProfile( + [ + ApprovalBallot([p["p1"], p["p3"]]), + ApprovalBallot([p["p1"], p["p2"]]), + ApprovalBallot([p["p1"]]), + ] + ) + assert offline_popularity(inst, lv, k=1) == {p["p1"]} + + def test_consensus_top_k(self): + p, inst, lv = consensus_data() + assert offline_consensus(inst, lv, k=1) == {p["p1"]} + + def test_controversiality_bottom_k(self): + p, inst, lv = consensus_data() + assert offline_controversiality(inst, lv, k=1) == {p["p2"]} + +# --------------------------------------------------------------------------- +# online_adaptive_controversial +# --------------------------------------------------------------------------- +class TestOnlineAdaptive(TestCase): + def test_adaptive_feedback_shifts_the_questions(self): + # One TV voter: p1, p2, p3 are tied as most controversial (consensus 0), + # ties broken by name, so the two questions are p1 then p2. + p = make_projects([("p1", 1), ("p2", 1), ("p3", 1), ("p4", 1)]) + inst = Instance(p.values(), budget_limit=4) + lv = ApprovalProfile( + [ + ApprovalBallot([p["p1"], p["p2"]]), + ApprovalBallot([p["p1"], p["p3"]]), + ApprovalBallot([p["p2"], p["p3"]]), + ApprovalBallot([]), + ] + ) + assert online_adaptive_controversial( + inst, lv, {"v5": {p["p1"], p["p2"]}}, k=2 + ) == {"v5": {p["p1"], p["p2"]}} + + # The feedback loop across voters. p1 and p2 start tied as most + # controversial (both split 1-1 among LV), so the first voter is + # always asked p1, and her answer - approval *or* disapproval - breaks + # the tie, steering the second voter to p2. + p = make_projects([("p1", 1), ("p2", 1), ("p3", 1)]) + inst = Instance(p.values(), budget_limit=3) + lv = ApprovalProfile([ApprovalBallot([p["p1"]]), ApprovalBallot([p["p2"]])]) + assert online_adaptive_controversial( + inst, lv, {"v3": {p["p1"]}, "v4": {p["p1"]}}, k=1 + ) == {"v3": {p["p1"]}, "v4": {p["p2"]}} + assert online_adaptive_controversial( + inst, lv, {"v3": set(), "v4": {p["p1"]}}, k=1 + ) == {"v3": {p["p1"]}, "v4": {p["p2"]}} + + # And across more than one step: after v3 approves p1 (2-1) and v4 + # approves p2 (2-1), both are equally off-split again for v5 and the + # name tie-break returns to p1. + assert online_adaptive_controversial( + inst, lv, {"v3": {p["p1"]}, "v4": {p["p2"]}, "v5": set()}, k=1 + ) == {"v3": {p["p1"]}, "v4": {p["p2"]}, "v5": {p["p1"]}} + + def test_step_function_reproduces_the_closed_loop(self): + # next_adaptive_question is the open-loop form of the same rule, for a + # live process where the answers arrive one at a time. Driving it with + # a voter's ballot must ask exactly what the closed loop asks. + rng = random.Random(0) + for _ in range(60): + n_projects, n_lv = rng.randint(3, 7), rng.randint(2, 6) + k = rng.randint(1, n_projects) + p = make_projects([(f"p{i}", 1) for i in range(n_projects)]) + projects = list(p.values()) + inst = Instance(projects, budget_limit=n_projects) + lv = ApprovalProfile([ + ApprovalBallot([q for q in projects if rng.random() < 0.5]) + for _ in range(n_lv) + ]) + truth = {q for q in projects if rng.random() < 0.5} + + closed = online_adaptive_controversial(inst, lv, {"v": truth}, k)["v"] + + answers = partial_ballot() + for _ in range(k): + asked = next_adaptive_question(inst, lv, answers) + approved = approved_projects(answers) + disapproved = disapproved_projects(answers) + if asked in truth: + approved = approved | {asked} + else: + disapproved = disapproved | {asked} + answers = partial_ballot( + approved=approved, disapproved=disapproved + ) + assert exposed_projects(answers) == closed + + def test_step_function_rejects_a_fully_answered_ballot(self): + p = make_projects([("p1", 1), ("p2", 1)]) + inst = Instance(p.values(), budget_limit=2) + lv = ApprovalProfile([ApprovalBallot([p["p1"]])]) + answered = partial_ballot(approved={p["p1"]}, disapproved={p["p2"]}) + with self.assertRaisesRegex(ValueError, "already been asked"): + next_adaptive_question(inst, lv, answered) + +# --------------------------------------------------------------------------- +# Partial (three-state) ballots, Section 2.3. Represented as a CardinalBallot +# under the convention: +1 approved, -1 disapproved, 0 (or absent) hidden. +# --------------------------------------------------------------------------- +class TestPartialBallot(TestCase): + def test_scores_follow_convention(self): + # The whole point of the encoding: +1 = approve, -1 = disapprove, + # 0 = unknown. Pin it down so the convention can't silently drift. + p = make_projects([("p1", 1), ("p2", 1), ("p3", 1)]) + b = partial_ballot( + approved={p["p1"]}, disapproved={p["p2"]}, hidden={p["p3"]} + ) + assert b[p["p1"]] == 1 + assert b[p["p2"]] == -1 + assert b[p["p3"]] == 0 + + def test_absent_project_is_hidden(self): + # A project never mentioned in the ballot is hidden, just like an + # explicit 0 - both belong to H_v. + p = make_projects([("p1", 1), ("p2", 1), ("p3", 1)]) + inst = Instance(p.values(), budget_limit=3) + b = partial_ballot(approved={p["p1"]}, disapproved={p["p2"]}) # p3 absent + assert hidden_projects(b, inst) == {p["p3"]} + + def test_reveal_example2_4(self): + # Example 2.4: full ballot {p1,p2}, exposed set {p1,p3}. + p = make_projects([("p1", 1), ("p2", 1), ("p3", 1), ("p4", 1)]) + inst = Instance(p.values(), budget_limit=4) + b = reveal_ballot(inst, {p["p1"], p["p2"]}, {p["p1"], p["p3"]}) + assert approved_projects(b) == {p["p1"]} + assert disapproved_projects(b) == {p["p3"]} + assert hidden_projects(b, inst) == {p["p2"], p["p4"]} + assert exposed_projects(b) == {p["p1"], p["p3"]} + + +# --------------------------------------------------------------------------- +# The three prediction modules of the paper: classification (XGBoost), MF, FM +# (Section 2.1). They share one contract - keep the exposed votes, predict the +# hidden ones - so the same behavioural invariants are checked for each. +# --------------------------------------------------------------------------- +LIBRARY_PREDICTORS = [ + predict_by_classification, + predict_by_matrix_factorization, + predict_by_factorization_machines, +] + + +class TestLibraryPredictors(TestCase): + @parameterized.expand([(p.__name__, p) for p in LIBRARY_PREDICTORS]) + def test_exposed_votes_are_respected(self, _name, predictor): + # The exposed approval p3 is kept and the exposed disapproval p1 stays + # rejected, regardless of what the model would otherwise predict. + require_predictor(predictor) + p = make_projects([("p1", 1), ("p2", 1), ("p3", 1)]) + inst = Instance(p.values(), budget_limit=3) + lv = ApprovalProfile([ApprovalBallot([p["p1"]])] * 3) + partial = partial_ballot( + approved={p["p3"]}, disapproved={p["p1"]}, hidden={p["p2"]} + ) + pred = predictor(inst, lv, {"v": partial})["v"] + assert p["p3"] in pred # exposed approval kept + assert p["p1"] not in pred # exposed disapproval rejected + + @requires("surprise", "lightfm") + def test_cf_pools_e_tv_and_fm_uses_the_cost_feature(self): + # Section 3.1: prediction is based on LV *and* E_TV, the exposed votes + # of all TV voters. The lone LV voter rejects p2, but five TV voters + # expose approvals of it; a sixth TV voter with nothing exposed must + # pick p2 up from their pooled answers. + p = make_projects([("p1", 1), ("p2", 1)]) + inst = Instance(p.values(), budget_limit=2) + lv = ApprovalProfile([ApprovalBallot([p["p1"]])]) + tv = { + f"b{i}": partial_ballot(approved={p["p2"]}, hidden={p["p1"]}) + for i in range(5) + } + tv["a"] = partial_ballot(hidden=set(p.values())) + completed = predict_by_matrix_factorization(inst, lv, tv) + assert p["p2"] in completed["a"] + + # The FM hybrid ingredient: the voter approves her exposed cheap + # projects and rejects the exposed expensive ones; the fitted cost + # slope carries that pattern to the hidden pair. Plain MF has no cost + # feature and can only reach the same answer through the e1/e2/e3 + # correlation in the LV ballots, so it separates the two far more + # weakly - which is exactly how FM and MF must differ. + p = make_projects( + [("c1", 1), ("c2", 1), ("c3", 1), ("e1", 10), ("e2", 10), ("e3", 10)] + ) + inst = Instance(p.values(), budget_limit=12) + cheap = {p["c1"], p["c2"], p["c3"]} + lv = ApprovalProfile([ApprovalBallot(p.values())] * 3 + + [ApprovalBallot(cheap)] * 2) + partial = partial_ballot( + approved={p["c1"], p["c2"]}, + disapproved={p["e1"], p["e2"]}, + hidden={p["c3"], p["e3"]}, + ) + fm = predict_by_factorization_machines(inst, lv, {"v": partial})["v"] + assert p["c3"] in fm and p["e3"] not in fm + # Compare the cheap/expensive score gaps rather than the completed + # ballots: whether MF's e3 lands either side of the 0.5 cut-off depends + # on its latent rank, but its gap is always the weaker of the two. + exposed = {"v": ({p["c1"], p["c2"]}, {p["e1"], p["e2"]})} + mf_scores = train_matrix_factorization(inst, lv, exposed)["v"] + fm_scores = train_factorization_machines(inst, lv, exposed)["v"] + gap = lambda s: s[p["c3"]] - s[p["e3"]] # noqa: E731 + assert gap(fm_scores) > gap(mf_scores) + + @requires("lightfm") + def test_fm_uses_the_pabulib_category_and_target_features(self): + # The hybrid ingredient of Section 2.1.2, on Table 2's *set-valued* + # project attributes rather than cost. Every project costs the same and + # every LV voter approves everything, so neither cost nor the approval + # scores can separate them: the only thing that can carry this voter's + # exposed pattern to the projects she was never asked about is the + # category / target metadata that ``parse_pabulib`` fills in. + p = make_projects([(name, 1) for name in + ("g1", "g2", "g3", "r1", "r2", "r3")]) + for name in ("g1", "g2", "g3"): + p[name].categories, p[name].targets = {"greenery"}, {"children"} + for name in ("r1", "r2", "r3"): + p[name].categories, p[name].targets = {"roads"}, {"drivers"} + inst = Instance(p.values(), budget_limit=6) + lv = ApprovalProfile([ApprovalBallot(p.values())] * 4) + + # She approves the greenery she was shown and rejects the roads. + exposed = {"v": ({p["g1"], p["g2"]}, {p["r1"], p["r2"]})} + scores = train_factorization_machines(inst, lv, exposed) + # So the unseen greenery project must outscore the unseen road one. + assert scores["v"][p["g3"]] > scores["v"][p["r3"]] + + +# --------------------------------------------------------------------------- +# run_pipeline (full pipeline) +# (split_lv_tv's Section 3.0.1 vote-budget arithmetic - TV = all non-LV +# voters, derived k, the lv_degree=1 baseline, degree validation - is pinned +# by its doctests in pabutools/recommendation/recommendation.py.) +# --------------------------------------------------------------------------- +class TestRunPipeline(TestCase): + def test_invalid_setup_or_k_raises_value_error(self): + p = make_projects([("p1", 1), ("p2", 1)]) + inst = Instance(p.values(), budget_limit=2) + lv = ApprovalProfile([ApprovalBallot([p["p1"]])] * 2) + with self.assertRaisesRegex(ValueError, "unknown setup 'by_magic'"): + run_pipeline(inst, lv, {"v1": {p["p1"]}}, k=1, + setup="by_magic", + predict=predict_by_matrix_factorization) + for bad_k in (-1, 3): # below 0 and above the number of projects + with self.assertRaisesRegex(ValueError, "must be between 0 and"): + run_pipeline(inst, lv, {"v1": {p["p1"]}}, k=bad_k, + setup="random", + predict=predict_by_matrix_factorization) + + @parameterized.expand([ + (f"{setup}_{predictor.__name__}", setup, predictor) + for setup in ( + "random", + "offline_popularity", + "offline_consensus", + "offline_controversiality", + "online_adaptive_controversial", + ) + for predictor in LIBRARY_PREDICTORS + ]) + def test_two_camp_electorate_recovers_real_bundle(self, _name, setup, predictor): + """ + The end-to-end criterion of the paper (Section 2.4): the pipeline must + estimate the *ideal* outcome. A structured two-camp electorate - a 60% + majority camp approving {p1, p2, p3} and a 40% minority camp approving + {p4, p5, p6} - fixes the real winning bundle at {p1, p2, p3}. A third + of the voters become TV with only k=2 of the 6 projects exposed, and + every setup x predictor combination must still reconstruct the exact + real bundle (FA = 1.0, SD = 0). + """ + require_predictor(predictor) + p = make_projects([(f"p{i}", 1) for i in range(1, 7)]) + camp_a = {p["p1"], p["p2"], p["p3"]} # 18 of 30 voters (60%) + camp_b = {p["p4"], p["p5"], p["p6"]} # 12 of 30 voters (40%) + inst = Instance(p.values(), budget_limit=3) + + # The real bundle, from all 30 full ballots: camp A's projects score + # 18 > 12, and the budget funds exactly three unit-cost projects. + full_profile = ApprovalProfile( + [ApprovalBallot(camp_a)] * 18 + [ApprovalBallot(camp_b)] * 12 + ) + real_bundle = set(greedy_approval(inst, full_profile)) + assert real_bundle == camp_a # sanity: the ground truth is as designed + + # LV/TV split preserving the 60/40 mix: 20 LV, 10 TV. + lv = ApprovalProfile([ApprovalBallot(camp_a)] * 12 + + [ApprovalBallot(camp_b)] * 8) + tv = {f"a{i}": set(camp_a) for i in range(6)} + tv.update({f"b{i}": set(camp_b) for i in range(4)}) + + predicted = set(run_pipeline(inst, lv, tv, k=2, + setup=setup, predict=predictor, seed=7)) + assert predicted == real_bundle + assert fractional_allocation_score( + real_bundle, predicted, inst.budget_limit) == 1.0 + assert len(real_bundle ^ predicted) == 0 # symmetric distance + + +# --------------------------------------------------------------------------- +# classification_metrics (Section 5.1) +# --------------------------------------------------------------------------- +class TestClassificationMetrics(TestCase): + def test_mixed_hit_miss_false_alarm(self): + p = make_projects([("p1", 1), ("p2", 1), ("p3", 1), ("p4", 1)]) + hidden = set(p.values()) + # really approves {p1, p2}, predicted {p1, p3}: hit p1, miss p2, false p3. + m = classification_metrics({p["p1"], p["p2"]}, {p["p1"], p["p3"]}, hidden) + assert m == {"precision": 0.5, "recall": 0.5, "f1": 0.5} + + def test_exposed_votes_excluded(self): + # A wrong prediction on an *exposed* project must not affect the metrics: + # only the hidden set is scored. Hidden = {p2}, predicted perfectly there. + p = make_projects([("p1", 1), ("p2", 1)]) + m = classification_metrics( + real_approved={p["p2"]}, # p1 exposed, p2 hidden+approved + predicted_approved={p["p1"], p["p2"]}, + hidden={p["p2"]}, + ) + assert m == {"precision": 1.0, "recall": 1.0, "f1": 1.0} + +# --------------------------------------------------------------------------- +# fractional_allocation_score +# --------------------------------------------------------------------------- +class TestFractionalAllocation(TestCase): + def test_partial_overlap(self): + p = make_projects([("p1", 2), ("p2", 3), ("p3", 5)]) + # overlap is {p2}, cost 3, budget 10 -> 0.3 + score = fractional_allocation_score( + {p["p1"], p["p2"]}, {p["p2"], p["p3"]}, budget_limit=10 + ) + self.assertAlmostEqual(float(score), 0.3) + + +# --------------------------------------------------------------------------- +# Running a real process: plan_sampling / elect / run_experiment +# --------------------------------------------------------------------------- +class TestRealProcess(TestCase): + def test_plan_sampling_matches_the_split_it_describes(self): + # plan_sampling is the arithmetic split_lv_tv performs, minus the + # ballots, so the two must agree on every point of the paper's grid. + p = make_projects([(f"p{i}", 1) for i in range(4)]) + inst = Instance(p.values(), budget_limit=4) + prof = ApprovalProfile([ApprovalBallot(set(p.values()))] * 10) + for sample_degree in (0.1, 0.3, 0.5, 0.9): + for lv_degree in (0.1, 0.5, 0.9, 1.0): + n_lv, k = plan_sampling(10, 4, sample_degree, lv_degree) + lv, tv, split_k = split_lv_tv( + inst, prof, sample_degree, lv_degree, seed=1 + ) + assert (lv.num_ballots(), split_k) == (n_lv, k) + assert len(tv) == (0 if lv_degree == 1 else 10 - n_lv) + + @requires("xgboost") + def test_elect_needs_no_ground_truth(self): + # The deployment path: nobody's full ballot is known to the algorithm - + # there are only the answers voters actually gave. Two camps disagree + # about p1 against p3, every voter was asked about just those two of + # the four projects, and the budget funds exactly one of them. + p = make_projects([("p1", 1), ("p2", 1), ("p3", 1), ("p4", 1)]) + inst = Instance(p.values(), budget_limit=1) + answers = {} + for i in range(12): # majority camp: approve p1/p2, reject p3/p4 + answers[f"a{i}"] = partial_ballot( + approved={p["p1"]}, disapproved={p["p3"]} + ) + for i in range(5): # minority camp: the other way round + answers[f"b{i}"] = partial_ballot( + approved={p["p3"]}, disapproved={p["p1"]} + ) + bundle = elect(inst, ApprovalProfile([]), answers) + assert total_cost(bundle) <= inst.budget_limit + assert p["p1"] in bundle and p["p3"] not in bundle + + @requires("xgboost") + def test_elect_keeps_every_answer_that_was_given(self): + # Whatever the model thinks, an answer a voter actually gave must + # survive into her counted ballot (Section 2.4: the completed instance + # has to agree with the partial one). + p = make_projects([("p1", 1), ("p2", 1), ("p3", 1)]) + inst = Instance(p.values(), budget_limit=3) + lv = ApprovalProfile([ApprovalBallot([p["p1"], p["p2"]])] * 4) + # She rejects p1 although every full ballot approves it. + answers = {"v": partial_ballot(approved={p["p3"]}, disapproved={p["p1"]})} + completed = predict_by_classification(inst, lv, answers)["v"] + assert p["p3"] in completed and p["p1"] not in completed + + @requires("xgboost") + def test_run_experiment_equals_split_then_pipeline(self): + # The convenience wrapper must be exactly its two parts, same seed. + p = make_projects([("p1", 2), ("p2", 2), ("p3", 3), ("p4", 3)]) + inst = Instance(p.values(), budget_limit=4) + prof = ApprovalProfile([ApprovalBallot([p["p1"], p["p2"]])] * 6) + lv, tv, k = split_lv_tv(inst, prof, 0.5, 0.3, seed=11) + expected = set(run_pipeline(inst, lv, tv, k, setup="offline_popularity", + predict=predict_by_classification, seed=11)) + got = set(run_experiment(inst, prof, 0.5, 0.3, + setup="offline_popularity", seed=11)) + assert got == expected + + def test_unknown_predictor_name_is_rejected(self): + p = make_projects([("p1", 1)]) + inst = Instance(p.values(), budget_limit=1) + with self.assertRaisesRegex(ValueError, "unknown predictor"): + elect(inst, ApprovalProfile([]), {}, predictor="by_magic") + + +if __name__ == "__main__": + import unittest + + unittest.main() From 66e5736ca89ab49209aeb65aa8e4140f4e7ab7fd Mon Sep 17 00:00:00 2001 From: Roei Date: Wed, 29 Jul 2026 01:42:11 +0300 Subject: [PATCH 16/16] Add a usage page for the recommendation module Every other module in the library has a narrative usage page alongside its API reference; this one only had the reference. The page walks through the two ways the module is used - running a process from partial ballots with elect, and measuring the loss on a known profile with run_experiment - with runnable examples and cross-references into the reference pages. Building the docs to check it also surfaced two markup errors in the module's own docstrings, fixed here: * |LV| and |TV| are substitution syntax in reStructuredText, so the cardinality notation in plan_sampling and split_lv_tv raised "Undefined substitution referenced" and never rendered; they are literals now. * the cross-reference to greedy_utilitarian_welfare pointed at the re-export path rather than pabutools.rules.greedywelfare, where Sphinx documents it. --- docs-source/source/usage/index.rst | 1 + docs-source/source/usage/recommendation.rst | 189 ++++++++++++++++++++ pabutools/recommendation/recommendation.py | 8 +- 3 files changed, 194 insertions(+), 4 deletions(-) create mode 100644 docs-source/source/usage/recommendation.rst diff --git a/docs-source/source/usage/index.rst b/docs-source/source/usage/index.rst index 2e9922d..f2e5606 100644 --- a/docs-source/source/usage/index.rst +++ b/docs-source/source/usage/index.rst @@ -15,5 +15,6 @@ refer to the :ref:`quickstart` page. rules outcomevisualisation analysis + recommendation tiebreaking fractions diff --git a/docs-source/source/usage/recommendation.rst b/docs-source/source/usage/recommendation.rst new file mode 100644 index 0000000..a8955d7 --- /dev/null +++ b/docs-source/source/usage/recommendation.rst @@ -0,0 +1,189 @@ +Recommendation +============== + +For reference, see the module :py:mod:`~pabutools.recommendation`. + +When a participatory budgeting instance has many projects, asking every voter about every +project is a burden, and ballots are often left unfinished. This module implements the +approach of Leibiker and Talmon (2023): ask each voter about only a few projects, estimate +the votes that were never asked for, and run a voting rule on the completed profile. + +There are two ways to use it. If you are running a process and only hold partial ballots, +:py:func:`~pabutools.recommendation.recommendation.elect` returns an outcome. If you hold +complete ballots and want to know what asking less would have cost you, +:py:func:`~pabutools.recommendation.recommendation.run_experiment` simulates the loss. + +Partial Ballots +--------------- + +A voter who was asked about only some of the projects has three kinds of opinion: projects +she approved, projects she rejected, and projects she was never shown. This is stored in a +:py:class:`~pabutools.election.ballot.cardinalballot.CardinalBallot` under the convention +*positive means approved, negative means disapproved, zero or absent means not asked*. +:py:func:`~pabutools.recommendation.recommendation.partial_ballot` builds one: + +.. code-block:: python + + from pabutools.election import Instance, Project, ApprovalProfile, ApprovalBallot + from pabutools.recommendation import partial_ballot + + garden = Project("Garden", 18) + crossings = Project("Crossings", 24) + library = Project("Library", 16) + shade = Project("Shade", 12) + instance = Instance([garden, crossings, library, shade], budget_limit=40) + + # A voter who approved the garden, rejected the crossings, and was asked nothing else. + ballot = partial_ballot(approved={garden}, disapproved={crossings}) + +Running a Process +----------------- + +:py:func:`~pabutools.recommendation.recommendation.elect` + +Some voters do provide a complete ballot; they are the *learning voters*, and their profile +is what the estimation learns from. Every other voter is asked about a limited number of +projects. + +Which projects to ask about is decided by a *sampling setup*. The offline setups choose one +set for everybody, out of the complete ballots already collected: + +.. code-block:: python + + from pabutools.recommendation import offline_controversiality + + lv_profile = ApprovalProfile([ + ApprovalBallot([garden, library]), + ApprovalBallot([crossings, shade]), + ApprovalBallot([garden, shade]), + ApprovalBallot([garden, library, shade]), + ]) + + offline_controversiality(instance, lv_profile, k=2) + # {Library, Shade} - the two projects the learning voters most disagree about + +The alternatives are :py:func:`~pabutools.recommendation.recommendation.random_setup`, +:py:func:`~pabutools.recommendation.recommendation.offline_popularity` and +:py:func:`~pabutools.recommendation.recommendation.offline_consensus`. + +Collect an answer for those projects from each remaining voter, and elect: + +.. code-block:: python + + from pabutools.recommendation import elect + + answers = { + "v5": partial_ballot(approved={garden}, disapproved={crossings}), + "v6": partial_ballot(approved={crossings}, disapproved={garden}), + } + + elect(instance, lv_profile, answers, predictor="classification") + # BudgetAllocation([Garden, Shade]) + +:py:func:`~pabutools.recommendation.recommendation.elect` returns a +:py:class:`~pabutools.rules.budgetallocation.BudgetAllocation`, like any other rule. An +answer a voter actually gave is never overwritten: the estimation only fills in the projects +she was not asked about. + +To decide how many questions to ask in the first place, +:py:func:`~pabutools.recommendation.recommendation.plan_sampling` turns a vote budget into a +plan. Collecting 30% of the votes from 5000 voters over 100 projects, a tenth of them as +complete ballots, means 150 people fill in the whole ballot and the rest answer 28 questions +each: + +.. code-block:: python + + from pabutools.recommendation import plan_sampling + + plan_sampling(5000, 100, sample_degree=0.3, lv_degree=0.1) + # (150, 28) + +Asking Adaptively +----------------- + +:py:func:`~pabutools.recommendation.recommendation.next_adaptive_question` + +The offline setups fix their questions in advance. The online setup cannot: it re-reads the +tally after every answer, so each reply changes what is asked next. A live process drives it +one question at a time: + +.. code-block:: python + + from pabutools.recommendation import next_adaptive_question, partial_ballot + + answers = partial_ballot() + for _ in range(2): + project = next_adaptive_question(instance, lv_profile, answers) + # ... put `project` to the voter, then fold her reply into `answers` ... + +Estimating the Missing Votes +---------------------------- + +Three prediction modules are provided: +:py:func:`~pabutools.recommendation.model_training.predict_by_classification` (one binary +classifier per project), +:py:func:`~pabutools.recommendation.model_training.predict_by_matrix_factorization` and +:py:func:`~pabutools.recommendation.model_training.predict_by_factorization_machines` +(collaborative filtering). + +They are selected by name, as in the ``predictor="classification"`` argument above. Any +callable with the same signature works, so you can supply your own: + +.. code-block:: python + + def approve_everything(instance, lv_profile, partial_ballots): + return {voter: ApprovalBallot(instance) for voter in partial_ballots} + + elect(instance, lv_profile, answers, predictor=approve_everything) + +These three modules rely on ``xgboost``, ``scikit-surprise`` and ``lightfm``, which are +**not** installed with Pabutools. They come with the optional ``recommendation`` extra:: + + pip install pabutools[recommendation] + +The libraries are imported only when a model is actually fitted, so everything else in this +module works without them. + +Measuring the Loss +------------------ + +:py:func:`~pabutools.recommendation.recommendation.run_experiment` + +If you already hold complete ballots, you can measure what asking less would have cost. This +is a simulation: the profile is split into the voters who answer in full and the ones who do +not, each of the latter has her answers to k projects read off her real ballot, and the rest +are withheld for the prediction module to estimate. The allocation that comes back can then +be compared with the one complete information produces: + +.. code-block:: python + + from pabutools.recommendation import ( + run_experiment, greedy_approval, fractional_allocation_score, + ) + + profile = ApprovalProfile([ + ApprovalBallot([garden, library]), ApprovalBallot([crossings, shade]), + ApprovalBallot([garden, shade]), ApprovalBallot([garden, library, shade]), + ApprovalBallot([crossings, shade]), ApprovalBallot([garden, library]), + ]) + + real = set(greedy_approval(instance, profile)) + predicted = set(run_experiment( + instance, profile, 0.5, 0.5, setup="offline_controversiality", seed=17 + )) + + fractional_allocation_score(real, predicted, instance.budget_limit) # 0.75 + len(real ^ predicted) # 0 + +The two arguments ``0.5, 0.5`` are the *sample degree* — the share of all voter-project +votes that is collected — and the *LV degree*, the share of those votes coming from complete +ballots. :py:func:`~pabutools.recommendation.recommendation.fractional_allocation_score` +reports the cost of the correctly predicted projects as a share of the budget, and the +symmetric difference counts the projects on which the two allocations disagree. + +:py:func:`~pabutools.recommendation.recommendation.run_all_experiments` repeats this over a +grid of both degrees, for every setup and every prediction module at once. + +To score the predicted votes themselves rather than the allocation, use +:py:func:`~pabutools.recommendation.recommendation.classification_metrics`, which reports +precision, recall and F1 over the projects a voter was never asked about. diff --git a/pabutools/recommendation/recommendation.py b/pabutools/recommendation/recommendation.py index bd0dabe..e06bd11 100644 --- a/pabutools/recommendation/recommendation.py +++ b/pabutools/recommendation/recommendation.py @@ -313,7 +313,7 @@ def greedy_approval( .. note:: Delegates to pabutools' - :py:func:`~pabutools.rules.greedy_utilitarian_welfare` with + :py:func:`~pabutools.rules.greedywelfare.greedy_utilitarian_welfare` with :py:class:`~pabutools.election.satisfaction.additivesatisfaction.Cost_Sat`: the rule ranks by marginal satisfaction divided by cost, and under Cost_Sat a project's marginal satisfaction is score(p)*cost(p), so the @@ -827,7 +827,7 @@ def plan_sampling( This is the arithmetic of Example 3.1 on its own. With n voters and m projects, ``sample_degree`` (s) fixes the total number of collected votes at s*n*m and ``lv_degree`` (l) is the share of those votes that come from full - ballots, so |LV| = round(s*l*n) and the remaining s*(1-l)*n*m votes are + ballots, so ``|LV|`` = round(s*l*n) and the remaining s*(1-l)*n*m votes are divided equally among the s*n*m Target Voters. ``lv_degree == 1`` is the paper's naive sampling baseline: the sampled voters answer in full and nobody else is asked anything, so k = 0. @@ -918,9 +918,9 @@ def split_lv_tv( votes at s*n*m and ``lv_degree`` (l) is the share of those votes that come from full ballots: - * |LV| = round(s*l*n) voters, drawn at random, keep their full ballots; + * ``|LV|`` = round(s*l*n) voters, drawn at random, keep their full ballots; * every *other* voter is a TV, and the remaining s*(1-l)*n*m votes are - divided equally among them: k = round(s*(1-l)*n*m / |TV|). + divided equally among them: k = round(s*(1-l)*n*m / ``|TV|``). ``lv_degree == 1`` is the paper's naive "sampling" baseline (Section 6): the sampled voters answer in full and the rest of the community does not