diff --git a/docs-source/source/reference/index.rst b/docs-source/source/reference/index.rst index f4be9667..e2decbe3 100644 --- a/docs-source/source/reference/index.rst +++ b/docs-source/source/reference/index.rst @@ -8,6 +8,7 @@ Reference election/index rules/index analysis/index + recommendation/index visualisation/index tiebreaking fractions diff --git a/docs-source/source/reference/recommendation/index.rst b/docs-source/source/reference/recommendation/index.rst new file mode 100644 index 00000000..f77277f6 --- /dev/null +++ b/docs-source/source/reference/recommendation/index.rst @@ -0,0 +1,16 @@ +Recommendation module +===================== + +.. automodule:: pabutools.recommendation + +Sampling, pipeline and evaluation +--------------------------------- + +.. automodule:: pabutools.recommendation.recommendation + :members: + +Model training and prediction +----------------------------- + +.. automodule:: pabutools.recommendation.model_training + :members: diff --git a/docs-source/source/usage/index.rst b/docs-source/source/usage/index.rst index 2e9922dd..f2e56069 100644 --- a/docs-source/source/usage/index.rst +++ b/docs-source/source/usage/index.rst @@ -15,5 +15,6 @@ refer to the :ref:`quickstart` page. rules outcomevisualisation analysis + recommendation tiebreaking fractions diff --git a/docs-source/source/usage/recommendation.rst b/docs-source/source/usage/recommendation.rst new file mode 100644 index 00000000..a8955d7c --- /dev/null +++ b/docs-source/source/usage/recommendation.rst @@ -0,0 +1,189 @@ +Recommendation +============== + +For reference, see the module :py:mod:`~pabutools.recommendation`. + +When a participatory budgeting instance has many projects, asking every voter about every +project is a burden, and ballots are often left unfinished. This module implements the +approach of Leibiker and Talmon (2023): ask each voter about only a few projects, estimate +the votes that were never asked for, and run a voting rule on the completed profile. + +There are two ways to use it. If you are running a process and only hold partial ballots, +:py:func:`~pabutools.recommendation.recommendation.elect` returns an outcome. If you hold +complete ballots and want to know what asking less would have cost you, +:py:func:`~pabutools.recommendation.recommendation.run_experiment` simulates the loss. + +Partial Ballots +--------------- + +A voter who was asked about only some of the projects has three kinds of opinion: projects +she approved, projects she rejected, and projects she was never shown. This is stored in a +:py:class:`~pabutools.election.ballot.cardinalballot.CardinalBallot` under the convention +*positive means approved, negative means disapproved, zero or absent means not asked*. +:py:func:`~pabutools.recommendation.recommendation.partial_ballot` builds one: + +.. code-block:: python + + from pabutools.election import Instance, Project, ApprovalProfile, ApprovalBallot + from pabutools.recommendation import partial_ballot + + garden = Project("Garden", 18) + crossings = Project("Crossings", 24) + library = Project("Library", 16) + shade = Project("Shade", 12) + instance = Instance([garden, crossings, library, shade], budget_limit=40) + + # A voter who approved the garden, rejected the crossings, and was asked nothing else. + ballot = partial_ballot(approved={garden}, disapproved={crossings}) + +Running a Process +----------------- + +:py:func:`~pabutools.recommendation.recommendation.elect` + +Some voters do provide a complete ballot; they are the *learning voters*, and their profile +is what the estimation learns from. Every other voter is asked about a limited number of +projects. + +Which projects to ask about is decided by a *sampling setup*. The offline setups choose one +set for everybody, out of the complete ballots already collected: + +.. code-block:: python + + from pabutools.recommendation import offline_controversiality + + lv_profile = ApprovalProfile([ + ApprovalBallot([garden, library]), + ApprovalBallot([crossings, shade]), + ApprovalBallot([garden, shade]), + ApprovalBallot([garden, library, shade]), + ]) + + offline_controversiality(instance, lv_profile, k=2) + # {Library, Shade} - the two projects the learning voters most disagree about + +The alternatives are :py:func:`~pabutools.recommendation.recommendation.random_setup`, +:py:func:`~pabutools.recommendation.recommendation.offline_popularity` and +:py:func:`~pabutools.recommendation.recommendation.offline_consensus`. + +Collect an answer for those projects from each remaining voter, and elect: + +.. code-block:: python + + from pabutools.recommendation import elect + + answers = { + "v5": partial_ballot(approved={garden}, disapproved={crossings}), + "v6": partial_ballot(approved={crossings}, disapproved={garden}), + } + + elect(instance, lv_profile, answers, predictor="classification") + # BudgetAllocation([Garden, Shade]) + +:py:func:`~pabutools.recommendation.recommendation.elect` returns a +:py:class:`~pabutools.rules.budgetallocation.BudgetAllocation`, like any other rule. An +answer a voter actually gave is never overwritten: the estimation only fills in the projects +she was not asked about. + +To decide how many questions to ask in the first place, +:py:func:`~pabutools.recommendation.recommendation.plan_sampling` turns a vote budget into a +plan. Collecting 30% of the votes from 5000 voters over 100 projects, a tenth of them as +complete ballots, means 150 people fill in the whole ballot and the rest answer 28 questions +each: + +.. code-block:: python + + from pabutools.recommendation import plan_sampling + + plan_sampling(5000, 100, sample_degree=0.3, lv_degree=0.1) + # (150, 28) + +Asking Adaptively +----------------- + +:py:func:`~pabutools.recommendation.recommendation.next_adaptive_question` + +The offline setups fix their questions in advance. The online setup cannot: it re-reads the +tally after every answer, so each reply changes what is asked next. A live process drives it +one question at a time: + +.. code-block:: python + + from pabutools.recommendation import next_adaptive_question, partial_ballot + + answers = partial_ballot() + for _ in range(2): + project = next_adaptive_question(instance, lv_profile, answers) + # ... put `project` to the voter, then fold her reply into `answers` ... + +Estimating the Missing Votes +---------------------------- + +Three prediction modules are provided: +:py:func:`~pabutools.recommendation.model_training.predict_by_classification` (one binary +classifier per project), +:py:func:`~pabutools.recommendation.model_training.predict_by_matrix_factorization` and +:py:func:`~pabutools.recommendation.model_training.predict_by_factorization_machines` +(collaborative filtering). + +They are selected by name, as in the ``predictor="classification"`` argument above. Any +callable with the same signature works, so you can supply your own: + +.. code-block:: python + + def approve_everything(instance, lv_profile, partial_ballots): + return {voter: ApprovalBallot(instance) for voter in partial_ballots} + + elect(instance, lv_profile, answers, predictor=approve_everything) + +These three modules rely on ``xgboost``, ``scikit-surprise`` and ``lightfm``, which are +**not** installed with Pabutools. They come with the optional ``recommendation`` extra:: + + pip install pabutools[recommendation] + +The libraries are imported only when a model is actually fitted, so everything else in this +module works without them. + +Measuring the Loss +------------------ + +:py:func:`~pabutools.recommendation.recommendation.run_experiment` + +If you already hold complete ballots, you can measure what asking less would have cost. This +is a simulation: the profile is split into the voters who answer in full and the ones who do +not, each of the latter has her answers to k projects read off her real ballot, and the rest +are withheld for the prediction module to estimate. The allocation that comes back can then +be compared with the one complete information produces: + +.. code-block:: python + + from pabutools.recommendation import ( + run_experiment, greedy_approval, fractional_allocation_score, + ) + + profile = ApprovalProfile([ + ApprovalBallot([garden, library]), ApprovalBallot([crossings, shade]), + ApprovalBallot([garden, shade]), ApprovalBallot([garden, library, shade]), + ApprovalBallot([crossings, shade]), ApprovalBallot([garden, library]), + ]) + + real = set(greedy_approval(instance, profile)) + predicted = set(run_experiment( + instance, profile, 0.5, 0.5, setup="offline_controversiality", seed=17 + )) + + fractional_allocation_score(real, predicted, instance.budget_limit) # 0.75 + len(real ^ predicted) # 0 + +The two arguments ``0.5, 0.5`` are the *sample degree* — the share of all voter-project +votes that is collected — and the *LV degree*, the share of those votes coming from complete +ballots. :py:func:`~pabutools.recommendation.recommendation.fractional_allocation_score` +reports the cost of the correctly predicted projects as a share of the budget, and the +symmetric difference counts the projects on which the two allocations disagree. + +:py:func:`~pabutools.recommendation.recommendation.run_all_experiments` repeats this over a +grid of both degrees, for every setup and every prediction module at once. + +To score the predicted votes themselves rather than the allocation, use +:py:func:`~pabutools.recommendation.recommendation.classification_metrics`, which reports +precision, recall and F1 over the projects a voter was never asked about. diff --git a/pabutools/recommendation/__init__.py b/pabutools/recommendation/__init__.py new file mode 100644 index 00000000..0277da46 --- /dev/null +++ b/pabutools/recommendation/__init__.py @@ -0,0 +1,109 @@ +""" +Module implementing the algorithms of +"A Recommendation System for Participatory Budgeting", +by Gil Leibiker and Nimrod Talmon (2023), https://optlearnmas23.github.io/files/p17.pdf + +:py:mod:`~pabutools.recommendation.recommendation` holds the sampling setups, +the pipeline, the voting rule and the evaluation metrics; +:py:mod:`~pabutools.recommendation.model_training` holds the fitting and +prediction of the three learning-based modules (whose ML libraries are optional +dependencies, installed with ``pip install pabutools[recommendation]``). + +Programmer: Roei Yanku +""" + +from pabutools.recommendation.recommendation import ( + APPROVAL, + DISAPPROVAL, + HIDDEN, + partial_ballot, + reveal_ballot, + approved_projects, + disapproved_projects, + exposed_projects, + hidden_projects, + as_approval_ballot, + consensus_levels, + most_consensual_projects, + greedy_approval, + random_setup, + offline_popularity, + offline_consensus, + offline_controversiality, + online_adaptive_controversial, + next_adaptive_question, + SETUPS, + PREDICTORS, + exposed_sets, + plan_sampling, + split_lv_tv, + complete_ballots, + run_pipeline, + run_experiment, + Predictor, + as_predictor, + elect, + SAMPLE_DEGREES, + LV_DEGREES, + run_all_experiments, + classification_metrics, + fractional_allocation_score, +) +from pabutools.recommendation.model_training import ( + train_classification, + train_matrix_factorization, + train_factorization_machines, + predict_by_classification, + predict_by_matrix_factorization, + predict_by_factorization_machines, +) + +__all__ = [ + # Section 2.3 - partial ballots and the +1/-1/0 score convention. + "APPROVAL", + "DISAPPROVAL", + "HIDDEN", + "partial_ballot", + "reveal_ballot", + "approved_projects", + "disapproved_projects", + "exposed_projects", + "hidden_projects", + "as_approval_ballot", + # Section 2.2 - popularity, consensus and the voting rule. + "consensus_levels", + "most_consensual_projects", + "greedy_approval", + # Section 3.1 - the sampling setups. + "SETUPS", + "random_setup", + "offline_popularity", + "offline_consensus", + "offline_controversiality", + "online_adaptive_controversial", + "next_adaptive_question", + "exposed_sets", + # Section 2.1 - the prediction modules. + "PREDICTORS", + "Predictor", + "as_predictor", + "train_classification", + "train_matrix_factorization", + "train_factorization_machines", + "predict_by_classification", + "predict_by_matrix_factorization", + "predict_by_factorization_machines", + # Running a process: a real one, or the paper's experiments. + "plan_sampling", + "elect", + "split_lv_tv", + "complete_ballots", + "run_pipeline", + "run_experiment", + "run_all_experiments", + "SAMPLE_DEGREES", + "LV_DEGREES", + # Section 5 - evaluation. + "classification_metrics", + "fractional_allocation_score", +] diff --git a/pabutools/recommendation/model_training.py b/pabutools/recommendation/model_training.py new file mode 100644 index 00000000..721c8036 --- /dev/null +++ b/pabutools/recommendation/model_training.py @@ -0,0 +1,1196 @@ +""" +Model *training* for the learning-based prediction modules of +"A Recommendation System for Participatory Budgeting", +by Gil Leibiker and Nimrod Talmon (2023), https://optlearnmas23.github.io/files/p17.pdf +(Section 2.1). + +This module contains both fitting and prediction for the three learning-based +modules. ``pabutools.recommendation`` is responsible only for sampling, orchestration, +and the voting rule. + +All three models train on the votes the process actually *collected*, which +Section 3.1.1 defines as the preferences of the LV voters **and** the exposed +preferences E_TV of all Target Voters. The Target Voters' hidden votes are the +paper's test set (Section 5) and are never read during training. + +* :py:func:`train_classification` is supervised: one binary classifier per + project, taking preferences on P \\ {p} as input and the preference on p as + its 0/1 output. Every LV voter trains every project's classifier; a Target + Voter trains project p's when p lies in her exposed set E_v, with the + projects she was not asked about left missing - the very gaps that the rows + to be predicted will carry. +* :py:func:`train_matrix_factorization` and + :py:func:`train_factorization_machines` are collaborative filtering: one + voter-project matrix holds the Learning Voters' full ballots **and every Target + Voter's exposed votes** (the paper's E_TV), factorised once for the whole TV + population. The FM hybrid additionally describes every project by Table 2's + attributes - cost, categories and target population segments - which is what + lets a voter's taste reach a project she was never asked about. + +Following Section 5, all three also weight the minority class against the ~10% +approval rate of Table 1, and choose their hyperparameters on a 15% validation +set scored by F1 (Section 5.1's "suitable measure for an imbalanced data"). + +.. note:: + Section 2.1.1 says only that binary classification is used and that XGBoost + is a good algorithm for it; it never states what one training row is, nor + which features it carries. The per-project reading implemented here - a + voter's opinions on the other projects as the features - is therefore one + defensible interpretation of an under-specified section, not something the + paper prescribes. + +.. note:: + The paper names one library only: ``xgboost``, for the classification module + (Section 2.1.1). It does not say what backs matrix factorization or + factorization machines, so ``scikit-surprise`` and ``lightfm`` are this + implementation's choices. Following the maintainer's advice, the ML libraries + are **not a hard requirement**: they are imported lazily by the corresponding + training function and are available through the ``recommendation`` optional + dependency. + +Programmer: Roei Yanku +Date: 2026-06-20. +""" + +from __future__ import annotations + +import logging + +import numpy as np + +from pabutools.election import ( + Instance, + Project, + ApprovalProfile, + ApprovalBallot, + CardinalBallot, +) + +logger = logging.getLogger(__name__) + +#: The exposed votes of every Target Voter, keyed by voter id: E_v split into +#: her approvals A_v and disapprovals D_v (Section 2.3). +TVExposed = dict[str, tuple[set[Project], set[Project]]] + + +# --------------------------------------------------------------------------- +# Section 2.1.1 - Binary classification (one classifier per project). +# --------------------------------------------------------------------------- +#: Hyperparameter settings scored against the validation set (Section 5). The +#: first entry is what gets fitted when tuning is switched off (``param_grid=()``). +#: +#: ``class_weight`` is deliberately *not* varied here; see :py:func:`_fit_per_project` +#: for why the minority-class correction of Section 5 is off by default. +DEFAULT_PARAM_GRID = ( + {"n_estimators": 100, "max_depth": 3, "learning_rate": 0.1}, + {"n_estimators": 200, "max_depth": 3, "learning_rate": 0.05}, + {"n_estimators": 200, "max_depth": 6, "learning_rate": 0.1}, +) + + +def _xgboost(): + """ + The lazily imported ``xgboost`` module (an optional dependency). + + Examples + -------- + >>> _xgboost().__name__ + 'xgboost' + """ + try: + import xgboost + except ImportError: + raise ImportError( + "You need to install xgboost to train the classification " + "predictor (pip install pabutools[recommendation])." + ) + return xgboost + + +def _masked_copies(lv_votes: np.ndarray, tv_votes: np.ndarray, seed: int): + """ + The LV rows to fit on: their full ballots, plus one blanked copy of each. + + At prediction time a classifier is asked about a project the voter was never + questioned on, from a row where nearly every input is missing. An LV row is + complete, and a Target Voter only supplies a row for the projects she *was* + asked about - so for the projects that actually need predicting, the + classifiers would never meet a missing value during training and would send + every blank one fixed way. That is what made the offline setups, where every + voter is asked the same projects, return one identical ballot for everybody. + + Each LV voter is therefore added a second time with a real Target Voter's + pattern of unanswered projects blanked out, so the classifiers train on rows + shaped like the ones they will be asked about. Her labels stay correct + because the label column is dropped from the features anyway, so blanking it + cannot leak anything. + + Returns ``(features, labels)``: the labels are always the true full ballots, + only the features are blanked. + + Examples + -------- + One LV voter who answered all three projects, and one Target Voter who was + asked only about the first. The LV row is kept as it is and added a second + time with the two projects she would not have been asked about blanked out; + her labels stay the true ballot both times. + + >>> lv = np.array([[1.0, 0.0, 1.0]]) + >>> tv = np.array([[1.0, np.nan, np.nan]]) + >>> features, labels = _masked_copies(lv, tv, seed=0) + >>> features.tolist() + [[1.0, 0.0, 1.0], [1.0, nan, nan]] + >>> labels.tolist() + [[1.0, 0.0, 1.0], [1.0, 0.0, 1.0]] + + With no Target Voters there is nothing to imitate, so the rows are returned + untouched. + + >>> features, labels = _masked_copies(lv, np.zeros((0, 3)), seed=0) + >>> features.tolist() + [[1.0, 0.0, 1.0]] + """ + if not len(lv_votes) or not len(tv_votes): + return lv_votes, lv_votes + unasked = np.isnan(tv_votes) # True where that Target Voter was not asked + chosen = np.random.default_rng(seed).integers(0, len(tv_votes), len(lv_votes)) + masked = lv_votes.copy() + masked[unasked[chosen]] = np.nan + return np.vstack([lv_votes, masked]), np.vstack([lv_votes, lv_votes]) + + +def _project_rows( + index: int, lv_features: np.ndarray, lv_labels: np.ndarray, + tv_votes: np.ndarray, +): + """ + The ``(X, y)`` rows for the classifier of the project in column ``index``: + every LV row, plus the Target Voters who were asked about that project + (a voter without a preference on it has no label to contribute). The target + project's own column is dropped from ``X`` - it is the label, not an input. + + Examples + -------- + Two projects and two LV voters, plus one Target Voter asked only about + project 0. Building project 0's classifier leaves project 1 as the sole + input, and all three voters supply a label for project 0. + + >>> lv = np.array([[1.0, 0.0], [0.0, 1.0]]) + >>> tv = np.array([[1.0, np.nan]]) + >>> X, y = _project_rows(0, lv, lv, tv) + >>> X.ravel().tolist() + [0.0, 1.0, nan] + >>> y.tolist() + [1.0, 0.0, 1.0] + + For project 1 the Target Voter has no preference to contribute, so only the + two LV voters appear. + + >>> X, y = _project_rows(1, lv, lv, tv) + >>> y.tolist() + [0.0, 1.0] + """ + answered = ~np.isnan(tv_votes[:, index]) + y = np.concatenate([lv_labels[:, index], tv_votes[answered, index]]) + X = np.vstack([ + np.delete(lv_features, index, axis=1), + np.delete(tv_votes[answered], index, axis=1), + ]) + return X, y + + +def _fit_per_project( + projects: list[Project], lv_features: np.ndarray, lv_labels: np.ndarray, + tv_votes: np.ndarray, params: dict, +) -> dict[Project, tuple]: + """ + Fit one classifier per project on the given collected votes. A project whose + collected votes all agree (or that nobody voted on) needs no classifier, so + its common/majority preference is stored directly. + + Examples + -------- + Two voters who agree exactly: everyone approves p1 and rejects p2. Neither + project needs a classifier, so each is stored as the constant both voters + gave, and ``xgboost`` is never even imported. + + >>> p1, p2 = Project("p1", 1), Project("p2", 1) + >>> votes = np.array([[1.0, 0.0], [1.0, 0.0]]) + >>> fitted = _fit_per_project([p1, p2], votes, votes, np.zeros((0, 2)), {}) + >>> fitted[p1], fitted[p2] + (('const', 1), ('const', 0)) + """ + per_project: dict[Project, tuple] = {} + for index, project in enumerate(projects): + X, y = _project_rows(index, lv_features, lv_labels, tv_votes) + # An empty y has fewer than 2 distinct labels too, so it is covered. + if X.shape[1] == 0 or len(set(y.tolist())) < 2: + per_project[project] = ( + "const", int(len(y) > 0 and 2 * int(y.sum()) >= len(y)) + ) + continue + settings = dict(params) + # Section 5 addresses the ~10% approval rate of Table 1 "by modifying + # the model loss function to give more weight to minority class + # samples", which for XGBoost is ``scale_pos_weight`` (the weighted-loss + # approach of the paper's reference [26]). ``class_weight`` is the + # exponent applied to the class ratio: 0 leaves the loss alone, 1 + # balances the two classes completely, 0.5 is halfway. + # + # It defaults to 0, i.e. no correction, which is a deliberate departure + # from Section 5. Measured on three real Warsaw districts, the full + # correction collapses the very metric the paper reports: + # + # Praga-Poludnie 2022, offline_popularity: FA 0.802 -> 0.007 + # Mokotow 2022, offline_popularity: FA 0.869 -> 0.114 + # Wola 2021, offline_popularity: FA 0.731 -> 0.082 + # + # The cause is an interaction with the offline setups, where every + # Target Voter is asked about the *same* k projects and so arrives with + # an identical pattern of missing inputs. A balanced loss then makes the + # classifiers approve most of the ballot for everyone alike, the + # approval scores flatten, and the ranking greedy approval needs is + # lost. (Under the ``random`` setup, where exposures differ per voter, + # the correction is harmless and occasionally helps.) Raise it if you + # care about per-voter recall rather than the winning bundle: recall + # goes 0.078 -> 0.773 across that same range, while precision stays + # near 0.08. + exponent = settings.pop("class_weight", 0.0) + positives = int(y.sum()) + negatives = len(y) - positives + ratio = (negatives / positives) if positives else 1.0 + classifier = _xgboost().XGBClassifier( + verbosity=0, + scale_pos_weight=ratio ** exponent, + **settings, + ) + classifier.fit(X, y) + per_project[project] = ("model", ([p for p in projects if p != project], + classifier)) + return per_project + + +def _pooled_f1( + per_project: dict[Project, tuple], projects: list[Project], + lv_features: np.ndarray, lv_labels: np.ndarray, tv_votes: np.ndarray, +) -> float: + """ + F1 of a fitted bundle over every held-out vote, pooled across projects. + Section 5.1 calls F1 the "suitable measure for an imbalanced data", so it is + the criterion the validation set selects hyperparameters on. + + Examples + -------- + One held-out voter who approves p1 and rejects p2, against a bundle that + predicts exactly that: no false positives and no misses, so F1 is 1. + + >>> p1, p2 = Project("p1", 1), Project("p2", 1) + >>> votes = np.array([[1.0, 0.0]]) + >>> bundle = {p1: ("const", 1), p2: ("const", 0)} + >>> _pooled_f1(bundle, [p1, p2], votes, votes, np.zeros((0, 2))) + 1.0 + + A bundle that approves both projects finds the one real approval but also + raises a false alarm on p2: precision 0.5, recall 1.0. + + >>> _pooled_f1({p1: ("const", 1), p2: ("const", 1)}, + ... [p1, p2], votes, votes, np.zeros((0, 2))) + 0.6666666666666666 + """ + tp = fp = fn = 0 + for index, project in enumerate(projects): + X, y = _project_rows(index, lv_features, lv_labels, tv_votes) + if len(y) == 0: + continue + kind, payload = per_project[project] + if kind == "const": + predicted = np.full(len(y), float(payload)) + else: + predicted = payload[1].predict(X) + tp += int(((y == 1) & (predicted == 1)).sum()) + fp += int(((y == 0) & (predicted == 1)).sum()) + fn += int(((y == 1) & (predicted == 0)).sum()) + return _f1(tp, fp, fn) + + +def _f1(tp: int, fp: int, fn: int) -> float: + """ + F1 from a confusion matrix (Section 5.1), 0.0 when it is undefined. + + Examples + -------- + >>> _f1(3, 1, 1) # precision 0.75, recall 0.75 + 0.75 + >>> _f1(0, 5, 5) # nothing found: precision and recall both 0 + 0.0 + >>> _f1(0, 0, 0) # undefined; 0.0 by the usual convention + 0.0 + """ + precision = tp / (tp + fp) if tp + fp else 0.0 + recall = tp / (tp + fn) if tp + fn else 0.0 + return 2 * precision * recall / (precision + recall) if precision + recall else 0.0 + + +def train_classification( + instance: Instance, + lv_profile: ApprovalProfile, + tv_ballots: dict[str, CardinalBallot] | None = None, + *, + validation_fraction: float = 0.15, + param_grid=DEFAULT_PARAM_GRID, + seed: int = 0, +) -> dict: + """ + Train the per-project binary classifiers of Section 2.1.1. For every project + p, one classifier uses preferences on P \\ {p} as input and the preference on + p as its 0/1 output. The resulting model bundle is trained once and reused + for every Target Voter; preferences the voter was not asked about are passed + to XGBoost as missing values. + + The training data is every vote the process *collected*, which Section 3.1.1 + defines as the preferences of the LV voters **and** the exposed preferences + E_TV of the Target Voters: each LV voter contributes a row to every + project's classifier, and a Target Voter contributes a row for project p + exactly when p lies in her exposed set E_v (otherwise there is no label to + learn from). Those rows carry NaN for the projects she was not asked about, + which is what teaches XGBoost a split direction for missing inputs - the + same missingness the rows to be predicted will carry. + + Following Section 5, the collected votes are split three ways rather than + used wholesale: ``validation_fraction`` of the *voters* are held back as the + validation set ("a predefined set of votes from closed set of voters"), each + setting in ``param_grid`` is fitted on the rest and scored on them by pooled + F1, and the winning setting is then refitted on all collected votes. (The + Target Voters' hidden votes are the paper's test set and are never touched + here.) One setting is chosen for the whole bundle, not per project, matching + the paper's singular "the hyperparameters of a model". + + Backed by the external ``xgboost`` library + (:py:class:`xgboost.XGBClassifier`, class-weighted for the imbalanced data), + imported lazily. When no collected vote covers a project, or they all agree + on it, no classifier is needed, so we use that common/majority preference + directly. + + Parameters + ---------- + instance : :py:class:`~pabutools.election.instance.Instance` + The PB instance. + lv_profile : :py:class:`~pabutools.election.profile.approvalprofile.ApprovalProfile` + The full ballots of the LV voters, used as the training data. + tv_ballots : dict[str, :py:class:`~pabutools.election.ballot.cardinalballot.CardinalBallot`], optional + The Target Voters' partial ballots, keyed by voter id. Their exposed + votes E_TV join the training data; their hidden votes are never + looked at. + validation_fraction : float, optional + Share of the voters held out to tune the hyperparameters on (default + 0.15, the paper's figure). Set to 0 to skip the search. + param_grid : Iterable[dict], optional + The settings to score, defaulting to :py:data:`DEFAULT_PARAM_GRID`. + Tuning multiplies the number of fits by ``len(param_grid)``, so pass + a single setting - or ``()`` - for large sweeps. + seed : int, optional + Seed for the validation split, for reproducibility. + Returns + ------- + dict + ``{"per_project": {project: ("const", 0/1) or + ("model", (input_projects, classifier))}, "params": the + hyperparameters used}``, consumed by + :py:func:`predict_by_classification`. + + Examples + -------- + With unanimous LV approvals, every project falls back to the majority + preference (1 for approval, 0 for disapproval). + + >>> p1, p2 = Project("p1", 1), Project("p2", 1) + >>> inst = Instance([p1, p2], budget_limit=2) + >>> lv = ApprovalProfile([ApprovalBallot([p1])] * 3) + >>> model = train_classification(inst, lv) + >>> model["per_project"][p1], model["per_project"][p2] + (('const', 1), ('const', 0)) + + A Target Voter's exposed votes join the training data. On the LV ballots + alone the lone voter's rejection of p2 is unanimous, so p2 needs no + classifier; once three Target Voters expose an approval of it the collected + votes disagree and a classifier is fitted after all. + + >>> lv = ApprovalProfile([ApprovalBallot([p1])]) + >>> train_classification(inst, lv)["per_project"][p2][0] + 'const' + >>> tv = {f"v{i}": CardinalBallot({p2: 1}) for i in range(3)} + >>> train_classification(inst, lv, tv)["per_project"][p2][0] + 'model' + """ + projects = sorted(instance, key=str) + lv = list(lv_profile) + tv = list((tv_ballots or {}).values()) + # The collected votes as two voter x project tables. An LV voter answered + # every project, so her row is dense; a Target Voter's row is NaN wherever + # she was not asked, and NaN is XGBoost's own missing-value marker. + def table(rows): # the reshape keeps the (0, m) shape when nobody is in it + return np.array(rows, dtype=float).reshape(len(rows), len(projects)) + + tv_sets = [_preference_sets(instance, ballot) for ballot in tv] + lv_votes = table([[float(p in ballot) for p in projects] for ballot in lv]) + tv_votes = table([ + [1.0 if p in approved else 0.0 if p in disapproved else np.nan + for p in projects] + for approved, disapproved, _ in tv_sets + ]) + + # Blanked copies of the LV ballots, so the classifiers see the sparse rows + # they will actually be asked about (see :py:func:`_masked_copies`). + lv_features, lv_labels = _masked_copies(lv_votes, tv_votes, seed) + copies = len(lv_features) // len(lv) if len(lv) else 1 + + param_grid = tuple(param_grid) + params = dict(param_grid[0]) if param_grid else dict(DEFAULT_PARAM_GRID[0]) + # Section 5's validation set: whole voters, so that no voter has some of her + # votes fitted on and others scored. Voters are numbered LV first, then TV. + n_voters = len(lv) + len(tv) + n_validation = int(round(validation_fraction * n_voters)) + if len(param_grid) > 1 and 0 < n_validation < n_voters: + held_out = np.random.default_rng(seed).permutation(n_voters)[:n_validation] + is_validation = np.zeros(n_voters, dtype=bool) + is_validation[held_out] = True + lv_held, tv_held = is_validation[:len(lv)], is_validation[len(lv):] + # Every LV voter now owns ``copies`` rows, in [full, blanked] order, so + # her copies must fall on the same side of the split as she does. + rows_held = np.tile(lv_held, copies) + best_f1 = -1.0 + for candidate in param_grid: + trained = _fit_per_project( + projects, lv_features[~rows_held], lv_labels[~rows_held], + tv_votes[~tv_held], candidate, + ) + score = _pooled_f1( + trained, projects, lv_features[rows_held], lv_labels[rows_held], + tv_votes[tv_held], + ) + logger.debug("train_classification: %s -> validation F1 %.4f", + candidate, score) + if score > best_f1: + params, best_f1 = dict(candidate), score + logger.info( + "train_classification: tuned on %d of %d held-out voters, " + "best pooled F1 %.4f with %s", + n_validation, n_voters, best_f1, params, + ) + per_project = _fit_per_project( + projects, lv_features, lv_labels, tv_votes, params + ) + logger.info( + "train_classification: trained %d per-project classifiers once on " + "%d LV ballots (x%d, blanked) and the exposed votes of %d TV voters", + len(per_project), len(lv), copies, len(tv), + ) + return {"per_project": per_project, "params": params} + + +# --------------------------------------------------------------------------- +# Section 2.1.2 - Collaborative filtering. +# --------------------------------------------------------------------------- +def _known_preferences( + instance: Instance, + lv_profile: ApprovalProfile, + tv_exposed: TVExposed, +) -> tuple[list[Project], list[tuple[tuple[str, object], Project, int]]]: + """ + Return the known preferences: full LV ballots plus exposed TV sets E_v. + + Examples + -------- + The LV voter contributes a vote on every project; the Target Voter only on + the one she was asked about. Voters are keyed ``("lv", index)`` and + ``("tv", voter id)`` so the two groups cannot collide. + + >>> p1, p2 = Project("p1", 1), Project("p2", 1) + >>> inst = Instance([p1, p2], budget_limit=2) + >>> lv = ApprovalProfile([ApprovalBallot([p1])]) + >>> projects, known = _known_preferences(inst, lv, {"v": ({p2}, set())}) + >>> known + [(('lv', 0), p1, 1), (('lv', 0), p2, 0), (('tv', 'v'), p2, 1)] + """ + projects = sorted(instance, key=str) + preferences = [ + (("lv", i), p, int(p in ballot)) + for i, ballot in enumerate(lv_profile) + for p in projects + ] + preferences += [ + (("tv", voter), p, int(p in approved)) + for voter, (approved, disapproved) in tv_exposed.items() + for p in sorted(approved | disapproved, key=str) + ] + return projects, preferences + + +#: Latent ranks scored against the validation set for matrix factorization. +DEFAULT_MF_RANKS = (5, 20, 50) +#: ``(rank, ridge)`` settings scored against the validation set for the hybrid. +DEFAULT_FM_SETTINGS = ((5, 0.0), (20, 0.0), (20, 1e-5), (50, 1e-5)) + + +def _minority_weight(preferences) -> int: + """ + How many times an approval must count for the two classes to weigh the same. + Section 5 addresses the ~10% approval rate of Table 1 "by modifying the model + loss function to give more weight to minority class samples"; with roughly + one approval per nine rejections this returns 9. + + Examples + -------- + >>> _minority_weight([("v", "p", 1)] + [("v", "p", 0)] * 9) + 9 + >>> _minority_weight([("v", "p", 1)] * 5 + [("v", "p", 0)] * 5) + 1 + >>> _minority_weight([]) # nothing collected + 1 + """ + positives = sum(label for *_, label in preferences) + negatives = len(preferences) - positives + return max(1, round(negatives / positives)) if positives else 1 + + +def _tune(preferences, settings, fit, validation_fraction: float, seed: int): + """ + Pick the setting with the best validation F1 (Sections 5 and 5.1). + + A share of the collected *votes* is held back, rather than a share of the + voters as in :py:func:`train_classification`: a collaborative-filtering model + cannot score a voter it never saw, so holding out whole voters would leave + nothing to validate on. ``fit(votes, setting)`` returns a scorer taking a + voter and a list of projects and returning one score in [0, 1] each. + + Examples + -------- + Twenty collected votes, half of them approvals, and two candidate settings + standing in for a real model: one that approves everything and one that + approves nothing. The first finds every approval, the second finds none, so + the F1 comparison picks the first. + + >>> votes = [("v", f"p{i}", i % 2) for i in range(20)] + >>> def fit(training, approve_everything): + ... return lambda voter, projects: [ + ... 1.0 if approve_everything else 0.0 for _ in projects + ... ] + >>> _tune(votes, (False, True), fit, validation_fraction=0.5, seed=0) + True + + With a single setting there is nothing to choose between, so it is returned + without fitting anything. + + >>> _tune(votes, (False,), fit, validation_fraction=0.5, seed=0) + False + """ + settings = tuple(settings) + n_validation = int(round(validation_fraction * len(preferences))) + if len(settings) < 2 or not 0 < n_validation < len(preferences): + return settings[0] + order = np.random.default_rng(seed).permutation(len(preferences)) + held_out = set(order[:n_validation].tolist()) + training = [pref for i, pref in enumerate(preferences) if i not in held_out] + validation: dict = {} + for index in sorted(held_out): + voter, project, label = preferences[index] + validation.setdefault(voter, []).append((project, label)) + + best, best_f1 = settings[0], -1.0 + for setting in settings: + score = fit(training, setting) + tp = fp = fn = 0 + for voter, votes in validation.items(): + values = score(voter, [project for project, _ in votes]) + for (_, label), value in zip(votes, values): + if value >= 0.5 and label: + tp += 1 + elif value >= 0.5: + fp += 1 + elif label: + fn += 1 + f1 = _f1(tp, fp, fn) + logger.debug("_tune: %s -> validation F1 %.4f", setting, f1) + if f1 > best_f1: + best, best_f1 = setting, f1 + logger.info( + "_tune: %d of %d collected votes held out, best F1 %.4f with %s", + n_validation, len(preferences), best_f1, best, + ) + return best + + +def _fit_mf(preferences, rank: int, weight: int): + """ + Fit one SVD and return ``score(voter, projects)``. + + Examples + -------- + One voter who approved p1 and rejected p2; the fitted model scores any + (voter, project) pair on the 0-1 scale the preferences were given on. + + >>> p1, p2 = Project("p1", 1), Project("p2", 1) + >>> score = _fit_mf([(("lv", 0), p1, 1), (("lv", 0), p2, 0)], 2, 1) + >>> values = score(("lv", 0), [p1, p2]) + >>> len(values) == 2 and all(0.0 <= v <= 1.0 for v in values) + True + """ + import pandas as pd + from surprise import Dataset, Reader, SVD + + # Surprise exposes no sample-weight API, so an approval is repeated + # ``weight`` times instead: that many more gradient steps is the same thing + # as that much more loss weight, and it lifts the reconstructed scores of + # the minority class towards the 0.5 threshold they are compared against. + rows = [pref for pref in preferences for _ in range(weight if pref[2] else 1)] + data = Dataset.load_from_df( + pd.DataFrame(rows, columns=["voter", "project", "preference"]), + Reader(rating_scale=(0, 1)), # Surprise's name for 0/1 preferences. + ) + model = SVD(n_factors=max(1, rank), n_epochs=50, random_state=0) + model.fit(data.build_full_trainset()) + return lambda voter, projects: [ + float(model.predict(voter, p).est) for p in projects + ] + + +def train_matrix_factorization( + instance: Instance, + lv_profile: ApprovalProfile, + tv_exposed: TVExposed, + rank: int | None = None, + *, + ranks=DEFAULT_MF_RANKS, + validation_fraction: float = 0.15, + seed: int = 0, +) -> dict[str, dict[Project, float]]: + """ + Fit ``R_vp ~= mu + b_v + b_p + q_p^T x_v`` (paper, Section 2.1.2). + + Approvals are up-weighted against the imbalance of Table 1, and the latent + rank is chosen by validation F1 over ``ranks`` (Section 5) unless an explicit + ``rank`` is given, in which case that one is used and nothing is tuned. + + Examples + -------- + Three LV voters who all approve p1 and reject p2, and one Target Voter whose + exposed votes agree with them. Every project gets a reconstructed score on + the 0-1 scale, and the pair the electorate likes scores above the pair it + does not. + + >>> p1, p2 = Project("p1", 1), Project("p2", 1) + >>> inst = Instance([p1, p2], budget_limit=2) + >>> lv = ApprovalProfile([ApprovalBallot([p1])] * 3) + >>> scores = train_matrix_factorization(inst, lv, {"v": ({p1}, {p2})}, rank=2) + >>> sorted(scores) == ["v"] and len(scores["v"]) == 2 + True + >>> scores["v"][p1] > scores["v"][p2] + True + """ + try: + import pandas # noqa: F401 (needed by _fit_mf) + import surprise # noqa: F401 + except ImportError: + raise ImportError( + "You need scikit-surprise and pandas to train the matrix-factorization " + "predictor (pip install pabutools[recommendation])." + ) + + projects, preferences = _known_preferences(instance, lv_profile, tv_exposed) + if not preferences: + return {v: {p: 0.0 for p in projects} for v in tv_exposed} + + weight = _minority_weight(preferences) + + def fit(votes, latent_rank): + """One candidate fit, in the shape :py:func:`_tune` expects.""" + return _fit_mf(votes, latent_rank, weight) + + if rank is None: + rank = _tune(preferences, ranks, fit, validation_fraction, seed) + logger.info( + "train_matrix_factorization: factorising %d collected votes over %d " + "projects at rank %d, approvals weighted x%d", + len(preferences), len(projects), rank, weight, + ) + score = fit(preferences, rank) + return { + v: dict(zip(projects, score(("tv", v), projects))) for v in tv_exposed + } + + +# --------------------------------------------------------------------------- +# Section 2.1.2 - Hybrid Factorization Machines. +# --------------------------------------------------------------------------- +def _fit_fm(projects, voters, preferences, rank: int, ridge: float, weight: int): + """ + Fit one LightFM hybrid and return ``score(voter, projects)``. + + Examples + -------- + Like :py:func:`_fit_mf`, but each project also carries Table 2's attributes. + The example is marked skipped so the doctest suite still passes without the + optional ``lightfm`` installed; the same path is covered by the unit tests, + which skip themselves when the library is absent. + + >>> p1, p2 = Project("p1", 1), Project("p2", 1) + >>> known = [(("lv", 0), p1, 1), (("lv", 0), p2, 0)] + >>> score = _fit_fm([p1, p2], [("lv", 0)], known, 2, 0.0, 1) # doctest: +SKIP + >>> [0.0 <= v <= 1.0 for v in score(("lv", 0), [p1, p2])] # doctest: +SKIP + [True, True] + """ + from lightfm import LightFM + from lightfm.data import Dataset + + # Table 2's project attributes. Every project also keeps its own identity + # feature (LightFM adds one by default), so these content features sit + # alongside the latent factor rather than replacing it - that combination is + # the hybrid of Section 2.1.2. + categories = sorted({c for p in projects for c in p.categories}) + targets = sorted({t for p in projects for t in p.targets}) + dataset = Dataset() + dataset.fit( + voters, + projects, + item_features=(["cost"] + + [f"category={c}" for c in categories] + + [f"target={t}" for t in targets]), + ) + preference_matrix, signs = dataset.build_interactions( + (v, p, 2 * preference - 1) for v, p, preference in preferences + ) + # Section 5's minority-class weighting. LightFM does honour sample weights + # under the logistic loss - only the k-OS loss ignores them - so an approval + # simply contributes ``weight`` times as much to the loss as a rejection. + sample_weight = signs.copy() + sample_weight.data = np.where( + signs.data > 0, float(weight), 1.0 + ).astype(np.float32) + preference_matrix.data = signs.data + costs = np.array([float(p.cost) for p in projects]) + costs = (costs - costs.mean()) / costs.std() if costs.std() else costs * 0 + project_attributes = dataset.build_item_features( + ( + (p, {"cost": cost} + | {f"category={c}": 1.0 for c in p.categories} + | {f"target={t}": 1.0 for t in p.targets}) + for p, cost in zip(projects, costs) + ), + normalize=False, + ) + model = LightFM( + no_components=max(1, rank), + loss="logistic", + user_alpha=ridge, + item_alpha=ridge, + random_state=0, + ) + model.fit( + preference_matrix, + item_features=project_attributes, + sample_weight=sample_weight, + epochs=30, + num_threads=1, + ) + user_ids, _, item_ids, _ = dataset.mapping() + + def score(voter, wanted): + """The fitted approval probability of each ``wanted`` project.""" + raw = model.predict( + user_ids[voter], + np.array([item_ids[p] for p in wanted]), + item_features=project_attributes, + ) + return 1 / (1 + np.exp(-raw)) + + return score + + +def train_factorization_machines( + instance: Instance, + lv_profile: ApprovalProfile, + tv_exposed: TVExposed, + rank: int | None = 20, + ridge: float | None = 0.0, + *, + settings=DEFAULT_FM_SETTINGS, + validation_fraction: float = 0.15, + seed: int = 0, +) -> dict[str, dict[Project, float]]: + """ + Fit the paper's hybrid latent model ``y(v,p)=FM(v, p, attributes(p))``. + + What makes this module *hybrid* (Section 2.1.2) rather than plain matrix + factorization is that projects are described by content features as well as + by their own latent factor, so a voter's taste can transfer to a project she + was never asked about. The features are Table 2's project attributes, all of + which ``parse_pabulib`` fills in: the numeric ``cost`` (standardised), plus + one multi-hot indicator per ``category`` and per ``target`` population + segment - multi-hot because Section 4 notes that "a certain proposal can + consist multiple topics and population segments". An instance whose projects + carry no categories or targets falls back to cost alone. + + Approvals are up-weighted against the imbalance of Table 1. ``rank`` and + ``ridge`` are *not* tuned by default, a deliberate departure from Section 5: + pass ``rank=None, ridge=None`` to search ``settings`` by validation F1 + instead. + + Measured on Praga-Poludnie 2022 with the offline-popularity setup, the + search costs accuracy rather than buying it - FA 0.787 at the fixed + ``rank=20, ridge=0.0`` against 0.590 for whatever the search selects - as + well as fitting the model four times over. The reason is that Section 5.1's + F1 is measured per vote while Section 5.2's FA depends on the *ranking* of + projects surviving, and the two do not agree; the same disagreement is why + :py:func:`_fit_per_project` leaves the minority-class correction off. Ridge + is no help either: sweeping it over ``0 .. 0.1`` moves offline FA around + erratically (0.787, 0.782, 0.346, 0.753, 0.007) without ever reaching the + 1.000 that plain matrix factorization gets on the same split. + + .. note:: + The content features are worth having under ``random`` sampling and + harmful under the offline setups, measured on that same district: + + ============================ ================== ======== + FM item features offline_popularity random + ============================ ================== ======== + none at all (i.e. plain MF) 1.000 0.927 + cost only 0.814 0.955 + cost + categories + targets 0.590 1.000 + ============================ ================== ======== + + The offline setups expose every voter to the *same*, deliberately + unrepresentative k projects (the most popular, or the most divisive), so + the shared content embeddings are fitted on that slice and then applied + to the projects nobody was asked about. Under ``random`` the exposed + projects are a fair sample and the features help. The paper treats the + sampling module and the prediction module as independent choices + (Section 3); this is evidence that they interact. + + Examples + -------- + The hybrid counterpart of :py:func:`train_matrix_factorization`: same + voter-project preferences, but every project also described by its cost, + categories and target segments. Skipped so the suite passes without the + optional ``lightfm``; the unit tests cover this path and skip themselves + when it is missing. + + >>> green = Project("green", 1, categories={"environment"}) + >>> road = Project("road", 2, categories={"transport"}) + >>> inst = Instance([green, road], budget_limit=3) + >>> lv = ApprovalProfile([ApprovalBallot([green])] * 3) + >>> scores = train_factorization_machines( # doctest: +SKIP + ... inst, lv, {"v": ({green}, {road})}) + >>> scores["v"][green] > scores["v"][road] # doctest: +SKIP + True + """ + try: + import lightfm # noqa: F401 (needed by _fit_fm) + except ImportError: + raise ImportError( + "You need lightfm to train the factorization-machines predictor " + "(pip install pabutools[recommendation])." + ) + + projects, preferences = _known_preferences(instance, lv_profile, tv_exposed) + voters = [("lv", i) for i in range(lv_profile.num_ballots())] + [ + ("tv", v) for v in tv_exposed + ] + if not projects or not preferences: + return {v: {p: 0.0 for p in projects} for v in tv_exposed} + + weight = _minority_weight(preferences) + + def fit(votes, setting): + """One candidate fit, in the shape :py:func:`_tune` expects.""" + return _fit_fm(projects, voters, votes, *setting, weight) + + if rank is None and ridge is None: + rank, ridge = _tune(preferences, settings, fit, validation_fraction, seed) + else: + rank, ridge = (20 if rank is None else rank, 0.0 if ridge is None else ridge) + logger.info( + "train_factorization_machines: fitting %d collected votes over %d " + "projects at rank %d, ridge %g, approvals weighted x%d, with %d " + "categories and %d target segments as content features", + len(preferences), len(projects), rank, ridge, weight, + len({c for p in projects for c in p.categories}), + len({t for p in projects for t in p.targets}), + ) + score = fit(preferences, (rank, ridge)) + return { + v: dict(zip(projects, score(("tv", v), projects))) for v in tv_exposed + } + + +# --------------------------------------------------------------------------- +# Complete the Target Voters' partial ballots with the fitted models. +# --------------------------------------------------------------------------- +def _preference_sets( + instance: Instance, ballot: CardinalBallot +) -> tuple[set[Project], set[Project], set[Project]]: + """ + Decode a partial ballot into A_v, D_v, and H_v. + + Examples + -------- + p1 is approved and p2 rejected; p3 was never scored, so it is hidden. + + >>> p1, p2, p3 = Project("p1", 1), Project("p2", 1), Project("p3", 1) + >>> inst = Instance([p1, p2, p3], budget_limit=3) + >>> a, d, h = _preference_sets(inst, CardinalBallot({p1: 1, p2: -1})) + >>> a == {p1}, d == {p2}, h == {p3} + (True, True, True) + """ + approved = {p for p, preference in ballot.items() if preference > 0} + disapproved = {p for p, preference in ballot.items() if preference < 0} + hidden = set(instance) - approved - disapproved + return approved, disapproved, hidden + + +def _complete_by_scores( + instance: Instance, lv_profile: ApprovalProfile, + tv_ballots: dict[str, CardinalBallot], train, +) -> dict[str, ApprovalBallot]: + """ + Complete every TV ballot from a collaborative-filtering score: ``train`` is + fitted on the LV ballots plus all the exposed sets E_TV, then a hidden + project is approved iff its score is >= 0.5, while A_v is kept and D_v stays + rejected. The MF and FM modules differ only in ``train``, so they share this. + + Examples + -------- + A stand-in model that scores p3 highly and everything else low, applied to a + voter who approved p1, rejected p2 and was never asked about p3. Her + approval of p1 survives, her rejection of p2 survives, and only the hidden + p3 is decided by the score. + + >>> p1, p2, p3 = Project("p1", 1), Project("p2", 1), Project("p3", 1) + >>> inst = Instance([p1, p2, p3], budget_limit=3) + >>> def train(instance, lv_profile, exposed): + ... return {v: {p: (0.9 if p == p3 else 0.1) for p in instance} + ... for v in exposed} + >>> done = _complete_by_scores( + ... inst, ApprovalProfile([]), {"v": CardinalBallot({p1: 1, p2: -1})}, train) + >>> done["v"] == ApprovalBallot({p1, p3}) + True + """ + sets = {v: _preference_sets(instance, b) for v, b in tv_ballots.items()} + scores = train(instance, lv_profile, + {v: (approved, disapproved) + for v, (approved, disapproved, _) in sets.items()}) + completed = { + v: ApprovalBallot(approved | {p for p in hidden if scores[v][p] >= 0.5}) + for v, (approved, _, hidden) in sets.items() + } + logger.info( + "%s: completed %d TV ballots, %d of %d hidden votes scored >= 0.5 " + "and became approvals", + getattr(train, "__name__", "collaborative filtering"), len(completed), + sum(len(completed[v]) - len(sets[v][0]) for v in sets), + sum(len(sets[v][2]) for v in sets), + ) + return completed + + +def predict_by_classification( + instance: Instance, + lv_profile: ApprovalProfile, + tv_ballots: dict[str, CardinalBallot], + **kwargs, +) -> dict[str, ApprovalBallot]: + """ + Prediction module - binary classification (Section 2.1.1): predict each + hidden project with a per-project binary classifier. One classifier is + trained for every project p, using preferences on P \\ {p} as input and the + preference on p as output. The collection is trained once by + :py:func:`train_classification` - on the LV ballots *and* the Target Voters' + exposed votes E_TV, per Section 3.1.1 - and applied to every TV voter; + preferences outside E_v are passed to XGBoost as missing values. Exposed + approvals A_v are kept and exposed disapprovals D_v stay rejected. + + Parameters + ---------- + instance : :py:class:`~pabutools.election.instance.Instance` + The PB instance. + lv_profile : :py:class:`~pabutools.election.profile.approvalprofile.ApprovalProfile` + The full ballots of the LV voters, used as the training data. + tv_ballots : dict[str, :py:class:`~pabutools.election.ballot.cardinalballot.CardinalBallot`] + Every Target Voter's partial ballot to complete, keyed by voter id. + Build these with ``pabutools.recommendation.partial_ballot`` / + ``reveal_ballot`` rather than by hand; the score convention is + +1 approved, -1 disapproved, 0 (or absent) hidden. + **kwargs + Forwarded to :py:func:`train_classification`. Pass ``param_grid=()`` + to skip the Section 5 hyperparameter search, which costs roughly ten + times a plain fit and is worth switching off for large sweeps. + + Returns + ------- + dict[str, :py:class:`~pabutools.election.ballot.approvalballot.ApprovalBallot`] + The full predicted approval ballot of every TV voter, keyed by + voter id. + + Examples + -------- + 2 LV voters both approve {p1, p2}. A TV voter with nothing exposed is + completed to {p1, p2} - on such an unambiguous pattern any correct + predictor (XGBoost included) agrees with the LV majority. + + >>> p1, p2, p3 = Project("p1", 4), Project("p2", 4), Project("p3", 6) + >>> inst = Instance([p1, p2, p3], budget_limit=6) + >>> lv = ApprovalProfile([ApprovalBallot([p1, p2]), ApprovalBallot([p1, p2])]) + >>> partial = CardinalBallot({p1: 0, p2: 0, p3: 0}) # nothing exposed + >>> predict_by_classification(inst, lv, {"v3": partial})["v3"] == {p1, p2} + True + """ + model = train_classification(instance, lv_profile, tv_ballots, **kwargs) + projects = sorted(instance, key=str) + voters = list(tv_ballots) + preferences = { + vid: _preference_sets(instance, tv_ballots[vid]) for vid in voters + } + # One row per Target Voter over every project, in the same column order the + # classifiers were fitted on: 1 approved, 0 disapproved, NaN not asked. + votes = np.array( + [ + [ + 1.0 if p in approved else 0.0 if p in disapproved else np.nan + for p in projects + ] + for approved, disapproved, _ in (preferences[v] for v in voters) + ], + dtype=float, + ).reshape(len(voters), len(projects)) + + # Classify per *project*, not per (voter, project): every voter with that + # project hidden goes through XGBoost in one call. On a Warsaw district this + # is ~100 calls rather than ~700 000. + predicted: dict[str, set[Project]] = {vid: set() for vid in voters} + for index, project in enumerate(projects): + rows = np.flatnonzero(np.isnan(votes[:, index])) # hidden for these + if not len(rows): + continue + kind, payload = model["per_project"][project] + if kind == "const": + labels = np.full(len(rows), payload) + else: + # Dropping this project's column reproduces the ``input_projects`` + # order the classifier was trained on. + labels = payload[1].predict(np.delete(votes[rows], index, axis=1)) + for row, label in zip(rows, labels): + if int(label) == 1: + predicted[voters[row]].add(project) + logger.info( + "predict_by_classification: completed %d TV ballots in %d batched calls", + len(voters), len(projects), + ) + return { + vid: ApprovalBallot(preferences[vid][0] | predicted[vid]) + for vid in voters + } + + +def predict_by_matrix_factorization( + instance: Instance, + lv_profile: ApprovalProfile, + tv_ballots: dict[str, CardinalBallot], +) -> dict[str, ApprovalBallot]: + """ + Prediction module - collaborative filtering via Matrix Factorization + (Section 2.1.2): one voter-project matrix holds the LV ballots and *every* TV + voter's exposed votes (the paper's E_TV; approve=1, disapprove=0), it is + factorised once by :py:func:`train_matrix_factorization`, and a hidden + project is predicted approved iff its reconstructed score is >= 0.5. + Approvals are up-weighted against the imbalance of Table 1 and the latent + rank is chosen on a 15% validation set (Section 5). Exposed approvals A_v + are kept and exposed disapprovals D_v stay rejected. + + Parameters + ---------- + instance : :py:class:`~pabutools.election.instance.Instance` + The PB instance. + lv_profile : :py:class:`~pabutools.election.profile.approvalprofile.ApprovalProfile` + The full ballots of the LV voters, used as the training data. + tv_ballots : dict[str, :py:class:`~pabutools.election.ballot.cardinalballot.CardinalBallot`] + Every Target Voter's partial ballot to complete, keyed by voter id. + + Returns + ------- + dict[str, :py:class:`~pabutools.election.ballot.approvalballot.ApprovalBallot`] + The full predicted approval ballot of every TV voter, keyed by + voter id. + + Examples + -------- + Three LV voters all support the cheap pair {p1, p2} and reject {p3, p4}. + The reconstructed matrix completes a TV voter with nothing exposed to + {p1, p2}. + + >>> p1, p2, p3, p4 = (Project("p1", 3), Project("p2", 3), + ... Project("p3", 4), Project("p4", 4)) + >>> inst = Instance([p1, p2, p3, p4], budget_limit=6) + >>> lv = ApprovalProfile([ApprovalBallot([p1, p2])] * 3) + >>> partial = CardinalBallot({p1: 0, p2: 0, p3: 0, p4: 0}) # nothing exposed + >>> predict_by_matrix_factorization(inst, lv, {"v4": partial})["v4"] == {p1, p2} + True + """ + return _complete_by_scores( + instance, lv_profile, tv_ballots, train_matrix_factorization + ) + + +def predict_by_factorization_machines( + instance: Instance, + lv_profile: ApprovalProfile, + tv_ballots: dict[str, CardinalBallot], +) -> dict[str, ApprovalBallot]: + """ + Prediction module - hybrid Factorization Machines (Section 2.1.2): fitted by + :py:func:`train_factorization_machines` on the same LV + E_TV voter-project + preferences as Matrix Factorization, but with each project also described by + Table 2's attributes - its cost, its categories and its target population + segments - which is what makes the module hybrid rather than plain MF. + Approvals are up-weighted against the imbalance of Table 1 and the latent + rank and regularisation are chosen on a 15% validation set (Section 5). A + hidden project is predicted approved iff the FM score is >= 0.5; exposed + approvals A_v are kept and exposed disapprovals D_v stay rejected. + + Parameters + ---------- + instance : :py:class:`~pabutools.election.instance.Instance` + The PB instance. + lv_profile : :py:class:`~pabutools.election.profile.approvalprofile.ApprovalProfile` + The full ballots of the LV voters, used as the training data. + tv_ballots : dict[str, :py:class:`~pabutools.election.ballot.cardinalballot.CardinalBallot`] + Every Target Voter's partial ballot to complete, keyed by voter id. + + Returns + ------- + dict[str, :py:class:`~pabutools.election.ballot.approvalballot.ApprovalBallot`] + The full predicted approval ballot of every TV voter, keyed by + voter id. + + Examples + -------- + 2 LV voters both approve {p1, p2}. A TV voter with nothing exposed is + completed to {p1, p2}. Skipped so the doctest suite passes without the + optional ``lightfm``; the unit tests cover this and skip themselves when the + library is absent. + + >>> p1, p2, p3 = Project("p1", 4), Project("p2", 4), Project("p3", 6) + >>> inst = Instance([p1, p2, p3], budget_limit=6) + >>> lv = ApprovalProfile([ApprovalBallot([p1, p2]), ApprovalBallot([p1, p2])]) + >>> partial = CardinalBallot({p1: 0, p2: 0, p3: 0}) # nothing exposed + >>> predict_by_factorization_machines( # doctest: +SKIP + ... inst, lv, {"v3": partial})["v3"] == {p1, p2} + True + """ + return _complete_by_scores( + instance, lv_profile, tv_ballots, train_factorization_machines + ) + + +if __name__ == "__main__": + import doctest + + doctest.testmod(verbose=True) diff --git a/pabutools/recommendation/recommendation.py b/pabutools/recommendation/recommendation.py new file mode 100644 index 00000000..e06bd119 --- /dev/null +++ b/pabutools/recommendation/recommendation.py @@ -0,0 +1,1614 @@ +""" +An implementation of the algorithms in: +"A Recommendation System for Participatory Budgeting", +by Gil Leibiker and Nimrod Talmon (2023), https://optlearnmas23.github.io/files/p17.pdf + +Programmer: Roei Yanku +Date: 2026-06-20. +""" + +from __future__ import annotations + +import logging +import random +from collections.abc import Callable, Iterable + +from pabutools.election import ( + Instance, + Project, + ApprovalProfile, + ApprovalBallot, + CardinalBallot, + total_cost, +) +from pabutools.election.satisfaction import Cost_Sat +from pabutools.rules import BudgetAllocation, greedy_utilitarian_welfare + +# All model fitting and prediction lives in ``model_training``; the three +# Section 2.1 prediction modules are used here as the ``predict`` argument of +# the pipeline and are listed in ``PREDICTORS``. +from pabutools.recommendation.model_training import ( + predict_by_classification, + predict_by_matrix_factorization, + predict_by_factorization_machines, +) + +# Log the steps of every algorithm at INFO/DEBUG level. +# Configure a handler in your own script to see them, e.g. +# ``logging.basicConfig(level=logging.INFO)``. +logger = logging.getLogger(__name__) + + +# =========================================================================== +# Section 2.3 - Partial (three-state) ballots, encoded as a CardinalBallot. +# =========================================================================== +# A Target Voter answers only k projects, so "unknown" must be distinct from +# "disapproved" - which a plain approval set (just the approved projects) cannot +# express. On the pabutools maintainer's advice (Simon Rey) we do NOT add a new +# ballot type; instead we reuse the existing +# :py:class:`~pabutools.election.ballot.cardinalballot.CardinalBallot` (a dict of +# project -> score) under this sign convention: +# +# score > 0 -> APPROVED (A_v) we store +1 (APPROVAL) +# score < 0 -> DISAPPROVED (D_v) we store -1 (DISAPPROVAL) +# score == 0 -> HIDDEN (H_v) we store 0 (HIDDEN) +# +# A project simply absent from the ballot is also HIDDEN. This convention is not +# self-evident, so never hand-write the scores: build ballots with +# ``partial_ballot`` / ``reveal_ballot`` and read them back with the +# ``*_projects`` accessors / ``as_approval_ballot``. + +#: Score stored for an approved project (A_v). Any strictly positive score works. +APPROVAL = 1 +#: Score stored for a disapproved project (D_v). Any strictly negative score works. +DISAPPROVAL = -1 +#: Score stored for a hidden project (H_v); absent projects mean the same. +HIDDEN = 0 + + +def partial_ballot( + approved: Iterable[Project] = (), + disapproved: Iterable[Project] = (), + hidden: Iterable[Project] = (), +) -> CardinalBallot: + """ + Build a partial ballot from the three explicit sets, hiding the convention: + approved projects get +1, disapproved get -1, and hidden get 0. This is the + intended way to create a partial ballot - callers name the three sets instead + of writing raw scores. + + Examples + -------- + >>> p1, p2, p3 = Project("p1", 1), Project("p2", 1), Project("p3", 1) + >>> b = partial_ballot(approved={p1}, disapproved={p2}, hidden={p3}) + >>> b[p1], b[p2], b[p3] + (1, -1, 0) + >>> exposed_projects(b) == {p1, p2} + True + """ + return CardinalBallot( + {p: APPROVAL for p in approved} + | {p: DISAPPROVAL for p in disapproved} + | {p: HIDDEN for p in hidden} + ) + + +def reveal_ballot( + instance: Instance, + full_ballot: set[Project], + exposed: set[Project], +) -> CardinalBallot: + """ + Build the partial ballot exposing a set of projects of a voter whose full + ballot in the ideal instance is known (Section 2.3): A_v = E_v ∩ full_ballot, + D_v = E_v \\ full_ballot, H_v = P \\ E_v. + + Parameters + ---------- + instance : :py:class:`~pabutools.election.instance.Instance` + The PB instance (used as the universe of projects P). + full_ballot : set[:py:class:`~pabutools.election.instance.Project`] + The voter's full ballot - her approval set A_v in the ideal instance. + exposed : set[:py:class:`~pabutools.election.instance.Project`] + The projects revealed for this voter (the exposed set E_v). + + Returns + ------- + :py:class:`~pabutools.election.ballot.cardinalballot.CardinalBallot` + The corresponding three-state ballot under the sign convention. + + Examples + -------- + Example 2.4 from the paper: P = {p1, p2, p3, p4}, the voter's full ballot is + {p1, p2}, and {p1, p3} is exposed. Then p1 is approved, p3 disapproved, and + p2, p4 remain hidden. + + >>> p1, p2, p3, p4 = (Project("p1", 1), Project("p2", 1), + ... Project("p3", 1), Project("p4", 1)) + >>> inst = Instance([p1, p2, p3, p4], budget_limit=4) + >>> b = reveal_ballot(inst, {p1, p2}, {p1, p3}) + >>> approved_projects(b) == {p1}, disapproved_projects(b) == {p3} + (True, True) + >>> hidden_projects(b, inst) == {p2, p4} + True + """ + return partial_ballot( + approved=exposed & full_ballot, # A_v = E_v ∩ full ballot + disapproved=exposed - full_ballot, # D_v = E_v \ full ballot + hidden=set(instance) - exposed, # H_v = P \ E_v + ) + + +def approved_projects(ballot: CardinalBallot) -> set[Project]: + """ + The approval set A_v: the projects with a strictly positive score. + + Examples + -------- + >>> p1, p2, p3 = Project("p1", 1), Project("p2", 1), Project("p3", 1) + >>> b = partial_ballot(approved={p1}, disapproved={p2}, hidden={p3}) + >>> approved_projects(b) == {p1} + True + """ + return {project for project, score in ballot.items() if score > 0} + + +def disapproved_projects(ballot: CardinalBallot) -> set[Project]: + """ + The disapproval set D_v: the projects with a strictly negative score. + + Examples + -------- + >>> p1, p2, p3 = Project("p1", 1), Project("p2", 1), Project("p3", 1) + >>> b = partial_ballot(approved={p1}, disapproved={p2}, hidden={p3}) + >>> disapproved_projects(b) == {p2} + True + """ + return {project for project, score in ballot.items() if score < 0} + + +def exposed_projects(ballot: CardinalBallot) -> set[Project]: + """ + The exposed set E_v = A_v ∪ D_v: the projects with a non-zero score. + + Examples + -------- + The hidden p3 is left out, whichever way the other two went. + + >>> p1, p2, p3 = Project("p1", 1), Project("p2", 1), Project("p3", 1) + >>> b = partial_ballot(approved={p1}, disapproved={p2}, hidden={p3}) + >>> exposed_projects(b) == {p1, p2} + True + """ + return {project for project, score in ballot.items() if score != 0} + + +def hidden_projects(ballot: CardinalBallot, instance: Instance) -> set[Project]: + """ + The hidden set H_v = P \\ E_v: every project of the instance that the voter + was not asked about (score 0 or absent from the ballot). + + Examples + -------- + p3 scores 0 and p4 is absent from the ballot altogether; both count as + hidden, so the voter is treated as never having been asked about either. + + >>> p1, p2, p3, p4 = (Project("p1", 1), Project("p2", 1), + ... Project("p3", 1), Project("p4", 1)) + >>> inst = Instance([p1, p2, p3, p4], budget_limit=4) + >>> b = partial_ballot(approved={p1}, disapproved={p2}, hidden={p3}) + >>> hidden_projects(b, inst) == {p3, p4} + True + """ + return set(instance) - exposed_projects(ballot) + + +def as_approval_ballot(ballot: CardinalBallot) -> ApprovalBallot: + """ + The pabutools approval ballot made of the approved projects A_v, so that a + (completed) partial ballot can be fed to approval-based voting rules. + + Examples + -------- + >>> p1, p2, p3 = Project("p1", 1), Project("p2", 1), Project("p3", 1) + >>> b = partial_ballot(approved={p1}, disapproved={p2}, hidden={p3}) + >>> as_approval_ballot(b) == ApprovalBallot({p1}) + True + """ + return ApprovalBallot(approved_projects(ballot)) + + +# --------------------------------------------------------------------------- +# Section 2.2.3 - Popularity and consensus (primitives used by every module). +# --------------------------------------------------------------------------- +# Definition 2.1 (Approval scores) needs no function of our own: it is exactly +# pabutools' ``profile.approval_scores()``. Callers below use the library method +# directly (``.get(p, 0)`` supplies the 0 for projects that nobody approved). +def consensus_levels( + instance: Instance, profile: ApprovalProfile +) -> dict[Project, int]: + """ + Definition 2.2 (Consensus levels): the absolute difference between the number + of voters approving a project and the number disapproving it (so a unanimous + approval or a unanimous rejection both reach the maximum n). + + Parameters + ---------- + instance : :py:class:`~pabutools.election.instance.Instance` + The PB instance. + profile : :py:class:`~pabutools.election.profile.approvalprofile.ApprovalProfile` + The approval profile. + + Returns + ------- + dict[:py:class:`~pabutools.election.instance.Project`, int] + A mapping from each project to its consensus level. + + Examples + -------- + Four LV voters: p1 approved 4/0, p2 split 2/2, p3 rejected 0/4. Both p1 + and p3 are in full consensus, p2 in none. + + >>> p1, p2, p3 = Project("p1", 1), Project("p2", 1), Project("p3", 1) + >>> inst = Instance([p1, p2, p3], budget_limit=3) + >>> prof = ApprovalProfile([ApprovalBallot([p1, p2]), ApprovalBallot([p1]), + ... ApprovalBallot([p1, p2]), ApprovalBallot([p1])]) + >>> c = consensus_levels(inst, prof) + >>> [c[p] for p in (p1, p2, p3)] + [4, 0, 4] + """ + # consensus(p) = |approvers(p) - disapprovers(p)|. With n voters and + # score(p) approvers, disapprovers = n - score(p), so the difference is + # |score - (n - score)| = |2*score - n|. Scores come from the library + # (Definition 2.1 = pabutools' approval_scores()). + n = profile.num_ballots() + scores = profile.approval_scores() + consensus = { + project: abs(2 * scores.get(project, 0) - n) for project in instance + } + logger.debug("consensus_levels (n=%d): %s", n, consensus) + return consensus + + +def most_consensual_projects( + instance: Instance, profile: ApprovalProfile +) -> list[Project]: + """ + The paper's ordering gamma (Section 2.2.3): the projects sorted by decreasing + consensus level, ties broken lexicographically by project name. gamma_1 is the + project most in consensus, gamma_m the "most controversial". Helper used by + offline revealing-by-consensus and by-controversiality. + + Examples + -------- + >>> p1, p2, p3 = Project("p1", 1), Project("p2", 1), Project("p3", 1) + >>> inst = Instance([p1, p2, p3], budget_limit=3) + >>> prof = ApprovalProfile([ApprovalBallot([p1, p2]), ApprovalBallot([p1]), + ... ApprovalBallot([p1, p2]), ApprovalBallot([p1])]) + >>> most_consensual_projects(inst, prof) # consensus p1=4, p3=4, p2=0 + [p1, p3, p2] + """ + consensus = consensus_levels(instance, profile) + gamma = sorted(instance, key=lambda project: (-consensus[project], str(project))) + if gamma: + logger.debug( + "most_consensual_projects: gamma_1=%s (consensus %d), " + "gamma_m=%s (consensus %d, the most controversial)", + gamma[0], consensus[gamma[0]], gamma[-1], consensus[gamma[-1]], + ) + return gamma + + +# --------------------------------------------------------------------------- +# Section 2.2.5 - The voting rule (greedy approval). +# --------------------------------------------------------------------------- +def greedy_approval( + instance: Instance, profile: ApprovalProfile +) -> BudgetAllocation: + """ + Greedy approval voting rule (Section 2.2.5). Projects are considered in + decreasing order of their approval score and funded while the budget + allows; a project that would exceed the remaining budget is skipped. Ties + are broken lexicographically by project name. + + .. note:: + Delegates to pabutools' + :py:func:`~pabutools.rules.greedywelfare.greedy_utilitarian_welfare` with + :py:class:`~pabutools.election.satisfaction.additivesatisfaction.Cost_Sat`: + the rule ranks by marginal satisfaction divided by cost, and under + Cost_Sat a project's marginal satisfaction is score(p)*cost(p), so the + ranking is by the **raw** approval score - exactly the paper's greedy + approval. (With the default Cardinality_Sat the ranking would be + score/cost, which is a different rule.) + + Parameters + ---------- + instance : :py:class:`~pabutools.election.instance.Instance` + The PB instance. + profile : :py:class:`~pabutools.election.profile.approvalprofile.ApprovalProfile` + The (completed) approval profile. + + Returns + ------- + :py:class:`~pabutools.rules.budgetallocation.BudgetAllocation` + The winning bundle. + + Examples + -------- + Example 2.3 from the paper: scores p1 = p2 = 2, p3 = 1, budget 3. p1 and p2 + are funded (cost 1 each); p3 (cost 2) no longer fits. + + >>> p1, p2, p3 = Project("p1", 1), Project("p2", 1), Project("p3", 2) + >>> inst = Instance([p1, p2, p3], budget_limit=3) + >>> prof = ApprovalProfile([ApprovalBallot([p1, p2]), + ... ApprovalBallot([p1, p3]), + ... ApprovalBallot([p2])]) + >>> sorted(greedy_approval(inst, prof), key=str) + [p1, p2] + """ + chosen = greedy_utilitarian_welfare( + instance, profile, sat_class=Cost_Sat, resoluteness=True + ) + assert isinstance(chosen, BudgetAllocation) # resolute mode: one allocation + logger.info( + "greedy_approval: funded %d/%d projects, cost %s of %s", + len(chosen), len(instance), total_cost(chosen), instance.budget_limit, + ) + return chosen + + +# --------------------------------------------------------------------------- +# Section 3.1.1 - Random setup. +# --------------------------------------------------------------------------- +def random_setup( + instance: Instance, + k: int, + seed: int | None = None, +) -> set[Project]: + """ + Random setup (Sections 2.4 / 3.1.1): the exposed set E_v of one Target + Voter, made of k projects chosen uniformly at random from P (the rest is + predicted later). Called once per TV voter with its own seed, so each + voter gets an independent draw ("possibly different E for each voter"); + it needs no ballot, since the choice is random. + + Parameters + ---------- + instance : :py:class:`~pabutools.election.instance.Instance` + The PB instance (the universe of projects P). + k : int + The number of projects to expose. + seed : int, optional + Seed for the random generator, for reproducibility. + + Returns + ------- + set[:py:class:`~pabutools.election.instance.Project`] + The exposed set E_v: k projects drawn uniformly at random from P. + + Examples + -------- + With k = 2, whatever the draw, exactly two projects are exposed and they + are real projects of the instance. + + >>> p1, p2, p3, p4 = (Project("p1", 2), Project("p2", 2), + ... Project("p3", 3), Project("p4", 3)) + >>> inst = Instance([p1, p2, p3, p4], budget_limit=6) + >>> exposed = random_setup(inst, k=2, seed=0) + >>> len(exposed) == 2 and exposed <= {p1, p2, p3, p4} + True + """ + rng = random.Random(seed) + # Sort first so the sample is reproducible from the seed (a set has no + # deterministic iteration order). + population = sorted(instance, key=str) + exposed = set(rng.sample(population, k)) + logger.info("random_setup: exposed %d of %d projects at random", k, len(population)) + return exposed + + +# --------------------------------------------------------------------------- +# Section 3.1.2 - Offline setup (popularity / consensus / controversiality). +# --------------------------------------------------------------------------- +def offline_popularity( + instance: Instance, lv_profile: ApprovalProfile, k: int +) -> set[Project]: + """ + Offline revealing by popularity (Section 3.1.2). The exposed + set E = {sigma_1, ..., sigma_k}: the k most approved projects among the LV + voters. The same set is exposed to every Target Voter, so it depends only on + the LV profile (the score ordering sigma is used only to pick the top k; the + exposed set itself is unordered). + + Parameters + ---------- + instance : :py:class:`~pabutools.election.instance.Instance` + The PB instance. + lv_profile : :py:class:`~pabutools.election.profile.approvalprofile.ApprovalProfile` + The full ballots of the Learning Voters (LV). + k : int + The number of projects to expose. + + Returns + ------- + set[:py:class:`~pabutools.election.instance.Project`] + The exposed set E of the k most popular projects (ties broken by name + when picking the top k). + + Examples + -------- + LV scores p1 = 3, p2 = 1, p3 = 1, so the single most popular project is p1. + + >>> p1, p2, p3 = Project("p1", 1), Project("p2", 1), Project("p3", 1) + >>> inst = Instance([p1, p2, p3], budget_limit=3) + >>> lv = ApprovalProfile([ApprovalBallot([p1, p3]), + ... ApprovalBallot([p1, p2]), + ... ApprovalBallot([p1])]) + >>> offline_popularity(inst, lv, k=1) == {p1} + True + """ + # The paper's sigma ordering: decreasing library approval score, ties by name. + scores = lv_profile.approval_scores() + sigma = sorted(instance, key=lambda p: (-scores.get(p, 0), str(p))) + exposed = set(sigma[:k]) + logger.info("offline_popularity: exposing top-%d popular projects", k) + return exposed + + +def offline_consensus( + instance: Instance, lv_profile: ApprovalProfile, k: int +) -> set[Project]: + """ + Offline revealing by consensus (Section 3.1.2). The exposed set + E = {gamma_1, ..., gamma_k}: the k projects with the highest consensus level + among the LV voters. The same set is exposed to every Target Voter. + + Parameters + ---------- + instance : :py:class:`~pabutools.election.instance.Instance` + The PB instance. + lv_profile : :py:class:`~pabutools.election.profile.approvalprofile.ApprovalProfile` + The full ballots of the Learning Voters (LV). + k : int + The number of projects to expose. + + Returns + ------- + set[:py:class:`~pabutools.election.instance.Project`] + The exposed set E of the k projects most in consensus (ties broken by + name when picking the top k). + + Examples + -------- + p1 and p3 both have consensus 4 (p1 unanimously approved, p3 unanimously + rejected); the tie is broken by name towards p1. + + >>> p1, p2, p3 = Project("p1", 1), Project("p2", 1), Project("p3", 1) + >>> inst = Instance([p1, p2, p3], budget_limit=3) + >>> lv = ApprovalProfile([ApprovalBallot([p1, p2]), ApprovalBallot([p1]), + ... ApprovalBallot([p1, p2]), ApprovalBallot([p1])]) + >>> offline_consensus(inst, lv, k=1) == {p1} + True + """ + exposed = set(most_consensual_projects(instance, lv_profile)[:k]) + logger.info("offline_consensus: exposing top-%d consensual projects", k) + return exposed + + +def offline_controversiality( + instance: Instance, lv_profile: ApprovalProfile, k: int +) -> set[Project]: + """ + Offline revealing by controversiality (Section 3.1.2). The exposed set + E = {gamma_{m-k+1}, ..., gamma_m}: the k projects *least* in consensus + among the LV voters (the hardest to predict, so asked directly). The same + set is exposed to every Target Voter. (The paper writes + {gamma_{m-k}, ..., gamma_m}, which is k+1 projects - an off-by-one; its + prose says "the k projects least in consensus", implemented here.) + + Parameters + ---------- + instance : :py:class:`~pabutools.election.instance.Instance` + The PB instance. + lv_profile : :py:class:`~pabutools.election.profile.approvalprofile.ApprovalProfile` + The full ballots of the Learning Voters (LV). + k : int + The number of projects to expose. + + Returns + ------- + set[:py:class:`~pabutools.election.instance.Project`] + The exposed set E of the k most controversial projects (ties broken + by name when picking the bottom k). + + Examples + -------- + Same data as the consensus example: p2 is split exactly in half + (consensus 0) and is therefore the single most controversial project. + + >>> p1, p2, p3 = Project("p1", 1), Project("p2", 1), Project("p3", 1) + >>> inst = Instance([p1, p2, p3], budget_limit=3) + >>> lv = ApprovalProfile([ApprovalBallot([p1, p2]), ApprovalBallot([p1]), + ... ApprovalBallot([p1, p2]), ApprovalBallot([p1])]) + >>> offline_controversiality(inst, lv, k=1) == {p2} + True + """ + # The k *least* consensual projects are the last k of gamma, + # {gamma_{m-k+1}, ..., gamma_m}. Slicing from m-k keeps k=0 correct. + gamma = most_consensual_projects(instance, lv_profile) + exposed = set(gamma[len(gamma) - k:]) + logger.info("offline_controversiality: exposing bottom-%d consensual projects", k) + return exposed + + +# --------------------------------------------------------------------------- +# Section 3.1.3 - Online setup (adaptive controversial). +# --------------------------------------------------------------------------- +def online_adaptive_controversial( + instance: Instance, + lv_profile: ApprovalProfile, + tv_ballots: dict[str, set[Project]], + k: int, +) -> dict[str, set[Project]]: + """ + Online adaptive-controversial setup (Section 3.1.3): the exposed set E_v of + every Target Voter, keyed by voter id. The voters are queried one after the + other (in ``tv_ballots`` order), each in k iterations. In every iteration + the most controversial not-yet-asked project is recomputed over *everyone + who has voted on it so far* - the LV voters plus every TV answer already + collected - and the current voter's answer (simulated from her full ballot) + is folded into that tally. Each answer thus shifts which projects later + voters are asked about; this feedback is what makes the setup adaptive and + distinguishes it from offline revealing-by-controversiality, per the + paper's "choose project proposals based on all voters preferences + iteratively" (Section 6). + + Parameters + ---------- + instance : :py:class:`~pabutools.election.instance.Instance` + The PB instance. + lv_profile : :py:class:`~pabutools.election.profile.approvalprofile.ApprovalProfile` + The full ballots of the Learning Voters (LV). + tv_ballots : dict[str, set[:py:class:`~pabutools.election.instance.Project`]] + The full ballot of every Target Voter (her approval set A_v in the + ideal instance), keyed by voter id - used to answer the adaptive + questions. Voters are queried in iteration order. + k : int + The number of iterations / projects to expose per voter. + + Returns + ------- + dict[str, set[:py:class:`~pabutools.election.instance.Project`]] + The exposed set E_v of each Target Voter, keyed by voter id. + + Examples + -------- + One voter, 4 LV voters, k = 2: p1, p2, p3 are tied as most controversial + (consensus 0); the first question (tie broken by name) is p1, the second + p2, so E_v = {p1, p2}. + + >>> p1, p2, p3, p4 = (Project("p1", 1), Project("p2", 1), + ... Project("p3", 1), Project("p4", 1)) + >>> inst = Instance([p1, p2, p3, p4], budget_limit=4) + >>> lv = ApprovalProfile([ApprovalBallot([p1, p2]), ApprovalBallot([p1, p3]), + ... ApprovalBallot([p2, p3]), ApprovalBallot([])]) + >>> online_adaptive_controversial(inst, lv, {"v5": {p1, p2}}, k=2)["v5"] == {p1, p2} + True + + The adaptive feedback: p1 and p2 start tied as most controversial, so the + first voter is asked p1. Her approval breaks the tie - p1 now leans + approved and is *less* controversial - so the second voter is asked p2. + + >>> q1, q2, q3 = Project("q1", 1), Project("q2", 1), Project("q3", 1) + >>> inst2 = Instance([q1, q2, q3], budget_limit=3) + >>> lv2 = ApprovalProfile([ApprovalBallot([q1]), ApprovalBallot([q2])]) + >>> online_adaptive_controversial(inst2, lv2, {"v3": {q1}, "v4": {q1}}, k=1) + {'v3': {q1}, 'v4': {q2}} + """ + # Definition 2.2 over everyone who has voted on p so far: the LV electorate + # first, then each collected TV answer. consensus(p) = |approvers - disapprovers|. + scores = lv_profile.approval_scores() + approvers = {p: scores.get(p, 0) for p in instance} + disapprovers = {p: lv_profile.num_ballots() - approvers[p] for p in instance} + exposed_by_voter: dict[str, set[Project]] = {} + for vid, ballot in tv_ballots.items(): + exposed: set[Project] = set() + for _ in range(k): + # gamma_m of the not-yet-asked projects: least consensus, ties by name. + asked = min( + set(instance) - exposed, + key=lambda p: (abs(approvers[p] - disapprovers[p]), str(p)), + ) + logger.info( + "online_adaptive_controversial: asks %s -> voter %s %s", asked, + vid, "approves" if asked in ballot else "disapproves", + ) + (approvers if asked in ballot else disapprovers)[asked] += 1 + exposed.add(asked) + exposed_by_voter[vid] = exposed + return exposed_by_voter + + +def next_adaptive_question( + instance: Instance, + lv_profile: ApprovalProfile, + answers: CardinalBallot, + collected: Iterable[CardinalBallot] = (), +) -> Project: + """ + The single project to put to a voter next, under the online adaptive- + controversial setup (Section 3.1.3): the most controversial project she has + not been asked about yet, where controversiality is measured over everyone + who has voted on it so far. + + :py:func:`online_adaptive_controversial` runs this rule as a closed loop by + reading each answer off a known ballot, which a live process cannot do - the + answers are exactly what it does not have yet. This is that loop's single + step, exposed so a caller can drive it: ask the returned project, add the + reply to ``answers``, and call again until ``answers`` holds k votes. Feeding + the replies back is what makes the setup adaptive; it is the only setup of + the five that cannot be planned in advance with :py:func:`exposed_sets`. + + Parameters + ---------- + instance : :py:class:`~pabutools.election.instance.Instance` + The PB instance (the universe of projects P). + lv_profile : :py:class:`~pabutools.election.profile.approvalprofile.ApprovalProfile` + The full ballots of the Learning Voters (LV). + answers : :py:class:`~pabutools.election.ballot.cardinalballot.CardinalBallot` + The replies this voter has given so far, as a partial ballot: her + answered projects are the ones excluded from the next question. + Pass an empty ``partial_ballot()`` for the first question. + collected : Iterable[:py:class:`~pabutools.election.ballot.cardinalballot.CardinalBallot`], optional + The partial ballots already gathered from *other* voters. Their votes + count towards controversiality but never towards what this voter is + asked, exactly as in the closed loop. + + Returns + ------- + :py:class:`~pabutools.election.instance.Project` + The project to ask about, ties broken lexicographically by name. + + Raises + ------ + ValueError + If the voter has already been asked about every project. + + Examples + -------- + Two LV voters split over q1 and q2, nobody approving q3: q1 and q2 are tied + at consensus 0 and q3 is settled, so the first question is q1 (tie broken by + name). Her approval of q1 tips it towards approved, so the second question + is q2 - the same order the closed loop produces. + + >>> q1, q2, q3 = Project("q1", 1), Project("q2", 1), Project("q3", 1) + >>> inst = Instance([q1, q2, q3], budget_limit=3) + >>> lv = ApprovalProfile([ApprovalBallot([q1]), ApprovalBallot([q2])]) + >>> first = next_adaptive_question(inst, lv, partial_ballot()) + >>> first + q1 + >>> next_adaptive_question(inst, lv, partial_ballot(approved={first})) + q2 + + Once every project has been answered there is nothing left to ask: + + >>> answered = partial_ballot(approved={q1, q2}, disapproved={q3}) + >>> next_adaptive_question(inst, lv, answered) + Traceback (most recent call last): + ... + ValueError: this voter has already been asked about all 3 projects + """ + remaining = set(instance) - exposed_projects(answers) + if not remaining: + raise ValueError( + f"this voter has already been asked about all {len(instance)} projects" + ) + # Definition 2.2 over everyone who has voted on p so far: the LV electorate, + # every answer collected from other voters, and this voter's own replies. + scores = lv_profile.approval_scores() + approvers = {p: scores.get(p, 0) for p in instance} + disapprovers = {p: lv_profile.num_ballots() - approvers[p] for p in instance} + for ballot in (*collected, answers): + for project, score in ballot.items(): + if score > 0: + approvers[project] += 1 + elif score < 0: + disapprovers[project] += 1 + asked = min( + remaining, + key=lambda p: (abs(approvers[p] - disapprovers[p]), str(p)), + ) + logger.info( + "next_adaptive_question: asking %s (consensus %d of %d voters so far)", + asked, abs(approvers[asked] - disapprovers[asked]), + approvers[asked] + disapprovers[asked], + ) + return asked + + + + +# --------------------------------------------------------------------------- +# Full pipeline (sampling -> prediction -> greedy approval). +# --------------------------------------------------------------------------- +# The paper's design space is a matrix: one setup (Section 3.1) times one +# predictor (Section 2.1). These registries name every choice so a caller can +# pick a cell (``run_pipeline``) or sweep the whole matrix (``run_all_experiments``). +SETUPS = ( + "random", + "offline_popularity", + "offline_consensus", + "offline_controversiality", + "online_adaptive_controversial", +) + +PREDICTORS = { + "classification": predict_by_classification, + "matrix_factorization": predict_by_matrix_factorization, + "factorization_machines": predict_by_factorization_machines, +} + + +def exposed_sets( + instance: Instance, + lv_profile: ApprovalProfile, + tv_ballots: dict[str, set[Project]], + setup: str, + k: int, + seed: int | None = None, +) -> dict[str, set[Project]]: + """ + The exposed set E_v of every Target Voter under one of the five Section 3.1 + setups, keyed by voter id. Helper that hides the setups' differing + signatures behind one interface: the three offline samplers expose the + *same* k projects to everyone, ``random`` draws independently per voter, + and ``online_adaptive_controversial`` queries the voters sequentially, + folding each answer into the controversiality tally that picks the later + voters' questions. + + Examples + -------- + Offline popularity exposes the single most popular LV project (p1) to every + Target Voter. + + >>> p1, p2, p3 = Project("p1", 1), Project("p2", 1), Project("p3", 1) + >>> inst = Instance([p1, p2, p3], budget_limit=3) + >>> lv = ApprovalProfile([ApprovalBallot([p1, p2]), ApprovalBallot([p1])]) + >>> exposed_sets(inst, lv, {"v3": {p1}}, "offline_popularity", k=1) + {'v3': {p1}} + + Invalid inputs raise a ValueError naming the problem: + + >>> exposed_sets(inst, lv, {"v3": {p1}}, "by_magic", k=1) + Traceback (most recent call last): + ... + ValueError: unknown setup 'by_magic'; expected one of: random, offline_popularity, offline_consensus, offline_controversiality, online_adaptive_controversial + >>> exposed_sets(inst, lv, {"v3": {p1}}, "random", k=99) + Traceback (most recent call last): + ... + ValueError: k=99 must be between 0 and the number of projects (3) + """ + if setup not in SETUPS: + raise ValueError( + f"unknown setup {setup!r}; expected one of: " + ", ".join(SETUPS) + ) + if not 0 <= k <= len(instance): + raise ValueError( + f"k={k} must be between 0 and the number of projects ({len(instance)})" + ) + logger.debug("exposed_sets: validated setup=%s, k=%d", setup, k) + if setup == "random": + # "possibly different E for each voter" (Section 2.4): one independent + # uniform draw per voter, derived from the single seed. + rng = random.Random(seed) + return {vid: random_setup(instance, k, rng.randrange(2**32)) for vid in tv_ballots} + if setup == "online_adaptive_controversial": + return online_adaptive_controversial(instance, lv_profile, tv_ballots, k) + offline = { + "offline_popularity": offline_popularity, + "offline_consensus": offline_consensus, + "offline_controversiality": offline_controversiality, + }[setup] + shared = offline(instance, lv_profile, k) # same set for every TV voter + return {vid: shared for vid in tv_ballots} + + +def plan_sampling( + n_voters: int, + n_projects: int, + sample_degree: float, + lv_degree: float, +) -> tuple[int, int]: + """ + Size a process from the two partiality knobs of Section 3.0.1, *before* any + ballot exists: how many voters must give a full ballot, and how many projects + each remaining voter is asked about. + + This is the arithmetic of Example 3.1 on its own. With n voters and m + projects, ``sample_degree`` (s) fixes the total number of collected votes at + s*n*m and ``lv_degree`` (l) is the share of those votes that come from full + ballots, so ``|LV|`` = round(s*l*n) and the remaining s*(1-l)*n*m votes are + divided equally among the s*n*m Target Voters. ``lv_degree == 1`` is the + paper's naive sampling baseline: the sampled voters answer in full and + nobody else is asked anything, so k = 0. + + Use it to plan a real election - "we have 5000 voters, 100 projects, and we + are willing to ask for 30% of the votes; how many questions each?" - + and :py:func:`split_lv_tv` to carve a simulated one out of a known profile. + + Parameters + ---------- + n_voters : int + The number n of voters in the electorate. + n_projects : int + The number m of projects on the ballot. + sample_degree : float + Fraction of all n*m votes that are collected, in [0, 1]. + lv_degree : float + Fraction of the collected votes coming from full ballots, in [0, 1]. + + Returns + ------- + tuple[int, int] + ``(number of Learning Voters, questions per Target Voter)``. + + Examples + -------- + Four voters and two projects. Collecting everything with ``lv_degree == 1`` + makes all four Learning Voters and leaves nobody to question; collecting + half the votes with half of them from full ballots gives one Learning Voter + and asks the other three about 0.5*0.5*4*2/3 ~ 1 project each. + + >>> plan_sampling(4, 2, sample_degree=1.0, lv_degree=1.0) + (4, 0) + >>> plan_sampling(4, 2, sample_degree=0.5, lv_degree=0.5) + (1, 1) + + A realistic district: 30% of the votes collected, a tenth of them as full + ballots, so 150 people fill in all 100 projects and the other 4850 are asked + about 28 each - instead of every one of the 5000 facing the whole ballot. + + >>> plan_sampling(5000, 100, sample_degree=0.3, lv_degree=0.1) + (150, 28) + + >>> plan_sampling(4, 2, sample_degree=1.5, lv_degree=0.5) + Traceback (most recent call last): + ... + ValueError: sample_degree=1.5 must be in [0, 1] + """ + for name, degree in (("sample_degree", sample_degree), ("lv_degree", lv_degree)): + if not 0 <= degree <= 1: + raise ValueError(f"{name}={degree} must be in [0, 1]") + n_lv = round(sample_degree * lv_degree * n_voters) + n_tv = n_voters - n_lv + if lv_degree == 1 or not n_tv: + logger.info( + "plan_sampling: sample=%.2f lv=%.2f -> all %d collected votes come " + "from %d full ballots, nobody else is asked anything (k=0)", + sample_degree, lv_degree, n_lv * n_projects, n_lv, + ) + return n_lv, 0 + votes_left = sample_degree * (1 - lv_degree) * n_voters * n_projects + k = round(votes_left / n_tv) + logger.info( + "plan_sampling: sample=%.2f lv=%.2f of %d voters x %d projects -> " + "%d full ballots, the other %d asked k=%d each " + "(%.0f of %d votes collected in total)", + sample_degree, lv_degree, n_voters, n_projects, n_lv, n_tv, k, + n_lv * n_projects + n_tv * k, n_voters * n_projects, + ) + return n_lv, k + + +def split_lv_tv( + instance: Instance, + profile: ApprovalProfile, + sample_degree: float, + lv_degree: float, + seed: int | None = None, +) -> tuple[ApprovalProfile, dict[str, set[Project]], int]: + """ + Step 1 of the pipeline (Sections 2.4 / 3.0.1): partition the n voters of + the *ideal* instance into Learning Voters (LV, full ballots) and Target + Voters (TV, queried and completed), and derive k, the number of projects + each TV voter is asked about. + + The two knobs are *vote* budgets (Example 3.1 of the paper). With n voters + and m projects, ``sample_degree`` (s) fixes the total number of collected + votes at s*n*m and ``lv_degree`` (l) is the share of those votes that come + from full ballots: + + * ``|LV|`` = round(s*l*n) voters, drawn at random, keep their full ballots; + * every *other* voter is a TV, and the remaining s*(1-l)*n*m votes are + divided equally among them: k = round(s*(1-l)*n*m / ``|TV|``). + + ``lv_degree == 1`` is the paper's naive "sampling" baseline (Section 6): + the sampled voters answer in full and the rest of the community does not + vote at all - no TV voters, k = 0. + + Parameters + ---------- + instance : :py:class:`~pabutools.election.instance.Instance` + The PB instance (m projects). + profile : :py:class:`~pabutools.election.profile.approvalprofile.ApprovalProfile` + The ideal instance's full ballots (all n voters). + sample_degree : float + Fraction of all n*m votes that are collected, in [0, 1]. + lv_degree : float + Fraction of the collected votes coming from full ballots, in [0, 1]. + seed : int, optional + Seed for the random partition, for reproducibility. + + Returns + ------- + tuple[:py:class:`~pabutools.election.profile.approvalprofile.ApprovalProfile`, dict[str, set[:py:class:`~pabutools.election.instance.Project`]], int] + The LV profile (full ballots), the TV voters' full ballots keyed by + voter id (their known ground truth, used to simulate the k answers), + and k. + + Examples + -------- + Four voters, two projects. Collecting everything with ``lv_degree == 1`` + makes everyone an LV and leaves no TV. Collecting half the votes with half + of them from full ballots gives one LV voter; the other three are all TV, + each asked k = 0.5*0.5*4*2/3 ~ 1 project. + + >>> p1, p2 = Project("p1", 1), Project("p2", 1) + >>> inst = Instance([p1, p2], budget_limit=2) + >>> prof = ApprovalProfile([ApprovalBallot([p1]), ApprovalBallot([p2]), + ... ApprovalBallot([p1, p2]), ApprovalBallot([])]) + >>> lv, tv, k = split_lv_tv(inst, prof, sample_degree=1.0, lv_degree=1.0) + >>> lv.num_ballots(), len(tv), k + (4, 0, 0) + >>> lv, tv, k = split_lv_tv(inst, prof, sample_degree=0.5, lv_degree=0.5, seed=0) + >>> lv.num_ballots(), len(tv), k + (1, 3, 1) + >>> split_lv_tv(inst, prof, sample_degree=1.5, lv_degree=0.5) + Traceback (most recent call last): + ... + ValueError: sample_degree=1.5 must be in [0, 1] + """ + voters = list(profile) + n, m = len(voters), len(instance) + n_lv, k = plan_sampling(n, m, sample_degree, lv_degree) + order = random.Random(seed).sample(range(n), n) # a random permutation + lv_profile = ApprovalProfile([voters[i] for i in order[:n_lv]]) + # lv_degree == 1 is the naive sampling baseline: nobody else votes at all. + tv_ballots = ( + {} if lv_degree == 1 + else {f"v{i}": set(voters[i]) for i in order[n_lv:]} + ) + logger.info( + "split_lv_tv: sample=%.2f lv=%.2f -> %d LV, %d TV asked k=%d each (of %d voters)", + sample_degree, lv_degree, lv_profile.num_ballots(), len(tv_ballots), k, n, + ) + return lv_profile, tv_ballots, k + + +def complete_ballots( + instance: Instance, + lv_profile: ApprovalProfile, + tv_ballots: dict[str, set[Project]], + k: int, + *, + setup: str, + predict, + seed: int | None = None, +) -> tuple[dict[str, ApprovalBallot], dict[str, set[Project]]]: + """ + Steps 2 and 3 of the pipeline: choose every Target Voter's exposed set under + ``setup``, then complete her ballot with the ``predict`` module. + + Returns the completed ballots *and* the exposed sets, both keyed by voter + id. The exposed sets come back because a caller needs them to tell which + votes were predicted rather than answered - Section 5.1 scores only the + hidden ones. Shared by :py:func:`run_pipeline`, which goes on to apply the + voting rule, and :py:func:`run_all_experiments`, which also scores the + ballots themselves. + + Examples + -------- + Three LV voters approve {p1, p2}; the single TV voter is shown p1 only, and + her hidden vote on p2 is filled in from theirs. + + >>> p1, p2, p3 = Project("p1", 3), Project("p2", 3), Project("p3", 4) + >>> inst = Instance([p1, p2, p3], budget_limit=6) + >>> lv = ApprovalProfile([ApprovalBallot([p1, p2])] * 3) + >>> done, shown = complete_ballots(inst, lv, {"v": {p1, p2}}, k=1, + ... setup="offline_popularity", + ... predict=predict_by_classification) + >>> shown["v"] == {p1}, done["v"] == {p1, p2} + (True, True) + """ + exposed = exposed_sets(instance, lv_profile, tv_ballots, setup, k, seed) + partial = { + vid: reveal_ballot(instance, tv_ballots[vid], exposed[vid]) + for vid in tv_ballots + } + hidden = sum(len(instance) - len(exposed[vid]) for vid in tv_ballots) + logger.info( + "complete_ballots: setup=%s exposed %d of %d votes across %d TV " + "voters, asking %s to predict the remaining %d", + setup, sum(len(e) for e in exposed.values()), + len(tv_ballots) * len(instance), len(tv_ballots), + getattr(predict, "__name__", predict), hidden, + ) + return predict(instance, lv_profile, partial), exposed + + +def run_pipeline( + instance: Instance, + lv_profile: ApprovalProfile, + tv_ballots: dict[str, set[Project]], + k: int, + *, + setup: str, + predict, + seed: int | None = None, +) -> BudgetAllocation: + """ + The complete Section 3 pipeline for one (setup, predictor) choice: expose k + projects to each TV voter with ``setup``, complete the hidden votes with the + ``predict`` module, then run greedy approval on the LV plus completed TV + ballots. ``setup`` and ``predict`` are required - the pipeline runs exactly + one cell of the design matrix, so the caller must name both explicitly (the + sweep over every cell is :py:func:`run_all_experiments`). + + Parameters + ---------- + instance : :py:class:`~pabutools.election.instance.Instance` + The PB instance. + lv_profile : :py:class:`~pabutools.election.profile.approvalprofile.ApprovalProfile` + The full ballots of the LV voters. + tv_ballots : dict[str, set[:py:class:`~pabutools.election.instance.Project`]] + The full ballot of every Target Voter (TV) - her approval set A_v in + the ideal instance - keyed by voter id. + k : int + The number of projects to expose per TV voter. + setup : str + The name of the Section 3.1 setup to use (one of :py:data:`SETUPS`). + predict : callable + The Section 2.1 prediction module (one of :py:data:`PREDICTORS`' + values): it takes the TV voters' partial ballots, keyed by voter + id, and returns their completed approval ballots. + seed : int, optional + Seed for the ``random`` setup, ignored by the others. + + Returns + ------- + :py:class:`~pabutools.rules.budgetallocation.BudgetAllocation` + The predicted winning bundle. + + Examples + -------- + The "perfect" case: three LV voters and one TV voter all support the two + cheap projects {p1, p2}; the predicted bundle coincides with the real one. + + >>> p1, p2, p3, p4 = (Project("p1", 3), Project("p2", 3), + ... Project("p3", 4), Project("p4", 4)) + >>> inst = Instance([p1, p2, p3, p4], budget_limit=6) + >>> lv = ApprovalProfile([ApprovalBallot([p1, p2])] * 3) + >>> sorted(run_pipeline(inst, lv, {"v4": {p1, p2}}, k=1, + ... setup="offline_popularity", + ... predict=predict_by_matrix_factorization), key=str) + [p1, p2] + """ + logger.info( + "run_pipeline: starting with setup=%s predict=%s k=%d on %d LV + %d TV ballots", + setup, predict.__name__, k, lv_profile.num_ballots(), len(tv_ballots), + ) + # Step 1 (the LV/TV split) is done by the caller / :py:func:`split_lv_tv`. + # Steps 2-3 - sampling + prediction. + completed, _ = complete_ballots( + instance, lv_profile, tv_ballots, k, + setup=setup, predict=predict, seed=seed, + ) + # Step 4 - combine with the known LV ballots. + combined = ApprovalProfile(list(lv_profile) + [completed[vid] for vid in tv_ballots]) + logger.info( + "run_pipeline: all %d TV ballots completed, running the voting rule", + len(tv_ballots), + ) + # Steps 5-6 - voting rule: greedy approval on the LV + completed TV ballots. + return greedy_approval(instance, combined) + + +def run_experiment( + instance: Instance, + profile: ApprovalProfile, + sample_degree: float, + lv_degree: float, + *, + setup: str, + predictor="classification", + seed: int | None = None, +) -> BudgetAllocation: + """ + One experiment end to end, straight from the two partiality knobs: split the + ideal ``profile`` into LV and TV with :py:func:`split_lv_tv` (which derives k + from the knobs) and run :py:func:`run_pipeline` on the result. A convenience + composition of the two, for when you want a single cell of the design matrix + without holding the split yourself. + + Like :py:func:`run_pipeline` this is a *simulation*: it needs the ideal + profile, because the Target Voters' answers are read off their true ballots. + To run a process whose answers you actually collected, use :py:func:`elect`. + + Parameters + ---------- + instance : :py:class:`~pabutools.election.instance.Instance` + The PB instance. + profile : :py:class:`~pabutools.election.profile.approvalprofile.ApprovalProfile` + The ideal instance's full ballots (all n voters). + sample_degree, lv_degree : float + The partiality knobs of Section 3.0.1, both in [0, 1]. + setup : str + The Section 3.1 setup to sample with (one of :py:data:`SETUPS`). + predictor : str or callable, optional + A name from :py:data:`PREDICTORS` or a prediction module itself. + Defaults to ``"classification"``, the paper's best performer. + seed : int, optional + Seed for the split and the sampling, for reproducibility. + + Returns + ------- + :py:class:`~pabutools.rules.budgetallocation.BudgetAllocation` + The predicted winning bundle. + + Examples + -------- + Four voters who all approve the two cheap projects; collecting half the votes + still reproduces the real bundle. + + >>> p1, p2, p3 = Project("p1", 3), Project("p2", 3), Project("p3", 4) + >>> inst = Instance([p1, p2, p3], budget_limit=6) + >>> prof = ApprovalProfile([ApprovalBallot([p1, p2])] * 4) + >>> sorted(run_experiment(inst, prof, 0.5, 0.5, setup="offline_popularity", + ... seed=0), key=str) + [p1, p2] + """ + logger.info( + "run_experiment: simulating sample_degree=%.2f lv_degree=%.2f with " + "setup=%s and the %s predictor", + sample_degree, lv_degree, setup, predictor, + ) + lv_profile, tv_ballots, k = split_lv_tv( + instance, profile, sample_degree, lv_degree, seed=seed + ) + return run_pipeline( + instance, lv_profile, tv_ballots, k, + setup=setup, predict=as_predictor(predictor), seed=seed, + ) + + +# --------------------------------------------------------------------------- +# Running a real process, as opposed to reproducing the paper's experiments. +# --------------------------------------------------------------------------- +#: A Section 2.1 prediction module: it completes the Target Voters' partial +#: ballots, keyed by voter id. Named in :py:data:`PREDICTORS`. +Predictor = Callable[ + [Instance, ApprovalProfile, dict[str, CardinalBallot]], + dict[str, ApprovalBallot], +] + + +def as_predictor(predictor: str | Predictor) -> Predictor: + """ + The prediction module named by ``predictor``, which may be a key of + :py:data:`PREDICTORS` or the function itself, so callers can say + ``"classification"`` instead of importing it. + + Examples + -------- + >>> as_predictor("classification") is predict_by_classification + True + >>> as_predictor(predict_by_classification) is predict_by_classification + True + >>> as_predictor("by_magic") + Traceback (most recent call last): + ... + ValueError: unknown predictor 'by_magic'; expected one of: classification, matrix_factorization, factorization_machines + """ + if callable(predictor): + return predictor + if predictor not in PREDICTORS: + raise ValueError( + f"unknown predictor {predictor!r}; expected one of: " + + ", ".join(PREDICTORS) + ) + return PREDICTORS[predictor] + + +def elect( + instance: Instance, + lv_profile: ApprovalProfile, + tv_ballots: dict[str, CardinalBallot], + predictor: str | Predictor = "classification", +) -> BudgetAllocation: + """ + Decide a real PB process from the ballots that were actually collected. + + This is the deployment counterpart of :py:func:`run_pipeline`. The pipeline + is a simulation: it takes the Target Voters' *true* ballots and reads their + answers off them, which you cannot do in a live process because those + answers are exactly what you do not have. ``elect`` instead takes the + answers themselves - each Target Voter's partial ballot, saying which of the + projects she was asked about she approved and which she rejected - completes + the gaps with a Section 2.1 prediction module, and applies greedy approval + to the LV ballots plus the completed TV ballots. + + A live process therefore runs in three steps: :py:func:`plan_sampling` to + size it, :py:func:`exposed_sets` to choose which projects to put in front of + each voter, and ``elect`` once the answers are in. + + .. note:: + Of the five setups, ``random`` and the three ``offline_*`` ones need + nothing but the voter ids to choose their questions, so they work in a + live process as they are. ``online_adaptive_controversial`` picks each + question from the answers gathered so far, so deploying it needs an + interactive loop rather than a single call. + + Parameters + ---------- + instance : :py:class:`~pabutools.election.instance.Instance` + The PB instance. + lv_profile : :py:class:`~pabutools.election.profile.approvalprofile.ApprovalProfile` + The full ballots that were submitted in full (the Learning Voters). + May be empty if every voter answered only part of the ballot. + tv_ballots : dict[str, :py:class:`~pabutools.election.ballot.cardinalballot.CardinalBallot`] + The partial ballots that came back, keyed by voter id: build each one + with :py:func:`partial_ballot` from the projects that voter approved + and rejected. Everything else is treated as unanswered and predicted. + predictor : str or callable, optional + A name from :py:data:`PREDICTORS` or a prediction module itself. + Defaults to ``"classification"``, the paper's best performer. + + Returns + ------- + :py:class:`~pabutools.rules.budgetallocation.BudgetAllocation` + The winning bundle, respecting the instance's budget limit. + + Examples + -------- + Three voters filled in the whole ballot and approved {p1, p2}. A fourth was + asked about only p1 and p3: she approved p1 and rejected p3, and was never + asked about p2. Her opinion on p2 is predicted from the others, and the + budget of 6 funds the two projects costing 3. + + >>> p1, p2, p3 = Project("p1", 3), Project("p2", 3), Project("p3", 4) + >>> inst = Instance([p1, p2, p3], budget_limit=6) + >>> lv = ApprovalProfile([ApprovalBallot([p1, p2])] * 3) + >>> answers = {"v4": partial_ballot(approved={p1}, disapproved={p3})} + >>> sorted(elect(inst, lv, answers), key=str) + [p1, p2] + + An election where nobody filled in a full ballot still works, as long as the + answers between them cover the projects. + + >>> answers = { + ... "a": partial_ballot(approved={p1}, disapproved={p3}), + ... "b": partial_ballot(approved={p2}, disapproved={p3}), + ... } + >>> sorted(elect(inst, ApprovalProfile([]), answers), key=str) + [p1, p2] + """ + completed = as_predictor(predictor)(instance, lv_profile, tv_ballots) + combined = ApprovalProfile( + list(lv_profile) + [completed[vid] for vid in tv_ballots] + ) + logger.info( + "elect: %d full ballots + %d partial ballots completed, running the rule", + lv_profile.num_ballots(), len(tv_ballots), + ) + return greedy_approval(instance, combined) + + +#: The partiality grid swept in the paper's experiments (Section 6). +SAMPLE_DEGREES = (0.1, 0.15, 0.3, 0.5, 0.7, 0.9) +LV_DEGREES = (0.1, 0.2, 0.3, 0.5, 0.7, 0.9, 1.0) + + +def run_all_experiments( + instance: Instance, + profile: ApprovalProfile, + *, + setups: Iterable[str] = SETUPS, + predictors: Iterable[str] = tuple(PREDICTORS), + sample_degrees: Iterable[float] = SAMPLE_DEGREES, + lv_degrees: Iterable[float] = LV_DEGREES, + n_repeat: int = 50, + seed: int | None = None, +) -> dict[tuple[float, float, str, str], dict[str, float]]: + """ + The paper's full treatment matrix (Section 6, Figure 4). For every cell - + setup x predictor x sample_degree x lv_degree - the ideal ``profile`` is + split into LV/TV with :py:func:`split_lv_tv` (which also derives k, the + number of questions per TV voter, from the two degrees - Section 3.0.1), + the pipeline is run with :py:func:`run_pipeline`, and the predicted bundle + is compared to the *real* bundle (greedy approval on the whole ideal + profile). Because the split is random, each cell is repeated ``n_repeat`` + times and the two bundle metrics - Fractional Allocation + (:py:func:`fractional_allocation_score`) and the Symmetric Distance + ``|rb △ pb|`` (Section 5.2.1, computed inline) - are averaged. + + .. warning:: + **The defaults reproduce the paper's grid and will not finish.** They + describe 6 sample degrees x 7 LV degrees x 5 setups x 3 predictors = 630 + cells, each repeated 50 times, i.e. 31 500 pipeline runs. Measured on a + real Warsaw district (Praga-Poludnie 2022, 10 424 voters, 96 projects), + one run takes 30-165 seconds depending on the predictor - so the full + default sweep is of the order of a year of compute. + + Scale it down deliberately. Shrinking ``sample_degrees``, + ``lv_degrees``, ``setups``, ``predictors`` and ``n_repeat`` all help + proportionally; passing a one-element ``param_grid`` through to + :py:func:`pabutools.recommendation.model_training.train_classification` removes the + Section 5 hyperparameter search, which costs roughly 10x a plain fit. + A few hundred voters, one sample degree and ``n_repeat=3`` runs in + minutes and is enough to see the trends. + + .. note:: + The paper repeats the sampling module 20 times and the prediction module + 50 times; ``n_repeat`` collapses both into one knob (default 50). + ``classification`` needs ``xgboost`` installed, matrix factorization + needs ``scikit-surprise``, and the hybrid needs ``lightfm``. + + Parameters + ---------- + instance : :py:class:`~pabutools.election.instance.Instance` + The PB instance. + profile : :py:class:`~pabutools.election.profile.approvalprofile.ApprovalProfile` + The ideal instance's full ballots (all voters). + setups : Iterable[str], optional + The setups to sweep (names from :py:data:`SETUPS`). + predictors : Iterable[str], optional + The predictors to sweep (names from :py:data:`PREDICTORS`). Defaults to + all three paper predictors (classification / MF / FM). + sample_degrees, lv_degrees : Iterable[float], optional + The partiality grid (Section 3.0.1). Default to the paper's ranges. + n_repeat : int, optional + Random splits averaged per cell (default 50). + seed : int, optional + Seed for the whole sweep, for reproducibility. + + Returns + ------- + dict[tuple[float, float, str, str], dict[str, float]] + Keyed by ``(sample_degree, lv_degree, setup, predictor)``. Each value + holds the two bundle metrics of Section 5.2 - ``"FA"`` and ``"SD"`` - + and the three classification metrics of Section 5.1 - + ``"precision"``, ``"recall"`` and ``"f1"``, averaged over the Target + Voters and scored only on the votes that were predicted rather than + answered. All five are means over ``n_repeat`` random splits. + + Examples + -------- + The "perfect" case, swept over a tiny grid. Every cell yields an FA in + [0, 1] and a non-negative SD. Only matrix factorization is swept here so + that the example runs wherever ``scikit-surprise`` is installed, without + also needing ``lightfm``. + + >>> p1, p2, p3, p4 = (Project("p1", 3), Project("p2", 3), + ... Project("p3", 4), Project("p4", 4)) + >>> inst = Instance([p1, p2, p3, p4], budget_limit=6) + >>> prof = ApprovalProfile([ApprovalBallot([p1, p2])] * 4) + >>> results = run_all_experiments( + ... inst, prof, predictors=("matrix_factorization",), + ... sample_degrees=(0.5,), lv_degrees=(0.5,), n_repeat=2, seed=0) + >>> len(results) == 5 * 1 # 5 setups x 1 predictor x 1 x 1 cells + True + >>> all(0.0 <= c["FA"] <= 1.0 and c["SD"] >= 0 for c in results.values()) + True + + Every cell carries Section 5.1's ballot metrics as well as Section 5.2's + bundle ones. + + >>> sorted(results[(0.5, 0.5, "random", "matrix_factorization")]) + ['FA', 'SD', 'f1', 'precision', 'recall'] + """ + setups, predictors = tuple(setups), tuple(predictors) + sample_degrees, lv_degrees = tuple(sample_degrees), tuple(lv_degrees) + logger.info( + "run_all_experiments: sweeping %d setups x %d predictors x " + "%d sample degrees x %d LV degrees = %d cells, %d repeats each", + len(setups), len(predictors), len(sample_degrees), len(lv_degrees), + len(setups) * len(predictors) * len(sample_degrees) * len(lv_degrees), + n_repeat, + ) + # The real bundle: greedy approval on the whole ideal profile (all voters). + real_bundle = set(greedy_approval(instance, profile)) + logger.info( + "run_all_experiments: real bundle has %d projects (the ground truth " + "every cell is scored against)", len(real_bundle), + ) + rng = random.Random(seed) + results: dict[tuple[float, float, str, str], dict[str, float]] = {} + for sample_degree in sample_degrees: + for lv_degree in lv_degrees: + for setup in setups: + for name in predictors: + predict = PREDICTORS[name] + totals = dict.fromkeys( + ("FA", "SD", "precision", "recall", "f1"), 0.0 + ) + for repeat in range(1, n_repeat + 1): + # The split derives k from the two degrees (Section 3.0.1). + lv_profile, tv_ballots, k = split_lv_tv( + instance, profile, sample_degree, lv_degree, + seed=rng.randrange(2**32), + ) + completed, exposed = complete_ballots( + instance, lv_profile, tv_ballots, k, + setup=setup, predict=predict, + seed=rng.randrange(2**32), + ) + combined = ApprovalProfile( + list(lv_profile) + + [completed[vid] for vid in tv_ballots] + ) + predicted = set(greedy_approval(instance, combined)) + fa = fractional_allocation_score( + real_bundle, predicted, instance.budget_limit + ) + # Symmetric Distance (Section 5.2.1): |rb △ pb|. + sd = len(real_bundle ^ predicted) + totals["FA"] += fa + totals["SD"] += sd + # Section 5.1, over the votes that were predicted rather + # than answered, averaged across the Target Voters (a + # voter with no hidden approvals scores 0, the usual + # convention for an undefined precision or recall). + for vid in tv_ballots: + scores = classification_metrics( + set(tv_ballots[vid]), set(completed[vid]), + set(instance) - exposed[vid], + ) + for metric, value in scores.items(): + totals[metric] += value / max(len(tv_ballots), 1) + logger.debug( + "run_all_experiments: repeat %d/%d of " + "(sample=%.2f, lv=%.2f, %s, %s): FA=%.3f SD=%d", + repeat, n_repeat, sample_degree, lv_degree, + setup, name, fa, sd, + ) + cell = {k2: v / n_repeat for k2, v in totals.items()} + results[(sample_degree, lv_degree, setup, name)] = cell + logger.info( + "run_all_experiments: sample=%.2f lv=%.2f setup=%s " + "predict=%s -> mean FA=%.3f mean SD=%.2f over %d repeats", + sample_degree, lv_degree, setup, name, + cell["FA"], cell["SD"], n_repeat, + ) + return results + + +# --------------------------------------------------------------------------- +# Section 5.1 - Classification accuracy metrics. +# --------------------------------------------------------------------------- +def classification_metrics( + real_approved: set[Project], + predicted_approved: set[Project], + hidden: set[Project], +) -> dict[str, float]: + """ + Section 5.1 (Classification Accuracy Metrics): precision, recall and F1 of a + prediction module, measured over one Target Voter's *hidden* projects (the + test set - the exposed votes are known, not predicted, so they are excluded). + Approval is the positive class, so from the confusion matrix over the hidden + projects: + + * TP = hidden projects the voter really approves and we predicted approve, + * FP = hidden projects we predicted approve but she really rejects, + * FN = hidden projects she really approves but we predicted reject, + + then ``precision = TP / (TP + FP)``, ``recall = TP / (TP + FN)`` and + ``F1 = 2 * precision * recall / (precision + recall)``. A denominator of 0 + (no predicted or no real approvals among the hidden projects) yields 0.0 for + that metric, the usual convention for an undefined score. + + Parameters + ---------- + real_approved : set[:py:class:`~pabutools.election.instance.Project`] + The projects the voter really approves (her ideal ballot A_v). + predicted_approved : set[:py:class:`~pabutools.election.instance.Project`] + The projects the prediction module marked as approved. + hidden : set[:py:class:`~pabutools.election.instance.Project`] + The voter's hidden set H_v, i.e. the projects scored (from + :py:func:`hidden_projects`). + + Returns + ------- + dict[str, float] + ``{"precision": ..., "recall": ..., "f1": ...}``, each in [0, 1]. + + Examples + -------- + Four hidden projects, the voter really approves {p1, p2}, the model predicts + {p1, p3}: one hit (p1), one false alarm (p3), one miss (p2), so precision = + recall = F1 = 0.5. A perfect prediction scores 1.0 across the board. + + >>> p1, p2, p3, p4 = (Project("p1", 1), Project("p2", 1), + ... Project("p3", 1), Project("p4", 1)) + >>> hidden = {p1, p2, p3, p4} + >>> classification_metrics({p1, p2}, {p1, p3}, hidden) + {'precision': 0.5, 'recall': 0.5, 'f1': 0.5} + >>> classification_metrics({p1, p2}, {p1, p2}, hidden) + {'precision': 1.0, 'recall': 1.0, 'f1': 1.0} + """ + # Restrict everything to the hidden projects: the exposed votes are known. + real = real_approved & hidden + predicted = predicted_approved & hidden + tp = len(real & predicted) + fp = len(predicted - real) + fn = len(real - predicted) + precision = tp / (tp + fp) if tp + fp else 0.0 + recall = tp / (tp + fn) if tp + fn else 0.0 + f1 = 2 * precision * recall / (precision + recall) if precision + recall else 0.0 + logger.info( + "classification_metrics: TP=%d FP=%d FN=%d -> P=%.3f R=%.3f F1=%.3f", + tp, fp, fn, precision, recall, f1, + ) + return {"precision": precision, "recall": recall, "f1": f1} + + +# --------------------------------------------------------------------------- +# Section 5.2 - Bundle evaluation metrics. +# --------------------------------------------------------------------------- +# The Symmetric Distance (Section 5.2.1), |rb △ pb|, is a one-liner +# ``len(real_bundle ^ predicted_bundle)`` computed inline where needed (see +# ``run_all_experiments``), so it gets no function of its own. (The paper's +# second toy example says SD({1,2,3}, {1,3,4}) = 1; by its own definition the +# symmetric difference is {2, 4}, i.e. 2 - its first and third examples do +# match |rb △ pb|.) +def fractional_allocation_score( + real_bundle: Iterable[Project], + predicted_bundle: Iterable[Project], + budget_limit: int, +) -> float: + """ + Definition 5.1 (Fractional Allocation score): the total cost of the + projects predicted correctly (those in both bundles) divided by the budget + limit, FA = lambda / B with lambda = sum of cost(p) over p in pb ∩ rb. + + Parameters + ---------- + real_bundle : Iterable[:py:class:`~pabutools.election.instance.Project`] + The bundle obtained from the real (full) ballots - any iterable of + projects, e.g. the :py:class:`~pabutools.rules.budgetallocation.BudgetAllocation` + returned by :py:func:`greedy_approval`, as is. + predicted_bundle : Iterable[:py:class:`~pabutools.election.instance.Project`] + The bundle obtained from the predicted ballots (same, any iterable). + budget_limit : int + The budget limit B of the instance. + + Returns + ------- + float + The fractional allocation score, in [0, 1]. + + Examples + -------- + pb = rb = {p1, p2}, costs 3 and 3, budget 6, so FA = 6/6 = 1.0. Disjoint + bundles give FA = 0/6 = 0.0. + + >>> p1, p2, p3 = Project("p1", 3), Project("p2", 3), Project("p3", 6) + >>> fractional_allocation_score({p1, p2}, {p1, p2}, budget_limit=6) + 1.0 + >>> fractional_allocation_score({p3}, {p1}, budget_limit=6) + 0.0 + """ + real_bundle, predicted_bundle = set(real_bundle), set(predicted_bundle) + # lambda = total cost of the correctly-predicted projects (pb ∩ rb). + correctly_predicted = real_bundle & predicted_bundle + score = total_cost(correctly_predicted) / budget_limit + logger.info( + "fractional_allocation_score: %d/%d projects correct, FA=%.3f", + len(correctly_predicted), len(real_bundle | predicted_bundle), score, + ) + return float(score) + + +if __name__ == "__main__": + import doctest + + doctest.testmod(verbose=True) diff --git a/pyproject.toml b/pyproject.toml index 47b5c625..d4942a77 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,6 +36,16 @@ dependencies = [ ] [project.optional-dependencies] +recommendation = [ + # predict_by_classification: XGBClassifier needs xgboost's sklearn API. + "xgboost", + "scikit-learn", + "pandas", + "scikit-surprise", + # lightfm-next is a drop-in, Python 3.12-compatible fork of LightFM. + "lightfm-next; python_version >= '3.12' and sys_platform != 'win32'", + "lightfm; python_version < '3.12'", +] dev = [ "coverage", "Sphinx", @@ -51,4 +61,4 @@ dev = [ [project.urls] "Homepage" = "https://github.com/comsoc-community/pabutools" -"Bug Tracker" = "https://github.com/comsoc-community/pabutools/issues" \ No newline at end of file +"Bug Tracker" = "https://github.com/comsoc-community/pabutools/issues" diff --git a/tests/test_recommendation.py b/tests/test_recommendation.py new file mode 100644 index 00000000..52f322f3 --- /dev/null +++ b/tests/test_recommendation.py @@ -0,0 +1,665 @@ +""" +Unit tests for `pabutools.recommendation`, the implementation of the algorithms in +"A Recommendation System for Participatory Budgeting", +by Gil Leibiker and Nimrod Talmon (2023), https://optlearnmas23.github.io/files/p17.pdf + +Run with: python -m unittest tests.test_recommendation -v + +Each test is the specification its function must satisfy. They come in three +kinds: + * small, hand-checked instances (the examples from the paper); + * edge cases (partial ballots and invalid setup/k values); + * large inputs, checked against an independent reference or via invariants. + +Programmer: Roei Yanku +Date: 2026-06-20. +""" + +import random +from collections import Counter +from importlib.util import find_spec +from unittest import SkipTest, TestCase, skipUnless + +from parameterized import parameterized + +from pabutools.election import ( + Instance, + Project, + ApprovalProfile, + ApprovalBallot, + total_cost, +) + +from pabutools.recommendation import ( + # Partial-ballot helpers + the +1/-1/0 convention constants. + partial_ballot, + reveal_ballot, + approved_projects, + disapproved_projects, + exposed_projects, + hidden_projects, + consensus_levels, + greedy_approval, + random_setup, + offline_popularity, + offline_consensus, + offline_controversiality, + online_adaptive_controversial, + next_adaptive_question, + exposed_sets, + predict_by_classification, + predict_by_matrix_factorization, + predict_by_factorization_machines, + run_pipeline, + run_experiment, + split_lv_tv, + plan_sampling, + elect, + classification_metrics, + fractional_allocation_score, +) +from pabutools.recommendation.model_training import ( + train_factorization_machines, + train_matrix_factorization, +) + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- +def make_projects(specs): + """specs: list of (name, cost) -> dict name -> Project.""" + return {name: Project(name, cost) for name, cost in specs} + + +# The three prediction modules are backed by the optional ``recommendation`` +# extra, which the library's own CI does not install, so a test that fits a +# model reports "skipped" there instead of failing. Everything that does not +# need a model - the samplers, the ballot algebra, the pipeline's validation, +# the metrics and the voting rule - runs unconditionally. +def _installed(module): + try: + return find_spec(module) is not None + except (ImportError, ValueError): + return False + + +PREDICTOR_LIBRARY = { + "predict_by_classification": "xgboost", + "predict_by_matrix_factorization": "surprise", + "predict_by_factorization_machines": "lightfm", +} + + +def requires(*modules): + """Skip the decorated test unless every named library is installed.""" + missing = [module for module in modules if not _installed(module)] + return skipUnless( + not missing, + f"needs {', '.join(missing)} (pip install pabutools[recommendation])", + ) + + +def require_predictor(predictor): + """Skip the running test when this predictor's library is absent. + + The per-predictor counterpart of :py:func:`requires`, for the tests + ``parameterized`` expands over :py:data:`LIBRARY_PREDICTORS`: which + library a case needs is only known once the case is running. + """ + module = PREDICTOR_LIBRARY[predictor.__name__] + if not _installed(module): + raise SkipTest( + f"needs {module} (pip install pabutools[recommendation])" + ) + + +def consensus_data(): + """4 LV voters: p1 approved 4/0, p2 split 2/2, p3 rejected 0/4.""" + p = make_projects([("p1", 1), ("p2", 1), ("p3", 1)]) + inst = Instance(p.values(), budget_limit=3) + lv = ApprovalProfile( + [ + ApprovalBallot([p["p1"], p["p2"]]), + ApprovalBallot([p["p1"]]), + ApprovalBallot([p["p1"], p["p2"]]), + ApprovalBallot([p["p1"]]), + ] + ) + return p, inst, lv + + +def random_instance(n_projects, n_voters, budget, seed): + """Build a random instance + approval profile for property testing.""" + rng = random.Random(seed) + projects = [Project(f"p{i}", rng.randint(1, 10)) for i in range(n_projects)] + inst = Instance(projects, budget_limit=budget) + ballots = [] + for _ in range(n_voters): + approved = [p for p in projects if rng.random() < 0.3] + ballots.append(ApprovalBallot(approved)) + return projects, inst, ApprovalProfile(ballots) + + +def padded_projects(m, cost): + """m unit-named projects p000, p001, ... so str-order == index-order.""" + width = len(str(m - 1)) + return [Project(f"p{i:0{width}d}", cost) for i in range(m)] + + +def manual_scores(projects, profile): + """Independent (Counter-based) reimplementation of the approval scores.""" + counts = Counter() + for ballot in profile: + for proj in ballot: + counts[proj] += 1 + return {proj: counts[proj] for proj in projects} + + +def reference_greedy_approval(projects, profile, budget): + """ + Independent reference implementation of greedy approval, used to cross-check + `greedy_approval` on random inputs (the "compare to another algorithm" + strategy). Projects are taken in decreasing score, ties broken by name, and + funded while they fit the budget. + """ + counts = manual_scores(projects, profile) + ordered = sorted(projects, key=lambda p: (-counts[p], str(p))) + chosen, spent = [], 0 + for p in ordered: + if spent + p.cost <= budget: + chosen.append(p) + spent += p.cost + return chosen + + +# --------------------------------------------------------------------------- +# consensus_levels +# (Approval scores, Definition 2.1, have no function of our own - the code uses +# pabutools' profile.approval_scores() directly, which the library itself tests.) +# --------------------------------------------------------------------------- +class TestConsensusLevels(TestCase): + def test_hand_checked_split(self): + p, inst, lv = consensus_data() + c = consensus_levels(inst, lv) + assert c[p["p1"]] == 4 + assert c[p["p2"]] == 0 + assert c[p["p3"]] == 4 + + def test_large_structured_even_split_is_zero(self): + # Exactly half approve and half reject each project -> consensus 0. + projects = padded_projects(20, cost=1) + inst = Instance(projects, budget_limit=10) + prof = ApprovalProfile( + [ApprovalBallot(projects)] * 50 + [ApprovalBallot([])] * 50 + ) + c = consensus_levels(inst, prof) + assert all(c[p] == 0 for p in projects) + + +# --------------------------------------------------------------------------- +# greedy_approval (voting rule) +# --------------------------------------------------------------------------- +class TestGreedyApproval(TestCase): + def test_skips_unaffordable_but_funds_affordable(self): + # A project that exceeds the budget is skipped, while a cheaper one that + # fits is funded. + p = make_projects([("cheap", 10), ("dear", 100)]) + inst = Instance(p.values(), budget_limit=10) + prof = ApprovalProfile([ApprovalBallot([p["cheap"], p["dear"]])]) + assert set(greedy_approval(inst, prof)) == {p["cheap"]} + + def test_matches_reference_random(self): + # Cross-check against the independent greedy reference, several seeds. + for seed in range(5): + projects, inst, prof = random_instance(30, 50, 40, seed=200 + seed) + got = set(greedy_approval(inst, prof)) + expected = set(reference_greedy_approval(projects, prof, inst.budget_limit)) + assert got == expected + + +# --------------------------------------------------------------------------- +# random_setup +# --------------------------------------------------------------------------- +class TestRandomSetup(TestCase): + def test_exposes_k_independently_per_voter(self): + # One voter's draw: exactly k real projects. + p = make_projects([("p1", 2), ("p2", 2), ("p3", 3), ("p4", 3)]) + inst = Instance(p.values(), budget_limit=6) + exposed = random_setup(inst, k=2, seed=0) + assert len(exposed) == 2 and exposed <= set(p.values()) + + # Section 2.4: "possibly different E for each voter" - one seed must + # not collapse the whole population onto a single shared random set, + # yet the same seed must reproduce the same draws. + projects = padded_projects(8, cost=1) + inst = Instance(projects, budget_limit=8) + lv = ApprovalProfile([ApprovalBallot(projects[:2])]) + tv = {f"v{i}": set() for i in range(12)} + exposed = exposed_sets(inst, lv, tv, "random", k=2, seed=123) + assert all(len(e) == 2 for e in exposed.values()) + assert len({frozenset(e) for e in exposed.values()}) > 1 + assert exposed == exposed_sets(inst, lv, tv, "random", k=2, seed=123) + +# --------------------------------------------------------------------------- +# offline_popularity / consensus / controversiality +# --------------------------------------------------------------------------- +class TestOfflineSamplers(TestCase): + def test_popularity_top_k(self): + p = make_projects([("p1", 1), ("p2", 1), ("p3", 1)]) + inst = Instance(p.values(), budget_limit=3) + lv = ApprovalProfile( + [ + ApprovalBallot([p["p1"], p["p3"]]), + ApprovalBallot([p["p1"], p["p2"]]), + ApprovalBallot([p["p1"]]), + ] + ) + assert offline_popularity(inst, lv, k=1) == {p["p1"]} + + def test_consensus_top_k(self): + p, inst, lv = consensus_data() + assert offline_consensus(inst, lv, k=1) == {p["p1"]} + + def test_controversiality_bottom_k(self): + p, inst, lv = consensus_data() + assert offline_controversiality(inst, lv, k=1) == {p["p2"]} + +# --------------------------------------------------------------------------- +# online_adaptive_controversial +# --------------------------------------------------------------------------- +class TestOnlineAdaptive(TestCase): + def test_adaptive_feedback_shifts_the_questions(self): + # One TV voter: p1, p2, p3 are tied as most controversial (consensus 0), + # ties broken by name, so the two questions are p1 then p2. + p = make_projects([("p1", 1), ("p2", 1), ("p3", 1), ("p4", 1)]) + inst = Instance(p.values(), budget_limit=4) + lv = ApprovalProfile( + [ + ApprovalBallot([p["p1"], p["p2"]]), + ApprovalBallot([p["p1"], p["p3"]]), + ApprovalBallot([p["p2"], p["p3"]]), + ApprovalBallot([]), + ] + ) + assert online_adaptive_controversial( + inst, lv, {"v5": {p["p1"], p["p2"]}}, k=2 + ) == {"v5": {p["p1"], p["p2"]}} + + # The feedback loop across voters. p1 and p2 start tied as most + # controversial (both split 1-1 among LV), so the first voter is + # always asked p1, and her answer - approval *or* disapproval - breaks + # the tie, steering the second voter to p2. + p = make_projects([("p1", 1), ("p2", 1), ("p3", 1)]) + inst = Instance(p.values(), budget_limit=3) + lv = ApprovalProfile([ApprovalBallot([p["p1"]]), ApprovalBallot([p["p2"]])]) + assert online_adaptive_controversial( + inst, lv, {"v3": {p["p1"]}, "v4": {p["p1"]}}, k=1 + ) == {"v3": {p["p1"]}, "v4": {p["p2"]}} + assert online_adaptive_controversial( + inst, lv, {"v3": set(), "v4": {p["p1"]}}, k=1 + ) == {"v3": {p["p1"]}, "v4": {p["p2"]}} + + # And across more than one step: after v3 approves p1 (2-1) and v4 + # approves p2 (2-1), both are equally off-split again for v5 and the + # name tie-break returns to p1. + assert online_adaptive_controversial( + inst, lv, {"v3": {p["p1"]}, "v4": {p["p2"]}, "v5": set()}, k=1 + ) == {"v3": {p["p1"]}, "v4": {p["p2"]}, "v5": {p["p1"]}} + + def test_step_function_reproduces_the_closed_loop(self): + # next_adaptive_question is the open-loop form of the same rule, for a + # live process where the answers arrive one at a time. Driving it with + # a voter's ballot must ask exactly what the closed loop asks. + rng = random.Random(0) + for _ in range(60): + n_projects, n_lv = rng.randint(3, 7), rng.randint(2, 6) + k = rng.randint(1, n_projects) + p = make_projects([(f"p{i}", 1) for i in range(n_projects)]) + projects = list(p.values()) + inst = Instance(projects, budget_limit=n_projects) + lv = ApprovalProfile([ + ApprovalBallot([q for q in projects if rng.random() < 0.5]) + for _ in range(n_lv) + ]) + truth = {q for q in projects if rng.random() < 0.5} + + closed = online_adaptive_controversial(inst, lv, {"v": truth}, k)["v"] + + answers = partial_ballot() + for _ in range(k): + asked = next_adaptive_question(inst, lv, answers) + approved = approved_projects(answers) + disapproved = disapproved_projects(answers) + if asked in truth: + approved = approved | {asked} + else: + disapproved = disapproved | {asked} + answers = partial_ballot( + approved=approved, disapproved=disapproved + ) + assert exposed_projects(answers) == closed + + def test_step_function_rejects_a_fully_answered_ballot(self): + p = make_projects([("p1", 1), ("p2", 1)]) + inst = Instance(p.values(), budget_limit=2) + lv = ApprovalProfile([ApprovalBallot([p["p1"]])]) + answered = partial_ballot(approved={p["p1"]}, disapproved={p["p2"]}) + with self.assertRaisesRegex(ValueError, "already been asked"): + next_adaptive_question(inst, lv, answered) + +# --------------------------------------------------------------------------- +# Partial (three-state) ballots, Section 2.3. Represented as a CardinalBallot +# under the convention: +1 approved, -1 disapproved, 0 (or absent) hidden. +# --------------------------------------------------------------------------- +class TestPartialBallot(TestCase): + def test_scores_follow_convention(self): + # The whole point of the encoding: +1 = approve, -1 = disapprove, + # 0 = unknown. Pin it down so the convention can't silently drift. + p = make_projects([("p1", 1), ("p2", 1), ("p3", 1)]) + b = partial_ballot( + approved={p["p1"]}, disapproved={p["p2"]}, hidden={p["p3"]} + ) + assert b[p["p1"]] == 1 + assert b[p["p2"]] == -1 + assert b[p["p3"]] == 0 + + def test_absent_project_is_hidden(self): + # A project never mentioned in the ballot is hidden, just like an + # explicit 0 - both belong to H_v. + p = make_projects([("p1", 1), ("p2", 1), ("p3", 1)]) + inst = Instance(p.values(), budget_limit=3) + b = partial_ballot(approved={p["p1"]}, disapproved={p["p2"]}) # p3 absent + assert hidden_projects(b, inst) == {p["p3"]} + + def test_reveal_example2_4(self): + # Example 2.4: full ballot {p1,p2}, exposed set {p1,p3}. + p = make_projects([("p1", 1), ("p2", 1), ("p3", 1), ("p4", 1)]) + inst = Instance(p.values(), budget_limit=4) + b = reveal_ballot(inst, {p["p1"], p["p2"]}, {p["p1"], p["p3"]}) + assert approved_projects(b) == {p["p1"]} + assert disapproved_projects(b) == {p["p3"]} + assert hidden_projects(b, inst) == {p["p2"], p["p4"]} + assert exposed_projects(b) == {p["p1"], p["p3"]} + + +# --------------------------------------------------------------------------- +# The three prediction modules of the paper: classification (XGBoost), MF, FM +# (Section 2.1). They share one contract - keep the exposed votes, predict the +# hidden ones - so the same behavioural invariants are checked for each. +# --------------------------------------------------------------------------- +LIBRARY_PREDICTORS = [ + predict_by_classification, + predict_by_matrix_factorization, + predict_by_factorization_machines, +] + + +class TestLibraryPredictors(TestCase): + @parameterized.expand([(p.__name__, p) for p in LIBRARY_PREDICTORS]) + def test_exposed_votes_are_respected(self, _name, predictor): + # The exposed approval p3 is kept and the exposed disapproval p1 stays + # rejected, regardless of what the model would otherwise predict. + require_predictor(predictor) + p = make_projects([("p1", 1), ("p2", 1), ("p3", 1)]) + inst = Instance(p.values(), budget_limit=3) + lv = ApprovalProfile([ApprovalBallot([p["p1"]])] * 3) + partial = partial_ballot( + approved={p["p3"]}, disapproved={p["p1"]}, hidden={p["p2"]} + ) + pred = predictor(inst, lv, {"v": partial})["v"] + assert p["p3"] in pred # exposed approval kept + assert p["p1"] not in pred # exposed disapproval rejected + + @requires("surprise", "lightfm") + def test_cf_pools_e_tv_and_fm_uses_the_cost_feature(self): + # Section 3.1: prediction is based on LV *and* E_TV, the exposed votes + # of all TV voters. The lone LV voter rejects p2, but five TV voters + # expose approvals of it; a sixth TV voter with nothing exposed must + # pick p2 up from their pooled answers. + p = make_projects([("p1", 1), ("p2", 1)]) + inst = Instance(p.values(), budget_limit=2) + lv = ApprovalProfile([ApprovalBallot([p["p1"]])]) + tv = { + f"b{i}": partial_ballot(approved={p["p2"]}, hidden={p["p1"]}) + for i in range(5) + } + tv["a"] = partial_ballot(hidden=set(p.values())) + completed = predict_by_matrix_factorization(inst, lv, tv) + assert p["p2"] in completed["a"] + + # The FM hybrid ingredient: the voter approves her exposed cheap + # projects and rejects the exposed expensive ones; the fitted cost + # slope carries that pattern to the hidden pair. Plain MF has no cost + # feature and can only reach the same answer through the e1/e2/e3 + # correlation in the LV ballots, so it separates the two far more + # weakly - which is exactly how FM and MF must differ. + p = make_projects( + [("c1", 1), ("c2", 1), ("c3", 1), ("e1", 10), ("e2", 10), ("e3", 10)] + ) + inst = Instance(p.values(), budget_limit=12) + cheap = {p["c1"], p["c2"], p["c3"]} + lv = ApprovalProfile([ApprovalBallot(p.values())] * 3 + + [ApprovalBallot(cheap)] * 2) + partial = partial_ballot( + approved={p["c1"], p["c2"]}, + disapproved={p["e1"], p["e2"]}, + hidden={p["c3"], p["e3"]}, + ) + fm = predict_by_factorization_machines(inst, lv, {"v": partial})["v"] + assert p["c3"] in fm and p["e3"] not in fm + # Compare the cheap/expensive score gaps rather than the completed + # ballots: whether MF's e3 lands either side of the 0.5 cut-off depends + # on its latent rank, but its gap is always the weaker of the two. + exposed = {"v": ({p["c1"], p["c2"]}, {p["e1"], p["e2"]})} + mf_scores = train_matrix_factorization(inst, lv, exposed)["v"] + fm_scores = train_factorization_machines(inst, lv, exposed)["v"] + gap = lambda s: s[p["c3"]] - s[p["e3"]] # noqa: E731 + assert gap(fm_scores) > gap(mf_scores) + + @requires("lightfm") + def test_fm_uses_the_pabulib_category_and_target_features(self): + # The hybrid ingredient of Section 2.1.2, on Table 2's *set-valued* + # project attributes rather than cost. Every project costs the same and + # every LV voter approves everything, so neither cost nor the approval + # scores can separate them: the only thing that can carry this voter's + # exposed pattern to the projects she was never asked about is the + # category / target metadata that ``parse_pabulib`` fills in. + p = make_projects([(name, 1) for name in + ("g1", "g2", "g3", "r1", "r2", "r3")]) + for name in ("g1", "g2", "g3"): + p[name].categories, p[name].targets = {"greenery"}, {"children"} + for name in ("r1", "r2", "r3"): + p[name].categories, p[name].targets = {"roads"}, {"drivers"} + inst = Instance(p.values(), budget_limit=6) + lv = ApprovalProfile([ApprovalBallot(p.values())] * 4) + + # She approves the greenery she was shown and rejects the roads. + exposed = {"v": ({p["g1"], p["g2"]}, {p["r1"], p["r2"]})} + scores = train_factorization_machines(inst, lv, exposed) + # So the unseen greenery project must outscore the unseen road one. + assert scores["v"][p["g3"]] > scores["v"][p["r3"]] + + +# --------------------------------------------------------------------------- +# run_pipeline (full pipeline) +# (split_lv_tv's Section 3.0.1 vote-budget arithmetic - TV = all non-LV +# voters, derived k, the lv_degree=1 baseline, degree validation - is pinned +# by its doctests in pabutools/recommendation/recommendation.py.) +# --------------------------------------------------------------------------- +class TestRunPipeline(TestCase): + def test_invalid_setup_or_k_raises_value_error(self): + p = make_projects([("p1", 1), ("p2", 1)]) + inst = Instance(p.values(), budget_limit=2) + lv = ApprovalProfile([ApprovalBallot([p["p1"]])] * 2) + with self.assertRaisesRegex(ValueError, "unknown setup 'by_magic'"): + run_pipeline(inst, lv, {"v1": {p["p1"]}}, k=1, + setup="by_magic", + predict=predict_by_matrix_factorization) + for bad_k in (-1, 3): # below 0 and above the number of projects + with self.assertRaisesRegex(ValueError, "must be between 0 and"): + run_pipeline(inst, lv, {"v1": {p["p1"]}}, k=bad_k, + setup="random", + predict=predict_by_matrix_factorization) + + @parameterized.expand([ + (f"{setup}_{predictor.__name__}", setup, predictor) + for setup in ( + "random", + "offline_popularity", + "offline_consensus", + "offline_controversiality", + "online_adaptive_controversial", + ) + for predictor in LIBRARY_PREDICTORS + ]) + def test_two_camp_electorate_recovers_real_bundle(self, _name, setup, predictor): + """ + The end-to-end criterion of the paper (Section 2.4): the pipeline must + estimate the *ideal* outcome. A structured two-camp electorate - a 60% + majority camp approving {p1, p2, p3} and a 40% minority camp approving + {p4, p5, p6} - fixes the real winning bundle at {p1, p2, p3}. A third + of the voters become TV with only k=2 of the 6 projects exposed, and + every setup x predictor combination must still reconstruct the exact + real bundle (FA = 1.0, SD = 0). + """ + require_predictor(predictor) + p = make_projects([(f"p{i}", 1) for i in range(1, 7)]) + camp_a = {p["p1"], p["p2"], p["p3"]} # 18 of 30 voters (60%) + camp_b = {p["p4"], p["p5"], p["p6"]} # 12 of 30 voters (40%) + inst = Instance(p.values(), budget_limit=3) + + # The real bundle, from all 30 full ballots: camp A's projects score + # 18 > 12, and the budget funds exactly three unit-cost projects. + full_profile = ApprovalProfile( + [ApprovalBallot(camp_a)] * 18 + [ApprovalBallot(camp_b)] * 12 + ) + real_bundle = set(greedy_approval(inst, full_profile)) + assert real_bundle == camp_a # sanity: the ground truth is as designed + + # LV/TV split preserving the 60/40 mix: 20 LV, 10 TV. + lv = ApprovalProfile([ApprovalBallot(camp_a)] * 12 + + [ApprovalBallot(camp_b)] * 8) + tv = {f"a{i}": set(camp_a) for i in range(6)} + tv.update({f"b{i}": set(camp_b) for i in range(4)}) + + predicted = set(run_pipeline(inst, lv, tv, k=2, + setup=setup, predict=predictor, seed=7)) + assert predicted == real_bundle + assert fractional_allocation_score( + real_bundle, predicted, inst.budget_limit) == 1.0 + assert len(real_bundle ^ predicted) == 0 # symmetric distance + + +# --------------------------------------------------------------------------- +# classification_metrics (Section 5.1) +# --------------------------------------------------------------------------- +class TestClassificationMetrics(TestCase): + def test_mixed_hit_miss_false_alarm(self): + p = make_projects([("p1", 1), ("p2", 1), ("p3", 1), ("p4", 1)]) + hidden = set(p.values()) + # really approves {p1, p2}, predicted {p1, p3}: hit p1, miss p2, false p3. + m = classification_metrics({p["p1"], p["p2"]}, {p["p1"], p["p3"]}, hidden) + assert m == {"precision": 0.5, "recall": 0.5, "f1": 0.5} + + def test_exposed_votes_excluded(self): + # A wrong prediction on an *exposed* project must not affect the metrics: + # only the hidden set is scored. Hidden = {p2}, predicted perfectly there. + p = make_projects([("p1", 1), ("p2", 1)]) + m = classification_metrics( + real_approved={p["p2"]}, # p1 exposed, p2 hidden+approved + predicted_approved={p["p1"], p["p2"]}, + hidden={p["p2"]}, + ) + assert m == {"precision": 1.0, "recall": 1.0, "f1": 1.0} + +# --------------------------------------------------------------------------- +# fractional_allocation_score +# --------------------------------------------------------------------------- +class TestFractionalAllocation(TestCase): + def test_partial_overlap(self): + p = make_projects([("p1", 2), ("p2", 3), ("p3", 5)]) + # overlap is {p2}, cost 3, budget 10 -> 0.3 + score = fractional_allocation_score( + {p["p1"], p["p2"]}, {p["p2"], p["p3"]}, budget_limit=10 + ) + self.assertAlmostEqual(float(score), 0.3) + + +# --------------------------------------------------------------------------- +# Running a real process: plan_sampling / elect / run_experiment +# --------------------------------------------------------------------------- +class TestRealProcess(TestCase): + def test_plan_sampling_matches_the_split_it_describes(self): + # plan_sampling is the arithmetic split_lv_tv performs, minus the + # ballots, so the two must agree on every point of the paper's grid. + p = make_projects([(f"p{i}", 1) for i in range(4)]) + inst = Instance(p.values(), budget_limit=4) + prof = ApprovalProfile([ApprovalBallot(set(p.values()))] * 10) + for sample_degree in (0.1, 0.3, 0.5, 0.9): + for lv_degree in (0.1, 0.5, 0.9, 1.0): + n_lv, k = plan_sampling(10, 4, sample_degree, lv_degree) + lv, tv, split_k = split_lv_tv( + inst, prof, sample_degree, lv_degree, seed=1 + ) + assert (lv.num_ballots(), split_k) == (n_lv, k) + assert len(tv) == (0 if lv_degree == 1 else 10 - n_lv) + + @requires("xgboost") + def test_elect_needs_no_ground_truth(self): + # The deployment path: nobody's full ballot is known to the algorithm - + # there are only the answers voters actually gave. Two camps disagree + # about p1 against p3, every voter was asked about just those two of + # the four projects, and the budget funds exactly one of them. + p = make_projects([("p1", 1), ("p2", 1), ("p3", 1), ("p4", 1)]) + inst = Instance(p.values(), budget_limit=1) + answers = {} + for i in range(12): # majority camp: approve p1/p2, reject p3/p4 + answers[f"a{i}"] = partial_ballot( + approved={p["p1"]}, disapproved={p["p3"]} + ) + for i in range(5): # minority camp: the other way round + answers[f"b{i}"] = partial_ballot( + approved={p["p3"]}, disapproved={p["p1"]} + ) + bundle = elect(inst, ApprovalProfile([]), answers) + assert total_cost(bundle) <= inst.budget_limit + assert p["p1"] in bundle and p["p3"] not in bundle + + @requires("xgboost") + def test_elect_keeps_every_answer_that_was_given(self): + # Whatever the model thinks, an answer a voter actually gave must + # survive into her counted ballot (Section 2.4: the completed instance + # has to agree with the partial one). + p = make_projects([("p1", 1), ("p2", 1), ("p3", 1)]) + inst = Instance(p.values(), budget_limit=3) + lv = ApprovalProfile([ApprovalBallot([p["p1"], p["p2"]])] * 4) + # She rejects p1 although every full ballot approves it. + answers = {"v": partial_ballot(approved={p["p3"]}, disapproved={p["p1"]})} + completed = predict_by_classification(inst, lv, answers)["v"] + assert p["p3"] in completed and p["p1"] not in completed + + @requires("xgboost") + def test_run_experiment_equals_split_then_pipeline(self): + # The convenience wrapper must be exactly its two parts, same seed. + p = make_projects([("p1", 2), ("p2", 2), ("p3", 3), ("p4", 3)]) + inst = Instance(p.values(), budget_limit=4) + prof = ApprovalProfile([ApprovalBallot([p["p1"], p["p2"]])] * 6) + lv, tv, k = split_lv_tv(inst, prof, 0.5, 0.3, seed=11) + expected = set(run_pipeline(inst, lv, tv, k, setup="offline_popularity", + predict=predict_by_classification, seed=11)) + got = set(run_experiment(inst, prof, 0.5, 0.3, + setup="offline_popularity", seed=11)) + assert got == expected + + def test_unknown_predictor_name_is_rejected(self): + p = make_projects([("p1", 1)]) + inst = Instance(p.values(), budget_limit=1) + with self.assertRaisesRegex(ValueError, "unknown predictor"): + elect(inst, ApprovalProfile([]), {}, predictor="by_magic") + + +if __name__ == "__main__": + import unittest + + unittest.main()