Add average rank computation column

documentation/docs/Organizers/Benchmark_Creation/Leaderboard-Functionality.md

@@ -49,6 +49,7 @@ Computation options are:
   - avg
   - min
   - max
+  - avg_rank
 
 These are applied across the columns specified as `computation_indexes`.
 

documentation/docs/Organizers/Benchmark_Creation/Yaml-Structure.md

@@ -259,7 +259,7 @@ fact_sheet: {
   - Ascending: smaller scores are better
   - Descending: larger scores are better
 - **computation:** computation to be applied *must be accompanied by computation indexes*
-  - computation options: sum, avg, min, max
+  - computation options: sum, avg, min, max, avg_rank
 - **computation_indexes:** an array of indexes of the columns the computation should be applied to
 - **precision:** (*integer, default=2*) to round the score to *precision* number of digits
 - **hidden:** (*boolean, default=False*) to hide/unhide a column on leaderboard

src/apps/api/serializers/leaderboards.py

@@ -101,18 +101,24 @@ def get_submissions(self, instance):
         # desc == -colname
         # asc == colname
         primary_col = instance.columns.get(index=instance.primary_index)
-        # Order first by primary column. Then order by other columns after for tie breakers.
-        ordering = [f'{"-" if primary_col.sorting == "desc" else ""}primary_col']
-        submissions = (
+        base_qs = (
             Submission.objects.filter(
                 leaderboard=instance,
                 is_specific_task_re_run=False
             )
             .select_related('owner')
             .prefetch_related('scores')
-            .annotate(primary_col=Sum('scores__score', filter=Q(scores__column=primary_col)))
         )
+        # AVERAGE_RANK columns have no stored scores; skip DB-level sort and re-sort in the view.
+        if primary_col.computation == Column.AVERAGE_RANK:
+            ordering = ['created_when']
+            submissions = base_qs
+        else:
+            ordering = [f'{"-" if primary_col.sorting == "desc" else ""}primary_col']
+            submissions = base_qs.annotate(primary_col=Sum('scores__score', filter=Q(scores__column=primary_col)))
         for column in instance.columns.exclude(id=primary_col.id).order_by('index'):
+            if column.computation == Column.AVERAGE_RANK:
+                continue
             col_name = f'col{column.index}'
             ordering.append(f'{"-" if column.sorting == "desc" else ""}{col_name}')
             kwargs = {
@@ -157,8 +163,7 @@ def get_submissions(self, instance):
         # desc == -colname
         # asc == colname
         primary_col = instance.leaderboard.columns.get(index=instance.leaderboard.primary_index)
-        ordering = [f'{"-" if primary_col.sorting == "desc" else ""}primary_col']
-        submissions = (
+        base_qs = (
             Submission.objects.filter(
                 phase=instance,
                 is_soft_deleted=False,
@@ -168,14 +173,22 @@ def get_submissions(self, instance):
             )
             .select_related('owner')
             .prefetch_related('scores', 'scores__column')
-            .annotate(primary_col=Sum('scores__score', filter=Q(scores__column=primary_col)))
         )
+        # AVERAGE_RANK columns have no stored scores; skip DB-level sort and re-sort in the view.
+        if primary_col.computation == Column.AVERAGE_RANK:
+            ordering = ['created_when']
+            submissions = base_qs
+        else:
+            ordering = [f'{"-" if primary_col.sorting == "desc" else ""}primary_col']
+            submissions = base_qs.annotate(primary_col=Sum('scores__score', filter=Q(scores__column=primary_col)))
         for column in (
             instance.leaderboard.columns
             .filter(hidden=False)
             .exclude(id=primary_col.id)
             .order_by('index')
         ):
+            if column.computation == Column.AVERAGE_RANK:
+                continue
             col_name = f'col{column.index}'
             ordering.append(f'{"-" if column.sorting == "desc" else ""}{col_name}')
             kwargs = {

src/apps/api/views/competitions.py

@@ -31,7 +31,8 @@
 from datasets.models import Data
 from competitions.tasks import batch_send_email, manual_migration, create_competition_dump
 from competitions.utils import get_popular_competitions, get_recent_competitions
-from leaderboards.models import Leaderboard
+from leaderboards.models import Leaderboard, Column
+from leaderboards.ranking import inject_average_ranks
 from utils.data import make_url_sassy
 from api.permissions import IsOrganizerOrCollaborator
 from django.db import transaction
@@ -861,6 +862,12 @@ def get_leaderboard(self, request, pk):
         for k, v in submissions_keys.items():
             response['submissions'][v]['detailed_results'] = submission_detailed_results[k]
 
+        # Compute average rank for any AVERAGE_RANK columns and inject into response.
+        col_by_index = {col['index']: col for col in columns}
+        avg_rank_cols = [col for col in columns if col.get('computation') == Column.AVERAGE_RANK]
+        if avg_rank_cols:
+            inject_average_ranks(response['submissions'], avg_rank_cols, col_by_index, response['primary_index'])
+
         # --- pagination addition ---
         total_count = len(response['submissions'])
         paginator = DynamicChoicePagination()

src/apps/competitions/models.py

@@ -12,7 +12,7 @@
 from decimal import Decimal
 
 from celery_config import app, app_for_vhost
-from leaderboards.models import SubmissionScore
+from leaderboards.models import SubmissionScore, Column
 from profiles.models import User, Organization
 from utils.data import PathWrapper
 from utils.storage import BundleStorage
@@ -694,7 +694,7 @@ def check_child_submission_statuses(self):
     def calculate_scores(self):
         # leaderboards = self.phase.competition.leaderboards.all()
         # for leaderboard in leaderboards:
-        columns = self.phase.leaderboard.columns.exclude(computation__isnull=True)
+        columns = self.phase.leaderboard.columns.exclude(computation__isnull=True).exclude(computation=Column.AVERAGE_RANK)
         for column in columns:
             scores = self.scores.filter(column__index__in=column.computation_indexes.split(',')).values_list('score',
                                                                                                              flat=True)

src/apps/leaderboards/migrations/0010_alter_column_computation.py

@@ -0,0 +1,18 @@
+# Generated by Django 5.2.12 on 2026-04-23 09:35
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('leaderboards', '0009_alter_column_id_alter_leaderboard_id_and_more'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='column',
+            name='computation',
+            field=models.TextField(blank=True, choices=[('avg', 'Average'), ('sum', 'Sum'), ('min', 'Min'), ('max', 'Max'), ('avg_rank', 'Average Rank')], null=True),
+        ),
+    ]

src/apps/leaderboards/models.py

@@ -36,11 +36,13 @@ class Column(models.Model):
     SUM = 'sum'
     MIN = 'min'
     MAX = 'max'
+    AVERAGE_RANK = 'avg_rank'
     COMPUTATION_CHOICES = (
         (AVERAGE, 'Average'),
         (SUM, 'Sum'),
         (MIN, 'Min'),
         (MAX, 'Max'),
+        (AVERAGE_RANK, 'Average Rank'),
     )
     SORTING = (
         ('desc', 'Descending'),

src/apps/leaderboards/ranking.py

@@ -0,0 +1,108 @@
+from leaderboards.models import Column
+
+
+def fractional_rank(values):
+    """
+    Fractional (average) ranking: tied values receive the mean of the ranks they
+    would occupy, identical to scipy.stats.rankdata(method='average').
+    Rank 1 is assigned to the smallest value.
+    """
+    sorted_vals = sorted(values)
+    rank_sum = {}
+    rank_count = {}
+    for rank, val in enumerate(sorted_vals, start=1):
+        rank_sum[val] = rank_sum.get(val, 0) + rank
+        rank_count[val] = rank_count.get(val, 0) + 1
+    return [rank_sum[v] / rank_count[v] for v in values]
+
+
+def inject_average_ranks(submissions, avg_rank_cols, col_by_index, primary_index):
+    """
+    For each AVERAGE_RANK column, rank submissions on each referenced sub-column
+    using fractional (average) ranking, compute the mean rank per submission, and
+    append it as a synthetic score entry.
+    If the primary column is AVERAGE_RANK, re-sort the list in-place afterward.
+
+    Fractional ranking: tied submissions share the mean of the ranks they occupy
+    (e.g. two entries tying for positions 2 and 3 both receive rank 2.5).
+    Submissions missing a score for a sub-column are placed last (rank = n).
+    When a submission has multiple scores for the same column (multi-task), they are
+    summed before ranking, consistent with the ORM annotation in the serializer.
+    """
+    # Pre-aggregate scores per submission per column (sum across tasks).
+    submission_col_scores = []
+    for sub in submissions:
+        col_scores = {}
+        for s in sub['scores']:
+            idx = s['index']
+            try:
+                val = float(s['score'])
+            except (ValueError, TypeError):
+                val = None
+            if idx not in col_scores:
+                col_scores[idx] = val
+            elif val is not None:
+                col_scores[idx] = (col_scores[idx] or 0) + val
+        submission_col_scores.append(col_scores)
+
+    n = len(submissions)
+
+    for col in avg_rank_cols:
+        if not col.get('computation_indexes'):
+            continue
+        sub_indices = [int(i) for i in col['computation_indexes']]
+
+        per_column_ranks = []
+        for sub_idx in sub_indices:
+            sub_col = col_by_index.get(sub_idx)
+            if sub_col is None:
+                continue
+
+            valid_indices = [i for i in range(n) if submission_col_scores[i].get(sub_idx) is not None]
+            valid_scores = [submission_col_scores[i][sub_idx] for i in valid_indices]
+
+            if not valid_scores:
+                continue
+
+            # Negate descending columns so rank 1 = highest score.
+            scores_for_rank = [-s for s in valid_scores] if sub_col['sorting'] == 'desc' else valid_scores
+            fractions = fractional_rank(scores_for_rank)
+
+            ranks = {i: float(n) for i in range(n)}  # default: worst rank for unscored
+            for pos, sub_i in enumerate(valid_indices):
+                ranks[sub_i] = fractions[pos]
+            per_column_ranks.append(ranks)
+
+        if not per_column_ranks:
+            continue
+
+        is_primary = col['index'] == primary_index
+        for i, sub in enumerate(submissions):
+            sub_ranks = [r[i] for r in per_column_ranks]
+            avg_rank = sum(sub_ranks) / len(sub_ranks)
+            score_entry = {
+                'index': col['index'],
+                'column_key': col['key'],
+                'score': str(round(avg_rank, col.get('precision', 2))),
+                'is_primary': is_primary,
+            }
+            # The frontend matches scores by (task_id, column_key). Average rank is
+            # cross-task, so inject one copy per task that already has scores here.
+            task_ids = {s['task_id'] for s in sub['scores'] if s.get('task_id') is not None}
+            for task_id in task_ids:
+                sub['scores'].append({**score_entry, 'task_id': task_id})
+
+    primary_col = col_by_index.get(primary_index)
+    if primary_col and primary_col.get('computation') == Column.AVERAGE_RANK:
+        reverse = primary_col['sorting'] == 'desc'
+
+        def _sort_key(sub):
+            for s in sub['scores']:
+                if s['index'] == primary_index:
+                    try:
+                        return float(s['score'])
+                    except (ValueError, TypeError):
+                        pass
+            return float('inf') if not reverse else float('-inf')
+
+        submissions.sort(key=_sort_key, reverse=reverse)