Report attainable coverage on tournament leaderboards

front_end/messages/en.json

@@ -540,7 +540,10 @@
   "average": "Average",
   "score": "Score",
   "coverage": "Coverage",
+  "coverageMax": "Coverage (max)",
   "totalCoverage": "Total Coverage",
+  "totalAttainableCoverage": "Total Attainable Coverage",
+  "effectiveCoverage": "Effective Coverage",
   "totalLiveCoverage": "Total Live Coverage",
   "predictedQuestions": "Predicted Questions",
   "totalScore": "Total Score",
@@ -905,6 +908,7 @@
   "deletedAuthor": "deleted author",
   "myScore": "My Score",
   "coverageInfo": "Your <link>Coverage</link> on that question. If question hasn't resolved yet, this is the amount of the question's lifetime you've covered so far. This is a live value and will change over time and may jump when the question resolves.",
+  "coverageMaxInfo": "Your <link>Coverage</link> on that question, followed in parentheses by the maximum coverage attainable on it. The maximum is less than 100% when a question closes early (for example when it resolves before its scheduled close time), so nobody could have covered its full scheduled lifetime.",
   "peerScoreInfo": "Your <link>Peer Score</link> on that question. If you see no score, then either the question has yet to resolve or you haven’t predicted it (or both!).",
   "spotPeerScoreInfo": "Your <link>Spot Peer Score</link> on that question. If you see no score, then either the question has yet to resolve or you haven’t predicted it (or both!).",
   "spotBaselineScoreInfo": "Your <link>Spot Baseline Score</link> on that question. If you see no score, then either the question has yet to resolve or you haven’t predicted it (or both!).",
@@ -920,6 +924,9 @@
   "totalRelativeScoreInfo": "The question-weighted sum of your <link>Relative Scores</link> on all questions in the tournament (but only those that close before the end of the tournament).",
   "predictedQuestionsInfo": "The number of resolved and open questions you predicted in this tournament.",
   "totalLiveCoverageInfo": "The total amount of live coverage you have in the tournament. This is the sum of your live coverages divided by the number of resolved and open questions. If all questions resolved right now, you would have this much coverage in the tournament.",
+  "totalCoverageInfo": "The average of your coverage across all resolved questions in the tournament (counting questions you didn't predict as 0% coverage).",
+  "totalAttainableCoverageInfo": "The average of the maximum attainable coverage across all resolved questions in the tournament. This is less than 100% when some questions closed early.",
+  "effectiveCoverageInfo": "Your total coverage divided by the total attainable coverage: the sum of your coverage over all resolved questions divided by the sum of the maximum attainable coverage, with both sums weighted by each question's weight. This matches the coverage shown on the tournament leaderboard.",
   "questionWeightInfo": "The weight of the question in the tournament. The score you earn from this question is multiplied by this weight.",
   "relativeTakeInfo": "Your Take is your coverage times e to the power of your total score. (c*e^s)",
   "backgroundInfo": "Background Info",

front_end/src/app/(main)/(leaderboards)/contributions/components/project_contributions.tsx

@@ -31,15 +31,44 @@ const ProjectContributions: FC<Props> = async ({ project, userId }) => {
       contribution.question_weight && contribution.question_weight !== 1.0
   );
 
-  const liveCoveragePercent =
-    (
-      (contributions.reduce(
+  const formatPercent = (value: number | null | undefined) =>
+    value == null ? "-" : `${(value * 100).toFixed(1)}%`;
+
+  // Questions for which the maximum attainable coverage is known (i.e. the
+  // question has resolved successfully). These are the questions that count
+  // towards the tournament's coverage.
+  const resolvedContributions = contributions.filter(
+    (contribution) => !isNil(contribution.attainable_coverage)
+  );
+
+  const totalCoverage = resolvedContributions.length
+    ? resolvedContributions.reduce(
         (acc, contribution) => acc + (contribution.coverage || 0),
         0
-      ) /
-        contributions.length) *
-      100
-    ).toFixed(1) + "%";
+      ) / resolvedContributions.length
+    : null;
+  const totalAttainableCoverage = resolvedContributions.length
+    ? resolvedContributions.reduce(
+        (acc, contribution) => acc + (contribution.attainable_coverage || 0),
+        0
+      ) / resolvedContributions.length
+    : null;
+  const coverageWeightedSum = resolvedContributions.reduce(
+    (acc, contribution) =>
+      acc + (contribution.coverage || 0) * (contribution.question_weight ?? 1),
+    0
+  );
+  const attainableWeightedSum = resolvedContributions.reduce(
+    (acc, contribution) =>
+      acc +
+      (contribution.attainable_coverage || 0) *
+        (contribution.question_weight ?? 1),
+    0
+  );
+  const effectiveCoverage = attainableWeightedSum
+    ? coverageWeightedSum / attainableWeightedSum
+    : null;
+
   const predictedQuestions = contributions.filter(
     (contribution) => contribution.coverage
   ).length;
@@ -55,7 +84,7 @@ const ProjectContributions: FC<Props> = async ({ project, userId }) => {
                   {t("Question")}
                 </th>
                 <th className="px-4 py-2.5 text-right text-sm font-bold">
-                  {t("coverage")}
+                  {t("coverageMax")}
                 </th>
                 <th className="px-4 py-2.5 text-right text-sm font-bold">
                   {t("score")}
@@ -82,9 +111,13 @@ const ProjectContributions: FC<Props> = async ({ project, userId }) => {
                     </Link>
                   </td>
                   <th className="px-4 py-2.5 text-right text-sm font-bold">
-                    {contribution.coverage
-                      ? `${(contribution.coverage * 100).toFixed(1)}%`
-                      : "-"}
+                    {!isNil(contribution.attainable_coverage)
+                      ? `${formatPercent(contribution.coverage || 0)} (${formatPercent(
+                          contribution.attainable_coverage
+                        )})`
+                      : contribution.coverage
+                        ? formatPercent(contribution.coverage)
+                        : "-"}
                   </th>
                   <td className="px-4 py-2.5 text-right text-sm font-bold text-orange-800 dark:text-orange-800-dark">
                     {contribution.score ? contribution.score.toFixed(3) : "-"}
@@ -111,10 +144,26 @@ const ProjectContributions: FC<Props> = async ({ project, userId }) => {
               </tr>
               <tr>
                 <th className="px-2 py-1 text-right text-sm">
-                  {t("totalLiveCoverage")}
+                  {t("totalCoverage")}
+                </th>
+                <td className="px-2 py-1 text-right text-sm font-bold text-orange-800 dark:text-orange-800-dark">
+                  {formatPercent(totalCoverage)}
+                </td>
+              </tr>
+              <tr>
+                <th className="px-2 py-1 text-right text-sm">
+                  {t("totalAttainableCoverage")}
                 </th>
                 <td className="px-2 py-1 text-right text-sm font-bold text-orange-800 dark:text-orange-800-dark">
-                  {liveCoveragePercent}
+                  {formatPercent(totalAttainableCoverage)}
+                </td>
+              </tr>
+              <tr>
+                <th className="px-2 py-1 text-right text-sm">
+                  {t("effectiveCoverage")}
+                </th>
+                <td className="px-2 py-1 text-right text-sm font-bold text-orange-800 dark:text-orange-800-dark">
+                  {formatPercent(effectiveCoverage)}
                 </td>
               </tr>
               <tr>
@@ -151,10 +200,10 @@ const ProjectContributions: FC<Props> = async ({ project, userId }) => {
               <dl className="m-0">
                 <div className="m-2 flex text-sm">
                   <dt className="mr-2 w-20 flex-none font-bold">
-                    {t("coverage")}
+                    {t("coverageMax")}
                   </dt>
                   <dd>
-                    {t.rich("coverageInfo", {
+                    {t.rich("coverageMaxInfo", {
                       link: (chunks) => (
                         <Link href={"/help/scores-faq/#coverage"}>
                           {chunks}
@@ -225,9 +274,21 @@ const ProjectContributions: FC<Props> = async ({ project, userId }) => {
                 </div>
                 <div className="m-2 flex text-sm">
                   <dt className="mr-2 w-20 flex-none font-bold">
-                    {t("totalLiveCoverage")}
+                    {t("totalCoverage")}
+                  </dt>
+                  <dd>{t("totalCoverageInfo")}</dd>
+                </div>
+                <div className="m-2 flex text-sm">
+                  <dt className="mr-2 w-20 flex-none font-bold">
+                    {t("totalAttainableCoverage")}
+                  </dt>
+                  <dd>{t("totalAttainableCoverageInfo")}</dd>
+                </div>
+                <div className="m-2 flex text-sm">
+                  <dt className="mr-2 w-20 flex-none font-bold">
+                    {t("effectiveCoverage")}
                   </dt>
-                  <dd>{t("totalLiveCoverageInfo")}</dd>
+                  <dd>{t("effectiveCoverageInfo")}</dd>
                 </div>
                 <div className="m-2 flex text-sm">
                   <dt className="mr-2 w-20 flex-none font-bold">

front_end/src/types/scoring.ts

@@ -149,6 +149,7 @@ export type LeaderboardFilters = {
 export type Contribution = {
   score: number | null;
   coverage: number | null;
+  attainable_coverage?: number | null;
   question_type?: QuestionType;
   question_resolution?: Resolution | "string";
   question_title?: string;

questions/models.py

@@ -351,6 +351,29 @@ def get_post(self) -> "Post | None":
         if self.post_id:
             return self.post
 
+    def get_attainable_coverage(self) -> float:
+        """
+        The maximum coverage a forecaster could attain on this question, i.e. the
+        fraction of the scheduled forecasting window during which the question was
+        actually open for forecasting:
+
+            (effective_close_time - open_time) / (scheduled_close_time - open_time)
+
+        This is 1.0 for questions that stay open until (or past) their scheduled
+        close time, and less than 1.0 for questions that close early (e.g. because
+        they resolved before their scheduled close time).
+        """
+        if not self.open_time or not self.scheduled_close_time:
+            return 0.0
+        scheduled_duration = (
+            self.scheduled_close_time - self.open_time
+        ).total_seconds()
+        if scheduled_duration <= 0:
+            return 0.0
+        effective_close_time = self.actual_close_time or self.scheduled_close_time
+        effective_duration = (effective_close_time - self.open_time).total_seconds()
+        return max(0.0, min(1.0, effective_duration / scheduled_duration))
+
     @property
     def status(self) -> QuestionStatus:
         """

scoring/serializers.py

@@ -92,7 +92,12 @@ def get_prize_pool(self, obj: Leaderboard):
 
     def get_max_coverage(self, obj: Leaderboard):
         if self.context.get("include_max_coverage", False):
-            return sum(
+            # The maximum attainable coverage over all successfully resolved
+            # questions, weighted by question weight. Questions that close early
+            # (e.g. resolve before their scheduled close time) contribute less
+            # than their full weight, so the leaderboard reports coverage against
+            # what was actually attainable rather than the full question window.
+            questions = (
                 obj.get_questions()
                 .filter(resolution__isnull=False)
                 .exclude(
@@ -101,7 +106,15 @@ def get_max_coverage(self, obj: Leaderboard):
                         UnsuccessfulResolutionType.AMBIGUOUS,
                     ]
                 )
-                .values_list("question_weight", flat=True)
+                .only(
+                    "open_time",
+                    "scheduled_close_time",
+                    "actual_close_time",
+                    "question_weight",
+                )
+            )
+            return sum(
+                q.get_attainable_coverage() * q.question_weight for q in questions
             )
 
     def get_is_primary_leaderboard(self, obj: Leaderboard):
@@ -113,6 +126,7 @@ def get_is_primary_leaderboard(self, obj: Leaderboard):
 class ContributionSerializer(serializers.Serializer):
     score = serializers.FloatField()
     coverage = serializers.FloatField(required=False)
+    attainable_coverage = serializers.FloatField(required=False, allow_null=True)
     question_type = serializers.CharField(source="question.type", required=False)
     question_resolution = serializers.CharField(
         source="question.resolution", required=False

scoring/utils.py

@@ -908,6 +908,7 @@ def update_leaderboard_from_csv_data(
 class Contribution:
     score: float | None
     coverage: float | None = None
+    attainable_coverage: float | None = None
     question: Question | None = None
     post: Post | None = None
     comment: Comment | None = None
@@ -1043,6 +1044,17 @@ def get_contribution_question_writing(user: User, leaderboard: Leaderboard):
     return contributions
 
 
+def _get_attainable_coverage(question: Question) -> float | None:
+    """
+    The maximum coverage attainable on a question, but only for successfully
+    resolved questions (so it lines up with the leaderboard's max coverage).
+    Returns None for unresolved or unsuccessfully resolved questions.
+    """
+    if not question.resolution or question.resolution in UnsuccessfulResolutionType:
+        return None
+    return question.get_attainable_coverage()
+
+
 def get_contributions(
     user: User,
     leaderboard: Leaderboard,
@@ -1120,6 +1132,7 @@ def get_contributions(
         Contribution(
             score=s.score,
             coverage=s.coverage,
+            attainable_coverage=_get_attainable_coverage(s.question),
             question=s.question,
             post=s.question.get_post(),
         )
@@ -1162,6 +1175,7 @@ def get_contributions(
                 contribution = Contribution(
                     score=None,
                     coverage=coverage or None,
+                    attainable_coverage=_get_attainable_coverage(question),
                     question=question,
                     post=question.get_post(),
                 )

tests/unit/test_questions/test_models.py

@@ -54,3 +54,42 @@ def test_initialize_multiple_choice_question():
     assert (
         question.options_history and question.options_history[0][1] == question.options
     )
+
+
+@pytest.mark.parametrize(
+    "open_time,scheduled_close_time,actual_close_time,expected",
+    [
+        # Stays open until its scheduled close: fully attainable
+        [
+            datetime_aware(2025, 1, 1),
+            datetime_aware(2025, 2, 1),
+            None,
+            1.0,
+        ],
+        # Closes exactly at scheduled close: fully attainable
+        [
+            datetime_aware(2025, 1, 1),
+            datetime_aware(2025, 2, 1),
+            datetime_aware(2025, 2, 1),
+            1.0,
+        ],
+        # Closes halfway through the scheduled window
+        [
+            datetime_aware(2025, 1, 1),
+            datetime_aware(2025, 1, 3),
+            datetime_aware(2025, 1, 2),
+            0.5,
+        ],
+    ],
+)
+def test_get_attainable_coverage(
+    open_time, scheduled_close_time, actual_close_time, expected
+):
+    question = create_question(
+        question_type=Question.QuestionType.BINARY,
+        open_time=open_time,
+        scheduled_close_time=scheduled_close_time,
+        actual_close_time=actual_close_time,
+    )
+
+    assert question.get_attainable_coverage() == pytest.approx(expected)