From 48e4fcc2feb3a7d0ffffb60d3754912fcb87f98c Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 7 Aug 2026 16:13:13 +0000 Subject: [PATCH 1/2] Report attainable coverage on tournament leaderboards The leaderboard coverage column now divides total coverage by the total *attainable* coverage instead of by the raw question count. Questions that close early (e.g. resolve before their scheduled close time) have a maximum attainable coverage below 100%, so coverage is now measured against what was actually attainable. - Add Question.get_attainable_coverage() = (effective_close_time - open_time) / (scheduled_close_time - open_time). - LeaderboardSerializer.get_max_coverage now sums attainable coverage weighted by question weight over successfully resolved questions. - Expose attainable_coverage per contribution. - "My Score" section: the Coverage column becomes "Coverage (max)" showing your coverage and the max attainable in parentheses, and the totals now show total coverage, total attainable coverage, and effective coverage (which matches the leaderboard value). Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01NmzezEyuu7d2sX214hZWqZ --- front_end/messages/en.json | 7 ++ .../components/project_contributions.tsx | 95 +++++++++++++++---- front_end/src/types/scoring.ts | 1 + questions/models.py | 23 +++++ scoring/serializers.py | 18 +++- scoring/utils.py | 14 +++ tests/unit/test_questions/test_models.py | 39 ++++++++ 7 files changed, 178 insertions(+), 19 deletions(-) diff --git a/front_end/messages/en.json b/front_end/messages/en.json index 3b4b48b1fd..24470a938f 100644 --- a/front_end/messages/en.json +++ b/front_end/messages/en.json @@ -540,7 +540,10 @@ "average": "Average", "score": "Score", "coverage": "Coverage", + "coverageMax": "Coverage (max)", "totalCoverage": "Total Coverage", + "totalAttainableCoverage": "Total Attainable Coverage", + "effectiveCoverage": "Effective Coverage", "totalLiveCoverage": "Total Live Coverage", "predictedQuestions": "Predicted Questions", "totalScore": "Total Score", @@ -905,6 +908,7 @@ "deletedAuthor": "deleted author", "myScore": "My Score", "coverageInfo": "Your Coverage on that question. If question hasn't resolved yet, this is the amount of the question's lifetime you've covered so far. This is a live value and will change over time and may jump when the question resolves.", + "coverageMaxInfo": "Your Coverage on that question, followed in parentheses by the maximum coverage attainable on it. The maximum is less than 100% when a question closes early (for example when it resolves before its scheduled close time), so nobody could have covered its full scheduled lifetime.", "peerScoreInfo": "Your Peer Score on that question. If you see no score, then either the question has yet to resolve or you haven’t predicted it (or both!).", "spotPeerScoreInfo": "Your Spot Peer Score on that question. If you see no score, then either the question has yet to resolve or you haven’t predicted it (or both!).", "spotBaselineScoreInfo": "Your Spot Baseline Score on that question. If you see no score, then either the question has yet to resolve or you haven’t predicted it (or both!).", @@ -920,6 +924,9 @@ "totalRelativeScoreInfo": "The question-weighted sum of your Relative Scores on all questions in the tournament (but only those that close before the end of the tournament).", "predictedQuestionsInfo": "The number of resolved and open questions you predicted in this tournament.", "totalLiveCoverageInfo": "The total amount of live coverage you have in the tournament. This is the sum of your live coverages divided by the number of resolved and open questions. If all questions resolved right now, you would have this much coverage in the tournament.", + "totalCoverageInfo": "The average of your coverage across all resolved questions in the tournament (counting questions you didn't predict as 0% coverage).", + "totalAttainableCoverageInfo": "The average of the maximum attainable coverage across all resolved questions in the tournament. This is less than 100% when some questions closed early.", + "effectiveCoverageInfo": "Your total coverage divided by the total attainable coverage: the sum of your coverage over all resolved questions divided by the sum of the maximum attainable coverage. This matches the coverage shown on the tournament leaderboard.", "questionWeightInfo": "The weight of the question in the tournament. The score you earn from this question is multiplied by this weight.", "relativeTakeInfo": "Your Take is your coverage times e to the power of your total score. (c*e^s)", "backgroundInfo": "Background Info", diff --git a/front_end/src/app/(main)/(leaderboards)/contributions/components/project_contributions.tsx b/front_end/src/app/(main)/(leaderboards)/contributions/components/project_contributions.tsx index ad90c0d181..d02e07a777 100644 --- a/front_end/src/app/(main)/(leaderboards)/contributions/components/project_contributions.tsx +++ b/front_end/src/app/(main)/(leaderboards)/contributions/components/project_contributions.tsx @@ -31,15 +31,44 @@ const ProjectContributions: FC = async ({ project, userId }) => { contribution.question_weight && contribution.question_weight !== 1.0 ); - const liveCoveragePercent = - ( - (contributions.reduce( + const formatPercent = (value: number | null | undefined) => + value == null ? "-" : `${(value * 100).toFixed(1)}%`; + + // Questions for which the maximum attainable coverage is known (i.e. the + // question has resolved successfully). These are the questions that count + // towards the tournament's coverage. + const resolvedContributions = contributions.filter( + (contribution) => !isNil(contribution.attainable_coverage) + ); + + const totalCoverage = resolvedContributions.length + ? resolvedContributions.reduce( (acc, contribution) => acc + (contribution.coverage || 0), 0 - ) / - contributions.length) * - 100 - ).toFixed(1) + "%"; + ) / resolvedContributions.length + : null; + const totalAttainableCoverage = resolvedContributions.length + ? resolvedContributions.reduce( + (acc, contribution) => acc + (contribution.attainable_coverage || 0), + 0 + ) / resolvedContributions.length + : null; + const coverageWeightedSum = resolvedContributions.reduce( + (acc, contribution) => + acc + (contribution.coverage || 0) * (contribution.question_weight ?? 1), + 0 + ); + const attainableWeightedSum = resolvedContributions.reduce( + (acc, contribution) => + acc + + (contribution.attainable_coverage || 0) * + (contribution.question_weight ?? 1), + 0 + ); + const effectiveCoverage = attainableWeightedSum + ? coverageWeightedSum / attainableWeightedSum + : null; + const predictedQuestions = contributions.filter( (contribution) => contribution.coverage ).length; @@ -55,7 +84,7 @@ const ProjectContributions: FC = async ({ project, userId }) => { {t("Question")} - {t("coverage")} + {t("coverageMax")} {t("score")} @@ -82,9 +111,13 @@ const ProjectContributions: FC = async ({ project, userId }) => { - {contribution.coverage - ? `${(contribution.coverage * 100).toFixed(1)}%` - : "-"} + {!isNil(contribution.attainable_coverage) + ? `${formatPercent(contribution.coverage || 0)} (${formatPercent( + contribution.attainable_coverage + )})` + : contribution.coverage + ? formatPercent(contribution.coverage) + : "-"} {contribution.score ? contribution.score.toFixed(3) : "-"} @@ -111,10 +144,26 @@ const ProjectContributions: FC = async ({ project, userId }) => { - {t("totalLiveCoverage")} + {t("totalCoverage")} + + + {formatPercent(totalCoverage)} + + + + + {t("totalAttainableCoverage")} - {liveCoveragePercent} + {formatPercent(totalAttainableCoverage)} + + + + + {t("effectiveCoverage")} + + + {formatPercent(effectiveCoverage)} @@ -151,10 +200,10 @@ const ProjectContributions: FC = async ({ project, userId }) => {
- {t("coverage")} + {t("coverageMax")}
- {t.rich("coverageInfo", { + {t.rich("coverageMaxInfo", { link: (chunks) => ( {chunks} @@ -225,9 +274,21 @@ const ProjectContributions: FC = async ({ project, userId }) => {
- {t("totalLiveCoverage")} + {t("totalCoverage")} +
+
{t("totalCoverageInfo")}
+
+
+
+ {t("totalAttainableCoverage")} +
+
{t("totalAttainableCoverageInfo")}
+
+
+
+ {t("effectiveCoverage")}
-
{t("totalLiveCoverageInfo")}
+
{t("effectiveCoverageInfo")}
diff --git a/front_end/src/types/scoring.ts b/front_end/src/types/scoring.ts index e82269ff3e..913bb901bb 100644 --- a/front_end/src/types/scoring.ts +++ b/front_end/src/types/scoring.ts @@ -149,6 +149,7 @@ export type LeaderboardFilters = { export type Contribution = { score: number | null; coverage: number | null; + attainable_coverage?: number | null; question_type?: QuestionType; question_resolution?: Resolution | "string"; question_title?: string; diff --git a/questions/models.py b/questions/models.py index e167a8be94..64f663a7c5 100644 --- a/questions/models.py +++ b/questions/models.py @@ -351,6 +351,29 @@ def get_post(self) -> "Post | None": if self.post_id: return self.post + def get_attainable_coverage(self) -> float: + """ + The maximum coverage a forecaster could attain on this question, i.e. the + fraction of the scheduled forecasting window during which the question was + actually open for forecasting: + + (effective_close_time - open_time) / (scheduled_close_time - open_time) + + This is 1.0 for questions that stay open until (or past) their scheduled + close time, and less than 1.0 for questions that close early (e.g. because + they resolved before their scheduled close time). + """ + if not self.open_time or not self.scheduled_close_time: + return 0.0 + scheduled_duration = ( + self.scheduled_close_time - self.open_time + ).total_seconds() + if scheduled_duration <= 0: + return 0.0 + effective_close_time = self.actual_close_time or self.scheduled_close_time + effective_duration = (effective_close_time - self.open_time).total_seconds() + return max(0.0, min(1.0, effective_duration / scheduled_duration)) + @property def status(self) -> QuestionStatus: """ diff --git a/scoring/serializers.py b/scoring/serializers.py index 09731ea262..83da5ebc28 100644 --- a/scoring/serializers.py +++ b/scoring/serializers.py @@ -92,7 +92,12 @@ def get_prize_pool(self, obj: Leaderboard): def get_max_coverage(self, obj: Leaderboard): if self.context.get("include_max_coverage", False): - return sum( + # The maximum attainable coverage over all successfully resolved + # questions, weighted by question weight. Questions that close early + # (e.g. resolve before their scheduled close time) contribute less + # than their full weight, so the leaderboard reports coverage against + # what was actually attainable rather than the full question window. + questions = ( obj.get_questions() .filter(resolution__isnull=False) .exclude( @@ -101,7 +106,15 @@ def get_max_coverage(self, obj: Leaderboard): UnsuccessfulResolutionType.AMBIGUOUS, ] ) - .values_list("question_weight", flat=True) + .only( + "open_time", + "scheduled_close_time", + "actual_close_time", + "question_weight", + ) + ) + return sum( + q.get_attainable_coverage() * q.question_weight for q in questions ) def get_is_primary_leaderboard(self, obj: Leaderboard): @@ -113,6 +126,7 @@ def get_is_primary_leaderboard(self, obj: Leaderboard): class ContributionSerializer(serializers.Serializer): score = serializers.FloatField() coverage = serializers.FloatField(required=False) + attainable_coverage = serializers.FloatField(required=False, allow_null=True) question_type = serializers.CharField(source="question.type", required=False) question_resolution = serializers.CharField( source="question.resolution", required=False diff --git a/scoring/utils.py b/scoring/utils.py index 4e287fbd7c..67601318a5 100644 --- a/scoring/utils.py +++ b/scoring/utils.py @@ -908,6 +908,7 @@ def update_leaderboard_from_csv_data( class Contribution: score: float | None coverage: float | None = None + attainable_coverage: float | None = None question: Question | None = None post: Post | None = None comment: Comment | None = None @@ -1043,6 +1044,17 @@ def get_contribution_question_writing(user: User, leaderboard: Leaderboard): return contributions +def _get_attainable_coverage(question: Question) -> float | None: + """ + The maximum coverage attainable on a question, but only for successfully + resolved questions (so it lines up with the leaderboard's max coverage). + Returns None for unresolved or unsuccessfully resolved questions. + """ + if not question.resolution or question.resolution in UnsuccessfulResolutionType: + return None + return question.get_attainable_coverage() + + def get_contributions( user: User, leaderboard: Leaderboard, @@ -1120,6 +1132,7 @@ def get_contributions( Contribution( score=s.score, coverage=s.coverage, + attainable_coverage=_get_attainable_coverage(s.question), question=s.question, post=s.question.get_post(), ) @@ -1162,6 +1175,7 @@ def get_contributions( contribution = Contribution( score=None, coverage=coverage or None, + attainable_coverage=_get_attainable_coverage(question), question=question, post=question.get_post(), ) diff --git a/tests/unit/test_questions/test_models.py b/tests/unit/test_questions/test_models.py index 74c5e49b3f..6b347cb974 100644 --- a/tests/unit/test_questions/test_models.py +++ b/tests/unit/test_questions/test_models.py @@ -54,3 +54,42 @@ def test_initialize_multiple_choice_question(): assert ( question.options_history and question.options_history[0][1] == question.options ) + + +@pytest.mark.parametrize( + "open_time,scheduled_close_time,actual_close_time,expected", + [ + # Stays open until its scheduled close: fully attainable + [ + datetime_aware(2025, 1, 1), + datetime_aware(2025, 2, 1), + None, + 1.0, + ], + # Closes exactly at scheduled close: fully attainable + [ + datetime_aware(2025, 1, 1), + datetime_aware(2025, 2, 1), + datetime_aware(2025, 2, 1), + 1.0, + ], + # Closes halfway through the scheduled window + [ + datetime_aware(2025, 1, 1), + datetime_aware(2025, 1, 3), + datetime_aware(2025, 1, 2), + 0.5, + ], + ], +) +def test_get_attainable_coverage( + open_time, scheduled_close_time, actual_close_time, expected +): + question = create_question( + question_type=Question.QuestionType.BINARY, + open_time=open_time, + scheduled_close_time=scheduled_close_time, + actual_close_time=actual_close_time, + ) + + assert question.get_attainable_coverage() == pytest.approx(expected) From f3e89d49739ef74134b6089f33b4cd4ed6e5e60c Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 8 Aug 2026 09:31:45 +0000 Subject: [PATCH 2/2] Clarify that effective coverage sums are question-weighted The effectiveCoverageInfo help text described dividing unweighted sums, but both sums are weighted by question weight. The weighting is required for the value to match the tournament leaderboard, which computes coverage as sum(coverage * question_weight) / sum(attainable_coverage * question_weight). Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01NmzezEyuu7d2sX214hZWqZ --- front_end/messages/en.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/front_end/messages/en.json b/front_end/messages/en.json index 24470a938f..44734b7890 100644 --- a/front_end/messages/en.json +++ b/front_end/messages/en.json @@ -926,7 +926,7 @@ "totalLiveCoverageInfo": "The total amount of live coverage you have in the tournament. This is the sum of your live coverages divided by the number of resolved and open questions. If all questions resolved right now, you would have this much coverage in the tournament.", "totalCoverageInfo": "The average of your coverage across all resolved questions in the tournament (counting questions you didn't predict as 0% coverage).", "totalAttainableCoverageInfo": "The average of the maximum attainable coverage across all resolved questions in the tournament. This is less than 100% when some questions closed early.", - "effectiveCoverageInfo": "Your total coverage divided by the total attainable coverage: the sum of your coverage over all resolved questions divided by the sum of the maximum attainable coverage. This matches the coverage shown on the tournament leaderboard.", + "effectiveCoverageInfo": "Your total coverage divided by the total attainable coverage: the sum of your coverage over all resolved questions divided by the sum of the maximum attainable coverage, with both sums weighted by each question's weight. This matches the coverage shown on the tournament leaderboard.", "questionWeightInfo": "The weight of the question in the tournament. The score you earn from this question is multiplied by this weight.", "relativeTakeInfo": "Your Take is your coverage times e to the power of your total score. (c*e^s)", "backgroundInfo": "Background Info",