diff --git a/front_end/messages/en.json b/front_end/messages/en.json index 3b4b48b1fd..44734b7890 100644 --- a/front_end/messages/en.json +++ b/front_end/messages/en.json @@ -540,7 +540,10 @@ "average": "Average", "score": "Score", "coverage": "Coverage", + "coverageMax": "Coverage (max)", "totalCoverage": "Total Coverage", + "totalAttainableCoverage": "Total Attainable Coverage", + "effectiveCoverage": "Effective Coverage", "totalLiveCoverage": "Total Live Coverage", "predictedQuestions": "Predicted Questions", "totalScore": "Total Score", @@ -905,6 +908,7 @@ "deletedAuthor": "deleted author", "myScore": "My Score", "coverageInfo": "Your Coverage on that question. If question hasn't resolved yet, this is the amount of the question's lifetime you've covered so far. This is a live value and will change over time and may jump when the question resolves.", + "coverageMaxInfo": "Your Coverage on that question, followed in parentheses by the maximum coverage attainable on it. The maximum is less than 100% when a question closes early (for example when it resolves before its scheduled close time), so nobody could have covered its full scheduled lifetime.", "peerScoreInfo": "Your Peer Score on that question. If you see no score, then either the question has yet to resolve or you haven’t predicted it (or both!).", "spotPeerScoreInfo": "Your Spot Peer Score on that question. If you see no score, then either the question has yet to resolve or you haven’t predicted it (or both!).", "spotBaselineScoreInfo": "Your Spot Baseline Score on that question. If you see no score, then either the question has yet to resolve or you haven’t predicted it (or both!).", @@ -920,6 +924,9 @@ "totalRelativeScoreInfo": "The question-weighted sum of your Relative Scores on all questions in the tournament (but only those that close before the end of the tournament).", "predictedQuestionsInfo": "The number of resolved and open questions you predicted in this tournament.", "totalLiveCoverageInfo": "The total amount of live coverage you have in the tournament. This is the sum of your live coverages divided by the number of resolved and open questions. If all questions resolved right now, you would have this much coverage in the tournament.", + "totalCoverageInfo": "The average of your coverage across all resolved questions in the tournament (counting questions you didn't predict as 0% coverage).", + "totalAttainableCoverageInfo": "The average of the maximum attainable coverage across all resolved questions in the tournament. This is less than 100% when some questions closed early.", + "effectiveCoverageInfo": "Your total coverage divided by the total attainable coverage: the sum of your coverage over all resolved questions divided by the sum of the maximum attainable coverage, with both sums weighted by each question's weight. This matches the coverage shown on the tournament leaderboard.", "questionWeightInfo": "The weight of the question in the tournament. The score you earn from this question is multiplied by this weight.", "relativeTakeInfo": "Your Take is your coverage times e to the power of your total score. (c*e^s)", "backgroundInfo": "Background Info", diff --git a/front_end/src/app/(main)/(leaderboards)/contributions/components/project_contributions.tsx b/front_end/src/app/(main)/(leaderboards)/contributions/components/project_contributions.tsx index ad90c0d181..d02e07a777 100644 --- a/front_end/src/app/(main)/(leaderboards)/contributions/components/project_contributions.tsx +++ b/front_end/src/app/(main)/(leaderboards)/contributions/components/project_contributions.tsx @@ -31,15 +31,44 @@ const ProjectContributions: FC = async ({ project, userId }) => { contribution.question_weight && contribution.question_weight !== 1.0 ); - const liveCoveragePercent = - ( - (contributions.reduce( + const formatPercent = (value: number | null | undefined) => + value == null ? "-" : `${(value * 100).toFixed(1)}%`; + + // Questions for which the maximum attainable coverage is known (i.e. the + // question has resolved successfully). These are the questions that count + // towards the tournament's coverage. + const resolvedContributions = contributions.filter( + (contribution) => !isNil(contribution.attainable_coverage) + ); + + const totalCoverage = resolvedContributions.length + ? resolvedContributions.reduce( (acc, contribution) => acc + (contribution.coverage || 0), 0 - ) / - contributions.length) * - 100 - ).toFixed(1) + "%"; + ) / resolvedContributions.length + : null; + const totalAttainableCoverage = resolvedContributions.length + ? resolvedContributions.reduce( + (acc, contribution) => acc + (contribution.attainable_coverage || 0), + 0 + ) / resolvedContributions.length + : null; + const coverageWeightedSum = resolvedContributions.reduce( + (acc, contribution) => + acc + (contribution.coverage || 0) * (contribution.question_weight ?? 1), + 0 + ); + const attainableWeightedSum = resolvedContributions.reduce( + (acc, contribution) => + acc + + (contribution.attainable_coverage || 0) * + (contribution.question_weight ?? 1), + 0 + ); + const effectiveCoverage = attainableWeightedSum + ? coverageWeightedSum / attainableWeightedSum + : null; + const predictedQuestions = contributions.filter( (contribution) => contribution.coverage ).length; @@ -55,7 +84,7 @@ const ProjectContributions: FC = async ({ project, userId }) => { {t("Question")} - {t("coverage")} + {t("coverageMax")} {t("score")} @@ -82,9 +111,13 @@ const ProjectContributions: FC = async ({ project, userId }) => { - {contribution.coverage - ? `${(contribution.coverage * 100).toFixed(1)}%` - : "-"} + {!isNil(contribution.attainable_coverage) + ? `${formatPercent(contribution.coverage || 0)} (${formatPercent( + contribution.attainable_coverage + )})` + : contribution.coverage + ? formatPercent(contribution.coverage) + : "-"} {contribution.score ? contribution.score.toFixed(3) : "-"} @@ -111,10 +144,26 @@ const ProjectContributions: FC = async ({ project, userId }) => { - {t("totalLiveCoverage")} + {t("totalCoverage")} + + + {formatPercent(totalCoverage)} + + + + + {t("totalAttainableCoverage")} - {liveCoveragePercent} + {formatPercent(totalAttainableCoverage)} + + + + + {t("effectiveCoverage")} + + + {formatPercent(effectiveCoverage)} @@ -151,10 +200,10 @@ const ProjectContributions: FC = async ({ project, userId }) => {
- {t("coverage")} + {t("coverageMax")}
- {t.rich("coverageInfo", { + {t.rich("coverageMaxInfo", { link: (chunks) => ( {chunks} @@ -225,9 +274,21 @@ const ProjectContributions: FC = async ({ project, userId }) => {
- {t("totalLiveCoverage")} + {t("totalCoverage")} +
+
{t("totalCoverageInfo")}
+
+
+
+ {t("totalAttainableCoverage")} +
+
{t("totalAttainableCoverageInfo")}
+
+
+
+ {t("effectiveCoverage")}
-
{t("totalLiveCoverageInfo")}
+
{t("effectiveCoverageInfo")}
diff --git a/front_end/src/types/scoring.ts b/front_end/src/types/scoring.ts index e82269ff3e..913bb901bb 100644 --- a/front_end/src/types/scoring.ts +++ b/front_end/src/types/scoring.ts @@ -149,6 +149,7 @@ export type LeaderboardFilters = { export type Contribution = { score: number | null; coverage: number | null; + attainable_coverage?: number | null; question_type?: QuestionType; question_resolution?: Resolution | "string"; question_title?: string; diff --git a/questions/models.py b/questions/models.py index e167a8be94..64f663a7c5 100644 --- a/questions/models.py +++ b/questions/models.py @@ -351,6 +351,29 @@ def get_post(self) -> "Post | None": if self.post_id: return self.post + def get_attainable_coverage(self) -> float: + """ + The maximum coverage a forecaster could attain on this question, i.e. the + fraction of the scheduled forecasting window during which the question was + actually open for forecasting: + + (effective_close_time - open_time) / (scheduled_close_time - open_time) + + This is 1.0 for questions that stay open until (or past) their scheduled + close time, and less than 1.0 for questions that close early (e.g. because + they resolved before their scheduled close time). + """ + if not self.open_time or not self.scheduled_close_time: + return 0.0 + scheduled_duration = ( + self.scheduled_close_time - self.open_time + ).total_seconds() + if scheduled_duration <= 0: + return 0.0 + effective_close_time = self.actual_close_time or self.scheduled_close_time + effective_duration = (effective_close_time - self.open_time).total_seconds() + return max(0.0, min(1.0, effective_duration / scheduled_duration)) + @property def status(self) -> QuestionStatus: """ diff --git a/scoring/serializers.py b/scoring/serializers.py index 09731ea262..83da5ebc28 100644 --- a/scoring/serializers.py +++ b/scoring/serializers.py @@ -92,7 +92,12 @@ def get_prize_pool(self, obj: Leaderboard): def get_max_coverage(self, obj: Leaderboard): if self.context.get("include_max_coverage", False): - return sum( + # The maximum attainable coverage over all successfully resolved + # questions, weighted by question weight. Questions that close early + # (e.g. resolve before their scheduled close time) contribute less + # than their full weight, so the leaderboard reports coverage against + # what was actually attainable rather than the full question window. + questions = ( obj.get_questions() .filter(resolution__isnull=False) .exclude( @@ -101,7 +106,15 @@ def get_max_coverage(self, obj: Leaderboard): UnsuccessfulResolutionType.AMBIGUOUS, ] ) - .values_list("question_weight", flat=True) + .only( + "open_time", + "scheduled_close_time", + "actual_close_time", + "question_weight", + ) + ) + return sum( + q.get_attainable_coverage() * q.question_weight for q in questions ) def get_is_primary_leaderboard(self, obj: Leaderboard): @@ -113,6 +126,7 @@ def get_is_primary_leaderboard(self, obj: Leaderboard): class ContributionSerializer(serializers.Serializer): score = serializers.FloatField() coverage = serializers.FloatField(required=False) + attainable_coverage = serializers.FloatField(required=False, allow_null=True) question_type = serializers.CharField(source="question.type", required=False) question_resolution = serializers.CharField( source="question.resolution", required=False diff --git a/scoring/utils.py b/scoring/utils.py index 4e287fbd7c..67601318a5 100644 --- a/scoring/utils.py +++ b/scoring/utils.py @@ -908,6 +908,7 @@ def update_leaderboard_from_csv_data( class Contribution: score: float | None coverage: float | None = None + attainable_coverage: float | None = None question: Question | None = None post: Post | None = None comment: Comment | None = None @@ -1043,6 +1044,17 @@ def get_contribution_question_writing(user: User, leaderboard: Leaderboard): return contributions +def _get_attainable_coverage(question: Question) -> float | None: + """ + The maximum coverage attainable on a question, but only for successfully + resolved questions (so it lines up with the leaderboard's max coverage). + Returns None for unresolved or unsuccessfully resolved questions. + """ + if not question.resolution or question.resolution in UnsuccessfulResolutionType: + return None + return question.get_attainable_coverage() + + def get_contributions( user: User, leaderboard: Leaderboard, @@ -1120,6 +1132,7 @@ def get_contributions( Contribution( score=s.score, coverage=s.coverage, + attainable_coverage=_get_attainable_coverage(s.question), question=s.question, post=s.question.get_post(), ) @@ -1162,6 +1175,7 @@ def get_contributions( contribution = Contribution( score=None, coverage=coverage or None, + attainable_coverage=_get_attainable_coverage(question), question=question, post=question.get_post(), ) diff --git a/tests/unit/test_questions/test_models.py b/tests/unit/test_questions/test_models.py index 74c5e49b3f..6b347cb974 100644 --- a/tests/unit/test_questions/test_models.py +++ b/tests/unit/test_questions/test_models.py @@ -54,3 +54,42 @@ def test_initialize_multiple_choice_question(): assert ( question.options_history and question.options_history[0][1] == question.options ) + + +@pytest.mark.parametrize( + "open_time,scheduled_close_time,actual_close_time,expected", + [ + # Stays open until its scheduled close: fully attainable + [ + datetime_aware(2025, 1, 1), + datetime_aware(2025, 2, 1), + None, + 1.0, + ], + # Closes exactly at scheduled close: fully attainable + [ + datetime_aware(2025, 1, 1), + datetime_aware(2025, 2, 1), + datetime_aware(2025, 2, 1), + 1.0, + ], + # Closes halfway through the scheduled window + [ + datetime_aware(2025, 1, 1), + datetime_aware(2025, 1, 3), + datetime_aware(2025, 1, 2), + 0.5, + ], + ], +) +def test_get_attainable_coverage( + open_time, scheduled_close_time, actual_close_time, expected +): + question = create_question( + question_type=Question.QuestionType.BINARY, + open_time=open_time, + scheduled_close_time=scheduled_close_time, + actual_close_time=actual_close_time, + ) + + assert question.get_attainable_coverage() == pytest.approx(expected)