diff --git a/client/platform/web-girder/api/configuration.service.ts b/client/platform/web-girder/api/configuration.service.ts index 78cef19c5..bb227fbee 100644 --- a/client/platform/web-girder/api/configuration.service.ts +++ b/client/platform/web-girder/api/configuration.service.ts @@ -57,6 +57,31 @@ export type DateRange = '60 days' | '3 months' | '6 months' | '1 year' | '3 year export type GroupBy = 'user' | 'month' | undefined; +export interface AnnotationStatsLabelCount { + label: string; + count: number; +} + +export interface AnnotationStatsReport { + generatedAt: string; + datasets: { + total: number; + mostlyManual: number; + mixedSubstantialCorrections: number; + mostlyComputed: number; + empty: number; + }; + tracks: { + total: number; + confidenceGte1: number; + }; + labels: AnnotationStatsLabelCount[]; +} + +export interface AnnotationStatsJobResponse { + jobId: string; +} + export type AddOns = [string, string, string, boolean][]; export interface DiveConfiguration { @@ -151,6 +176,15 @@ function getStats(dateRange?: DateRange, overrideDateTime?: string, groupBy?: Gr }); } +function startAnnotationStatsJob() { + return girderRest.post('dive_configuration/annotation_stats'); +} + +/** Latest report saved under the admin user's Stats folder, or null. */ +function getLatestAnnotationStats() { + return girderRest.get('dive_configuration/annotation_stats'); +} + export { getBrandData, getConfig, @@ -164,4 +198,6 @@ export { postAddons, updateContainers, getStats, + startAnnotationStatsJob, + getLatestAnnotationStats, }; diff --git a/client/platform/web-girder/views/Admin/AdminStats.vue b/client/platform/web-girder/views/Admin/AdminStats.vue index 0c5f2e5ed..18e9ebdc3 100644 --- a/client/platform/web-girder/views/Admin/AdminStats.vue +++ b/client/platform/web-girder/views/Admin/AdminStats.vue @@ -6,13 +6,22 @@ import { } from 'vue'; import { getStats, DateRange, GroupBy, StatsResponse, + startAnnotationStatsJob, getLatestAnnotationStats, AnnotationStatsReport, } from 'platform/web-girder/api/configuration.service'; +import girderRest from 'platform/web-girder/plugins/girder'; +import { isJobFinished, jobSucceeded } from 'platform/web-girder/store/useJobs'; import * as d3 from 'd3'; type DataRecord = Record; type ColorMapping = Record; +type AnnotationStatsJobDoc = { + status: number; + title?: string; + meta?: { annotationStats?: AnnotationStatsReport }; +}; + export default defineComponent({ name: 'StatsComponent', setup() { @@ -28,6 +37,12 @@ export default defineComponent({ const dateRangeOptions = ref(['60 days', '3 months', '6 months', '1 year', '3 years', '5 years']); const groupByOptions = ref(['', 'user', 'month']); + const annotationStatsRunning = ref(false); + const annotationStatsLoading = ref(false); + const annotationStatsJobId = ref(null); + const annotationStatsError = ref(null); + const annotationStatsReport = ref(null); + const totalJobs = computed(() => { if (statsTableData.value) { return Object.values(statsTableData.value.jobs).reduce((prev, current) => (prev + current), 0); @@ -147,8 +162,78 @@ export default defineComponent({ .attr('fill', '#3498db'); }; + const sleep = (ms: number) => new Promise((resolve) => { + setTimeout(resolve, ms); + }); + + const loadSavedAnnotationStats = async () => { + annotationStatsLoading.value = true; + try { + const { data } = await getLatestAnnotationStats(); + if (data && typeof data === 'object' && data.datasets && data.tracks) { + annotationStatsReport.value = data; + } + } catch { + // No saved report yet (or folder missing) is fine on first visit. + } finally { + annotationStatsLoading.value = false; + } + }; + + const runAnnotationStats = async () => { + annotationStatsRunning.value = true; + annotationStatsError.value = null; + annotationStatsJobId.value = null; + try { + const { data } = await startAnnotationStatsJob(); + annotationStatsJobId.value = data.jobId; + const deadline = Date.now() + 60 * 60 * 1000; + /* eslint-disable no-await-in-loop -- poll until the inventory job finishes */ + while (Date.now() < deadline) { + const { data: job } = await girderRest.get(`job/${data.jobId}`); + if (isJobFinished(job.status)) { + if (!jobSucceeded(job.status)) { + throw new Error(job.title || 'Annotation stats job failed'); + } + // Prefer the persisted Stats-folder copy so refresh matches this view. + const { data: saved } = await getLatestAnnotationStats(); + const report = saved || job.meta?.annotationStats; + if (!report) { + throw new Error('Annotation stats job finished without a report payload'); + } + annotationStatsReport.value = report; + return; + } + await sleep(2000); + } + /* eslint-enable no-await-in-loop */ + throw new Error('Timed out waiting for annotation stats job'); + } catch (err) { + annotationStatsError.value = err instanceof Error ? err.message : String(err); + } finally { + annotationStatsRunning.value = false; + } + }; + + const downloadAnnotationStats = () => { + if (!annotationStatsReport.value) return; + const blob = new Blob( + [JSON.stringify(annotationStatsReport.value, null, 2)], + { type: 'application/json' }, + ); + const url = URL.createObjectURL(blob); + const anchor = document.createElement('a'); + anchor.href = url; + anchor.download = `annotation-stats-${annotationStatsReport.value.generatedAt}.json`; + anchor.click(); + URL.revokeObjectURL(url); + }; + // Fetch initial data on mounted - onMounted(() => fetchData()); + onMounted(() => { + fetchData(); + loadSavedAnnotationStats(); + }); // Watch for changes in dropdowns watch([selectedDateRange, selectedGroupBy], () => fetchData()); @@ -165,6 +250,13 @@ export default defineComponent({ userColorMapping, responseData, userLimit, + annotationStatsRunning, + annotationStatsLoading, + annotationStatsJobId, + annotationStatsError, + annotationStatsReport, + runAnnotationStats, + downloadAnnotationStats, }; }, }); @@ -333,6 +425,96 @@ export default defineComponent({ + + + + + Annotation inventory + +

+ Run a background job that counts datasets, tracks, confidence ≥ 1, + curation classes (mostly manual / substantially corrected / mostly computed), + and label totals. Results are saved under your user + Stats folder so they remain after refresh. +

+ + Generate annotation report + + + View job + + + {{ annotationStatsError }} + +
+ Loading saved report... +
+
+ No saved report yet. Generate one to keep results available after refresh. +
+ + + + + diff --git a/server/dive_server/crud_annotation_stats.py b/server/dive_server/crud_annotation_stats.py new file mode 100644 index 000000000..1dbe3220d --- /dev/null +++ b/server/dive_server/crud_annotation_stats.py @@ -0,0 +1,429 @@ +"""Admin annotation inventory stats (datasets / tracks / labels / curation classes).""" + +from datetime import datetime, timezone +from io import BytesIO +import json +from typing import Any, Callable, Dict, Iterable, List, Optional, TypedDict + +from dive_utils import TRUTHY_META_VALUES, constants, types + +ProgressCallback = Callable[[str], None] + +ANNOTATION_STATS_LATEST = constants.AnnotationStatsLatestFileName + +# Classification thresholds (aligned with admin Mongo sketches) +MIN_MANUAL_TRACKS = 5 +MANUAL_FRAC = 0.9 +MIN_COMPUTED_FOR_MIXED = 10 +MIN_CORRECTED_FOR_MIXED = 15 +CORRECTED_FRAC_OF_COMPUTED = 0.25 + + +class DatasetKindCounts(TypedDict): + total: int + mostlyManual: int + mixedSubstantialCorrections: int + mostlyComputed: int + empty: int + + +class TrackCounts(TypedDict): + total: int + confidenceGte1: int + + +class LabelCount(TypedDict): + label: str + count: int + + +class AnnotationStatsReport(TypedDict): + generatedAt: str + datasets: DatasetKindCounts + tracks: TrackCounts + labels: List[LabelCount] + + +class DatasetTrackRow(TypedDict): + """Per-dataset live-track counters from the trackItem aggregation.""" + + dataset: Any + total: int + manual: int + # Non-manual tracks whose first confidence pair is >= 1 (accept / type-set proxy) + conf1NonManual: int + confidenceGte1: int + + +def classify_dataset_row(row: DatasetTrackRow) -> str: + """ + Classify a dataset with live tracks. + + Returns one of: mostlyManual, mixedSubstantialCorrections, mostlyComputed. + Corrections are inferred from non-manual tracks with first-pair confidence >= 1 + (no features.userModified scan). + """ + total = int(row.get('total') or 0) + manual = int(row.get('manual') or 0) + computed = max(total - manual, 0) + corrected = int(row.get('conf1NonManual') or 0) + manual_frac = (manual / total) if total else 0.0 + corrected_frac = (corrected / computed) if computed else 0.0 + + if total >= MIN_MANUAL_TRACKS and manual_frac >= MANUAL_FRAC: + return 'mostlyManual' + if ( + computed >= MIN_COMPUTED_FOR_MIXED + and corrected >= MIN_CORRECTED_FOR_MIXED + and corrected_frac >= CORRECTED_FRAC_OF_COMPUTED + ): + return 'mixedSubstantialCorrections' + return 'mostlyComputed' + + +def fold_dataset_rows( + rows: Iterable[DatasetTrackRow], + dataset_folder_total: int, +) -> Dict[str, Any]: + """Fold per-dataset aggregation rows into totals + classification counts.""" + datasets: DatasetKindCounts = { + 'total': int(dataset_folder_total), + 'mostlyManual': 0, + 'mixedSubstantialCorrections': 0, + 'mostlyComputed': 0, + 'empty': 0, + } + tracks: TrackCounts = { + 'total': 0, + 'confidenceGte1': 0, + } + datasets_with_tracks = 0 + + for row in rows: + total = int(row.get('total') or 0) + if total <= 0: + continue + datasets_with_tracks += 1 + tracks['total'] += total + tracks['confidenceGte1'] += int(row.get('confidenceGte1') or 0) + kind = classify_dataset_row(row) + datasets[kind] = datasets[kind] + 1 # type: ignore[literal-required] + + empty = max(int(dataset_folder_total) - datasets_with_tracks, 0) + datasets['empty'] = empty + return {'datasets': datasets, 'tracks': tracks} + + +def _top_pair_label_expr() -> Dict[str, Any]: + """Mongo expression: raw type name of the highest-score confidence pair.""" + return { + '$let': { + 'vars': { + 'top': { + '$reduce': { + 'input': {'$ifNull': ['$confidencePairs', []]}, + 'initialValue': [], + 'in': { + '$cond': [ + { + '$or': [ + {'$eq': [{'$size': '$$value'}, 0]}, + { + '$gt': [ + {'$arrayElemAt': ['$$this', 1]}, + {'$arrayElemAt': ['$$value', 1]}, + ] + }, + ] + }, + '$$this', + '$$value', + ] + }, + } + } + }, + 'in': {'$arrayElemAt': ['$$top', 0]}, + } + } + + +def _dataset_track_pipeline() -> List[Dict[str, Any]]: + """Aggregate live tracks per dataset with curation signals. + + Only reads ``attributes`` / ``confidencePairs`` — not ``features`` — so the + scan stays viable on very large ``trackItem`` collections. + """ + return [ + {'$match': {'rev_deleted': {'$exists': False}}}, + { + '$project': { + '_id': 0, + 'dataset': 1, + 'userCreated': {'$eq': ['$attributes.userCreated', True]}, + 'firstConf': {'$arrayElemAt': [{'$arrayElemAt': ['$confidencePairs', 0]}, 1]}, + } + }, + { + '$group': { + '_id': '$dataset', + 'total': {'$sum': 1}, + 'manual': {'$sum': {'$cond': ['$userCreated', 1, 0]}}, + 'conf1NonManual': { + '$sum': { + '$cond': [ + { + '$and': [ + {'$not': ['$userCreated']}, + {'$gte': ['$firstConf', 1]}, + ] + }, + 1, + 0, + ] + } + }, + 'confidenceGte1': {'$sum': {'$cond': [{'$gte': ['$firstConf', 1]}, 1, 0]}}, + } + }, + { + '$project': { + '_id': 0, + 'dataset': '$_id', + 'total': 1, + 'manual': 1, + 'conf1NonManual': 1, + 'confidenceGte1': 1, + } + }, + ] + + +def _label_pipeline() -> List[Dict[str, Any]]: + """Count live tracks by highest raw confidence-pair type name.""" + return [ + {'$match': {'rev_deleted': {'$exists': False}}}, + {'$project': {'_id': 0, 'label': _top_pair_label_expr()}}, + {'$match': {'$expr': {'$eq': [{'$type': '$label'}, 'string']}}}, + {'$group': {'_id': '$label', 'count': {'$sum': 1}}}, + {'$sort': {'count': -1, '_id': 1}}, + {'$project': {'_id': 0, 'label': '$_id', 'count': 1}}, + ] + + +def _annotate_folder_query() -> Dict[str, Any]: + return {f'meta.{constants.DatasetMarker}': {'$in': TRUTHY_META_VALUES}} + + +def compute_annotation_stats( + *, + dataset_rows: Optional[Iterable[DatasetTrackRow]] = None, + labels: Optional[Iterable[LabelCount]] = None, + dataset_folder_total: Optional[int] = None, + progress: Optional[ProgressCallback] = None, +) -> AnnotationStatsReport: + """ + Build the annotation stats report. + + Optional kwargs allow unit tests to inject precomputed rows without Mongo. + Girder models are imported lazily so pure helpers stay importable in unit tests. + ``progress`` receives human-readable stage messages for job logs. + """ + + def _progress(message: str) -> None: + if progress is not None: + progress(message) + + if dataset_folder_total is None: + from girder.models.folder import Folder + + _progress('Counting annotate dataset folders...') + dataset_folder_total = Folder().find(_annotate_folder_query()).count() + _progress(f'Found {dataset_folder_total} dataset folders') + + if dataset_rows is None: + from dive_server.crud_annotation import TrackItem + + _progress( + 'Aggregating live tracks by dataset ' + '(confidence / userCreated only; this may take a while)...' + ) + dataset_rows = list( + TrackItem().collection.aggregate( + _dataset_track_pipeline(), + allowDiskUse=True, + ) + ) + _progress(f'Track aggregation done: {len(dataset_rows)} datasets with tracks') + else: + dataset_rows = list(dataset_rows) + + _progress('Classifying datasets (manual / corrected / computed / empty)...') + folded = fold_dataset_rows(dataset_rows, dataset_folder_total) + _progress( + 'Classification done: ' + f"mostlyManual={folded['datasets']['mostlyManual']}, " + f"mixedSubstantialCorrections=" + f"{folded['datasets']['mixedSubstantialCorrections']}, " + f"mostlyComputed={folded['datasets']['mostlyComputed']}, " + f"empty={folded['datasets']['empty']}; " + f"tracks={folded['tracks']['total']}, " + f"confidenceGte1={folded['tracks']['confidenceGte1']}" + ) + + if labels is None: + from dive_server.crud_annotation import TrackItem + + _progress( + 'Aggregating label totals from confidence pairs ' + '(this may take a while on large deployments)...' + ) + labels = list( + TrackItem().collection.aggregate( + _label_pipeline(), + allowDiskUse=True, + ) + ) + _progress(f'Label aggregation done: {len(labels)} distinct labels') + else: + labels = list(labels) + + _progress('Building report payload...') + return { + 'generatedAt': datetime.now(timezone.utc).isoformat(), + 'datasets': folded['datasets'], + 'tracks': folded['tracks'], + 'labels': labels, + } + + +def annotation_stats_folder(user: types.GirderUserModel) -> types.GirderModel: + """Ensure the admin user has a private Stats folder for inventory reports.""" + from girder.models.folder import Folder + + return Folder().createFolder( + user, + constants.AnnotationStatsFolderName, + description='Admin annotation inventory reports.', + parentType='user', + public=False, + creator=user, + reuseExisting=True, + ) + + +def _upload_json_file( + user: types.GirderUserModel, + folder: types.GirderModel, + name: str, + payload: bytes, +) -> types.GirderModel: + from girder.models.item import Item + from girder.models.upload import Upload + + existing = Item().findOne({'folderId': folder['_id'], 'name': name}) + if existing: + Item().remove(existing) + return Upload().uploadFromFile( + BytesIO(payload), + len(payload), + name, + parentType='folder', + parent=folder, + user=user, + mimeType='application/json', + ) + + +def save_annotation_stats_report( + user: types.GirderUserModel, + report: AnnotationStatsReport, +) -> Dict[str, str]: + """ + Persist the report under the user's Stats folder. + + Writes ``annotation-stats-latest.json`` (overwritten each run) and a + timestamped archive copy for history. + """ + folder = annotation_stats_folder(user) + payload = json.dumps(report, indent=2).encode('utf-8') + latest = _upload_json_file(user, folder, ANNOTATION_STATS_LATEST, payload) + stamp = report['generatedAt'].replace(':', '-').replace('+', '_') + archive_name = f'annotation-stats-{stamp}.json' + archive = _upload_json_file(user, folder, archive_name, payload) + return { + 'folderId': str(folder['_id']), + 'latestItemId': str(latest.get('itemId') or latest.get('_id')), + 'archiveItemId': str(archive.get('itemId') or archive.get('_id')), + 'folderName': constants.AnnotationStatsFolderName, + 'latestFileName': ANNOTATION_STATS_LATEST, + } + + +def _load_report_from_stats_folder( + user: types.GirderUserModel, +) -> Optional[AnnotationStatsReport]: + """Load ``annotation-stats-latest.json`` from the user's Stats folder, if present.""" + from girder.exceptions import GirderException + from girder.models.file import File + from girder.models.folder import Folder + from girder.models.item import Item + + folder = Folder().findOne( + { + 'parentId': user['_id'], + 'parentCollection': 'user', + 'name': constants.AnnotationStatsFolderName, + } + ) + if not folder: + return None + item = Item().findOne({'folderId': folder['_id'], 'name': ANNOTATION_STATS_LATEST}) + if not item: + return None + files = list(Item().childFiles(item, limit=1)) + if not files: + Item().remove(item) + return None + try: + with File().open(files[0]) as handle: + data = json.loads(handle.read().decode('utf-8')) + except (GirderException, OSError, ValueError, json.JSONDecodeError): + # Orphaned DB record or corrupt JSON — drop it so a fresh job can rewrite. + Item().remove(item) + return None + return data # type: ignore[return-value] + + +def _load_report_from_latest_job( + user: types.GirderUserModel, +) -> Optional[AnnotationStatsReport]: + """Fall back to ``meta.annotationStats`` on the user's newest successful stats job.""" + from girder_jobs.models.job import Job, JobStatus + + job = Job().findOne( + { + 'userId': user['_id'], + 'type': 'Annotation Stats', + 'status': JobStatus.SUCCESS, + 'meta.annotationStats': {'$exists': True}, + }, + sort=[('created', -1)], + ) + if not job: + return None + report = (job.get('meta') or {}).get('annotationStats') + return report if isinstance(report, dict) else None + + +def load_latest_annotation_stats_report( + user: types.GirderUserModel, +) -> Optional[AnnotationStatsReport]: + """ + Return the newest annotation inventory report for ``user``. + + Prefers the Stats-folder JSON file; if that is missing, uses the latest + successful Annotation Stats job meta so Admin Stats can still download after + refresh. + """ + return _load_report_from_stats_folder(user) or _load_report_from_latest_job(user) diff --git a/server/dive_server/views_configuration.py b/server/dive_server/views_configuration.py index d0bfe9d27..763d89b48 100644 --- a/server/dive_server/views_configuration.py +++ b/server/dive_server/views_configuration.py @@ -79,6 +79,8 @@ def __init__(self, resourceName): self.route("POST", ("upgrade_pipelines",), self.upgrade_pipelines) self.route("POST", ("update_containers",), self.update_containers) self.route("GET", ("stats",), self.get_dataset_stats) + self.route("GET", ("annotation_stats",), self.get_annotation_stats) + self.route("POST", ("annotation_stats",), self.start_annotation_stats) @access.public @autoDescribeRoute(Description("Get Configuration Information")) @@ -308,6 +310,44 @@ def update_containers(self): except requests.exceptions.RequestException as err: return f"Something went wrong: {err}" + @access.admin + @autoDescribeRoute( + Description( + "Return the latest saved annotation inventory report for the current admin, " + "loaded from their private Stats/annotation-stats-latest.json file. " + "Returns null when no report has been generated yet." + ) + ) + def get_annotation_stats(self): + from dive_server.crud_annotation_stats import load_latest_annotation_stats_report + + return load_latest_annotation_stats_report(self.getCurrentUser()) + + @access.admin + @autoDescribeRoute( + Description( + "Start a background job that builds a global annotation inventory report " + "(dataset counts, track confidence, curation classes, and label totals). " + "The finished report is stored on job.meta.annotationStats and saved under " + "the admin user's private Stats folder." + ) + ) + def start_annotation_stats(self): + token = Token().createToken(user=self.getCurrentUser(), days=1) + from dive_tasks.annotation_stats import run_annotation_stats_job + + # One titled Girder job via girder_job_title (not createLocalJob + .delay(), + # which left an untitled Celery job in the UI). + async_result = run_annotation_stats_job.apply_async( + queue='local', + kwargs=dict( + girder_job_title='Annotation Stats Report', + girder_job_type='Annotation Stats', + girder_client_token=str(token['_id']), + ), + ) + return {'jobId': str(async_result.job['_id'])} + @access.admin @autoDescribeRoute( Description("Get dataset and job statistics within a specified time range") diff --git a/server/dive_tasks/__init__.py b/server/dive_tasks/__init__.py index 60fa449ef..b167d0aab 100644 --- a/server/dive_tasks/__init__.py +++ b/server/dive_tasks/__init__.py @@ -24,6 +24,7 @@ def task_imports(self): 'dive_tasks.import_coco', 'dive_tasks.split_stitched', 'dive_tasks.finalize_multicam', + 'dive_tasks.annotation_stats', 'dive_tasks.tasks', 'dive_tasks.local_tasks', ] diff --git a/server/dive_tasks/annotation_stats.py b/server/dive_tasks/annotation_stats.py new file mode 100644 index 000000000..76420a512 --- /dev/null +++ b/server/dive_tasks/annotation_stats.py @@ -0,0 +1,98 @@ +"""Background job that computes a global annotation inventory report for admins.""" + +from girder.models.user import User +from girder_jobs.models.job import Job, JobStatus +from girder_worker.app import app +from girder_worker.task import Task +from girder_worker.utils import JobManager + +from dive_server.crud_annotation_stats import compute_annotation_stats, save_annotation_stats_report +from dive_tasks.manager import patch_manager + +# Approximate stage count for the Girder progress bar (logs still list every step). +PROGRESS_TOTAL = 9 + + +def _girder_job_id(manager: JobManager, task: Task): + """Resolve the Girder job id for this Celery task.""" + # jobInfoSpec sets JobManager.reference to str(job['_id']). + ref = getattr(manager, 'reference', None) + if ref: + return ref + # Fallback: jobs scheduled by girder-worker store celeryTaskId. + task_id = getattr(getattr(task, 'request', None), 'id', None) + if task_id: + found = Job().findOne({'celeryTaskId': task_id}) + if found: + return found['_id'] + return None + + +@app.task(bind=True, queue='local', acks_late=True, ignore_result=True) +def run_annotation_stats_job(self: Task) -> None: + """ + Build the annotation inventory report, store it on ``job.meta.annotationStats``, + and save JSON under the running admin user's ``Stats`` folder. + + Scheduled with ``girder_job_title`` so Girder shows one named job (same pattern + as finalize_multicam — not createLocalJob + .delay()). + """ + manager = patch_manager(self.job_manager) + manager.updateStatus(JobStatus.RUNNING) + manager.updateProgress(total=PROGRESS_TOTAL, current=0, message='Starting') + manager.write('Started annotation stats report\n') + + step = {'n': 0} + + def on_progress(message: str) -> None: + step['n'] += 1 + manager.updateProgress( + total=PROGRESS_TOTAL, + current=min(step['n'], PROGRESS_TOTAL - 1), + message=message, + ) + manager.write(f'{message}\n') + + try: + report = compute_annotation_stats(progress=on_progress) + job_id = _girder_job_id(manager, self) + if not job_id: + raise RuntimeError('Annotation stats job is missing its Girder job id') + job = Job().load(job_id, force=True) + user = User().load(job['userId'], force=True) + if not user: + raise RuntimeError('Annotation stats job is missing its owning user') + + on_progress(f'Saving report to user Stats folder for {user.get("login", user["_id"])}...') + saved = save_annotation_stats_report(user, report) + manager.write( + f'Saved {saved["latestFileName"]} under ~/{saved["folderName"]} ' + f'(folderId={saved["folderId"]})\n' + ) + + meta = dict(job.get('meta') or {}) + meta['annotationStats'] = report + meta['annotationStatsSaved'] = saved + summary = ( + 'Finished annotation stats report: ' + f"{report['datasets']['total']} datasets, " + f"{report['tracks']['total']} tracks, " + f"{report['tracks']['confidenceGte1']} confidence>=1, " + f"{len(report['labels'])} labels\n" + ) + manager.updateProgress( + total=PROGRESS_TOTAL, + current=PROGRESS_TOTAL, + message='Complete', + forceFlush=True, + ) + Job().updateJob( + job, + log=summary, + status=JobStatus.SUCCESS, + otherFields={'meta': meta}, + ) + except Exception as exc: + manager.write(f'Annotation stats report failed: {exc}\n') + manager.updateStatus(JobStatus.ERROR) + raise diff --git a/server/dive_utils/constants.py b/server/dive_utils/constants.py index 90eb8e5a0..16b226871 100644 --- a/server/dive_utils/constants.py +++ b/server/dive_utils/constants.py @@ -165,6 +165,9 @@ ViameDataFolderName = "VIAME" # The name of the subfolder for training results TrainingOutputFolderName = "VIAME Training Results" +# Admin annotation inventory reports (created under the running admin's user) +AnnotationStatsFolderName = "Stats" +AnnotationStatsLatestFileName = "annotation-stats-latest.json" # The name of the source folder holding zip backups SourceFolderName = "source" # The name of the auxiliary folder diff --git a/server/tests/test_annotation_stats.py b/server/tests/test_annotation_stats.py new file mode 100644 index 000000000..e8928e2bb --- /dev/null +++ b/server/tests/test_annotation_stats.py @@ -0,0 +1,104 @@ +from dive_server.crud_annotation_stats import ( + DatasetTrackRow, + classify_dataset_row, + compute_annotation_stats, + fold_dataset_rows, +) + + +def _row(**overrides) -> DatasetTrackRow: + base: DatasetTrackRow = { + 'dataset': 'ds', + 'total': 0, + 'manual': 0, + 'conf1NonManual': 0, + 'confidenceGte1': 0, + } + base.update(overrides) # type: ignore[typeddict-item] + return base + + +def test_classify_mostly_manual(): + assert classify_dataset_row(_row(total=20, manual=19, confidenceGte1=19)) == 'mostlyManual' + + +def test_classify_mostly_manual_requires_min_tracks(): + assert classify_dataset_row(_row(total=4, manual=4, confidenceGte1=4)) == 'mostlyComputed' + + +def test_classify_mixed_substantial_corrections(): + assert ( + classify_dataset_row( + _row( + total=100, + manual=5, + conf1NonManual=30, + confidenceGte1=35, + ) + ) + == 'mixedSubstantialCorrections' + ) + + +def test_classify_mixed_needs_enough_corrected(): + # 10 corrected of 95 computed is ~10.5%, below 25% and below absolute 15 + assert ( + classify_dataset_row( + _row( + total=100, + manual=5, + conf1NonManual=10, + confidenceGte1=10, + ) + ) + == 'mostlyComputed' + ) + + +def test_fold_dataset_rows_totals_and_empty(): + rows = [ + _row( + dataset='a', + total=20, + manual=20, + confidenceGte1=20, + ), + _row( + dataset='b', + total=100, + manual=0, + conf1NonManual=30, + confidenceGte1=30, + ), + _row( + dataset='c', + total=50, + manual=0, + confidenceGte1=0, + ), + ] + folded = fold_dataset_rows(rows, dataset_folder_total=5) + assert folded['datasets']['total'] == 5 + assert folded['datasets']['mostlyManual'] == 1 + assert folded['datasets']['mixedSubstantialCorrections'] == 1 + assert folded['datasets']['mostlyComputed'] == 1 + assert folded['datasets']['empty'] == 2 + assert folded['tracks']['total'] == 170 + assert folded['tracks']['confidenceGte1'] == 50 + + +def test_compute_annotation_stats_with_injected_rows(): + report = compute_annotation_stats( + dataset_rows=[ + _row(dataset='a', total=10, manual=10, confidenceGte1=10), + ], + labels=[{'label': 'fish', 'count': 7}, {'label': 'shark', 'count': 3}], + dataset_folder_total=3, + ) + assert report['datasets']['total'] == 3 + assert report['datasets']['mostlyManual'] == 1 + assert report['datasets']['empty'] == 2 + assert report['tracks']['total'] == 10 + assert report['tracks']['confidenceGte1'] == 10 + assert report['labels'][0]['label'] == 'fish' + assert report['generatedAt']