From 143f3d30088bb49a8f406d916cb183c98d8fd6f6 Mon Sep 17 00:00:00 2001 From: Mackenzie Mathis Date: Fri, 13 Jun 2025 16:15:46 +0200 Subject: [PATCH 01/12] Update benchmark.py - changed benchmarking link; needs testing --- dlclive/benchmark.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dlclive/benchmark.py b/dlclive/benchmark.py index 4cb4fb1..a07faca 100644 --- a/dlclive/benchmark.py +++ b/dlclive/benchmark.py @@ -37,7 +37,7 @@ def download_benchmarking_data( target_dir=".", - url="http://deeplabcut.rowland.harvard.edu/datasets/dlclivebenchmark.tar.gz", + url="https://huggingface.co/datasets/mwmathis/DLCspeed_benchmarking/blob/main/Data-DLC-live-benchmark.zip", ): """ Downloads a DeepLabCut-Live benchmarking Data (videos & DLC models). From f8fc8ba13bddb2aecbfe80c85d7cff0da2f76622 Mon Sep 17 00:00:00 2001 From: Mackenzie Mathis Date: Sat, 14 Jun 2025 14:12:53 +0200 Subject: [PATCH 02/12] Update benchmark.py --- dlclive/benchmark.py | 39 ++++++++++++++++++--------------------- 1 file changed, 18 insertions(+), 21 deletions(-) diff --git a/dlclive/benchmark.py b/dlclive/benchmark.py index 0c79e65..1230150 100644 --- a/dlclive/benchmark.py +++ b/dlclive/benchmark.py @@ -42,37 +42,34 @@ def download_benchmarking_data( """ Downloads a DeepLabCut-Live benchmarking Data (videos & DLC models). """ + import os import urllib.request - import tarfile from tqdm import tqdm + import zipfile def show_progress(count, block_size, total_size): pbar.update(block_size) - def tarfilenamecutting(tarf): - """' auxfun to extract folder path - ie. /xyz-trainsetxyshufflez/ - """ - for memberid, member in enumerate(tarf.getmembers()): - if memberid == 0: - parent = str(member.path) - l = len(parent) + 1 - if member.path.startswith(parent): - member.path = member.path[l:] - yield member - response = urllib.request.urlopen(url) + total_size = int(response.getheader("Content-Length")) + pbar = tqdm(unit="B", total=total_size, position=0, desc="Downloading") + filename, _ = urllib.request.urlretrieve(url, filename=zip_path, reporthook=show_progress) + pbar.close() + + class DownloadProgressBar(tqdm): + def update_to(self, b=1, bsize=1, tsize=None): + if tsize is not None: + self.total = tsize + self.update(b * bsize - self.n) + + zip_path = os.path.join(target_dir, "Data-DLC-live-benchmark.zip") print( - "Downloading the benchmarking data from the DeepLabCut server @Harvard -> Go Crimson!!! {}....".format( - url - ) + f"Downloading the benchmarking data from {url} ..." ) - total_size = int(response.getheader("Content-Length")) - pbar = tqdm(unit="B", total=total_size, position=0) - filename, _ = urllib.request.urlretrieve(url, reporthook=show_progress) - with tarfile.open(filename, mode="r:gz") as tar: - tar.extractall(target_dir, members=tarfilenamecutting(tar)) + print(f"Extracting {zip_path} to {target_dir} ...") + with zipfile.ZipFile(zip_path, 'r') as zip_ref: + zip_ref.extractall(target_dir) def get_system_info() -> dict: """ Return summary info for system running benchmark From 6c61b29ecce4361fbc56f46a15d21aeb75d96e13 Mon Sep 17 00:00:00 2001 From: maximpavliv Date: Mon, 16 Jun 2025 10:26:03 +0200 Subject: [PATCH 03/12] Fix download_benchmarking_data() --- dlclive/benchmark.py | 35 +++++++++++++++++------------------ 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/dlclive/benchmark.py b/dlclive/benchmark.py index 1230150..9643b09 100644 --- a/dlclive/benchmark.py +++ b/dlclive/benchmark.py @@ -37,35 +37,34 @@ def download_benchmarking_data( target_dir=".", - url="https://huggingface.co/datasets/mwmathis/DLCspeed_benchmarking/blob/main/Data-DLC-live-benchmark.zip", + url="https://huggingface.co/datasets/mwmathis/DLCspeed_benchmarking/resolve/main/Data-DLC-live-benchmark.zip", ): """ - Downloads a DeepLabCut-Live benchmarking Data (videos & DLC models). + Downloads and extracts DeepLabCut-Live benchmarking data (videos & DLC models). """ import os import urllib.request from tqdm import tqdm import zipfile - def show_progress(count, block_size, total_size): - pbar.update(block_size) + # Avoid nested folder issue + if os.path.basename(os.path.normpath(target_dir)) == "Data-DLC-live-benchmark": + target_dir = os.path.dirname(os.path.normpath(target_dir)) + os.makedirs(target_dir, exist_ok=True) # Ensure target directory exists - response = urllib.request.urlopen(url) - total_size = int(response.getheader("Content-Length")) - pbar = tqdm(unit="B", total=total_size, position=0, desc="Downloading") - filename, _ = urllib.request.urlretrieve(url, filename=zip_path, reporthook=show_progress) - pbar.close() + zip_path = os.path.join(target_dir, "Data-DLC-live-benchmark.zip") - class DownloadProgressBar(tqdm): - def update_to(self, b=1, bsize=1, tsize=None): - if tsize is not None: - self.total = tsize - self.update(b * bsize - self.n) + if os.path.exists(zip_path): + print(f"{zip_path} already exists. Skipping download.") + else: + def show_progress(count, block_size, total_size): + pbar.update(block_size) - zip_path = os.path.join(target_dir, "Data-DLC-live-benchmark.zip") - print( - f"Downloading the benchmarking data from {url} ..." - ) + print(f"Downloading the benchmarking data from {url} ...") + pbar = tqdm(unit="B", total=0, position=0, desc="Downloading") + + filename, _ = urllib.request.urlretrieve(url, filename=zip_path, reporthook=show_progress) + pbar.close() print(f"Extracting {zip_path} to {target_dir} ...") with zipfile.ZipFile(zip_path, 'r') as zip_ref: From ef377467c2515b1fbdf1c8e2b59800e5dc4d0092 Mon Sep 17 00:00:00 2001 From: maximpavliv Date: Mon, 16 Jun 2025 10:26:25 +0200 Subject: [PATCH 04/12] Ignore __MACOSX directories --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 7fa3a49..2ce1454 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,8 @@ benchmarking/results* **DS_Store* *vscode* +**/__MACOSX/ + # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] From 5ed1910e8332ee8fc616abbd26b20532bec731a7 Mon Sep 17 00:00:00 2001 From: Mackenzie Mathis Date: Mon, 16 Jun 2025 11:19:36 +0200 Subject: [PATCH 05/12] pytest for benchmarking --- tests/test_benchmark_script.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 tests/test_benchmark_script.py diff --git a/tests/test_benchmark_script.py b/tests/test_benchmark_script.py new file mode 100644 index 0000000..3cb209c --- /dev/null +++ b/tests/test_benchmark_script.py @@ -0,0 +1,29 @@ +import os +import glob +import pathlib +import pytest +from dlclive import benchmark_videos, download_benchmarking_data + +@pytest.mark.functional +def test_benchmark_script_runs(tmp_path): + datafolder = tmp_path / "Data-DLC-live-benchmark" + download_benchmarking_data(str(datafolder)) + + dog_models = glob.glob(str(datafolder / "dog" / "*[!avi]")) + dog_video = glob.glob(str(datafolder / "dog" / "*.avi"))[0] + mouse_models = glob.glob(str(datafolder / "mouse_lick" / "*[!avi]")) + mouse_video = glob.glob(str(datafolder / "mouse_lick" / "*.avi"))[0] + + out_dir = tmp_path / "results" + out_dir.mkdir(exist_ok=True) + + pixels = [2500, 10000] + n_frames = 10 + + for m in dog_models: + benchmark_videos(m, dog_video, output=str(out_dir), n_frames=n_frames, pixels=pixels) + + for m in mouse_models: + benchmark_videos(m, mouse_video, output=str(out_dir), n_frames=n_frames, pixels=pixels) + + assert any(out_dir.iterdir()) From 27e84619f35819ee123b690366921ef7397fa9cf Mon Sep 17 00:00:00 2001 From: Mackenzie Mathis Date: Mon, 16 Jun 2025 11:20:54 +0200 Subject: [PATCH 06/12] Update testing.yml --- .github/workflows/testing.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 27ea85a..83ba47c 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -72,3 +72,6 @@ jobs: - name: Run DLC Live Tests run: poetry run dlc-live-test --nodisplay + + - name: Run Functional Benchmark Test + run: poetry run pytest tests/test_benchmark_script.py From 006b1cad50635732642f2d9f83f8a6d507d5267e Mon Sep 17 00:00:00 2001 From: Mackenzie Mathis Date: Mon, 16 Jun 2025 11:25:36 +0200 Subject: [PATCH 07/12] Create pytest.ini --- pytest.ini | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 pytest.ini diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000..d08b10e --- /dev/null +++ b/pytest.ini @@ -0,0 +1,3 @@ +[pytest] +markers = + functional: functional tests From 8dd7d6d211cd4f60999e3a2ff8d6e3007169dd02 Mon Sep 17 00:00:00 2001 From: Mackenzie Mathis Date: Mon, 16 Jun 2025 11:26:23 +0200 Subject: [PATCH 08/12] Update pyproject.toml --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 9fb4eed..7451d7e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,6 +36,7 @@ torch = ">=1.10,<3.0" dlclibrary = ">=0.0.6" pandas = "^1.3" tables = "^3.6" +pytest = "^8.0" # OS-specific TensorFlow packages tensorflow = [ @@ -44,7 +45,6 @@ tensorflow = [ ] tensorflow-macos = { version = ">=2.7.0,<2.12", markers = "sys_platform == 'darwin'" } - [tool.poetry.group.dev.dependencies] [build-system] From c499cb73b8d659c829cd3bf6ac0e8586b132723f Mon Sep 17 00:00:00 2001 From: Mackenzie Mathis Date: Mon, 16 Jun 2025 11:38:14 +0200 Subject: [PATCH 09/12] Update testing.yml --- .github/workflows/testing.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 83ba47c..25c9a92 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -68,7 +68,7 @@ jobs: run: poetry lock --no-cache - name: Install project dependencies - run: poetry install --no-root + run: poetry install --with dev - name: Run DLC Live Tests run: poetry run dlc-live-test --nodisplay From 4c58c7200e8ef7c12a9dfa353776fb1dc9da9ab2 Mon Sep 17 00:00:00 2001 From: Mackenzie Mathis Date: Mon, 16 Jun 2025 12:25:51 +0200 Subject: [PATCH 10/12] Update test_benchmark_script.py --- tests/test_benchmark_script.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tests/test_benchmark_script.py b/tests/test_benchmark_script.py index 3cb209c..8254db0 100644 --- a/tests/test_benchmark_script.py +++ b/tests/test_benchmark_script.py @@ -18,12 +18,16 @@ def test_benchmark_script_runs(tmp_path): out_dir.mkdir(exist_ok=True) pixels = [2500, 10000] - n_frames = 10 + n_frames = 15 for m in dog_models: - benchmark_videos(m, dog_video, output=str(out_dir), n_frames=n_frames, pixels=pixels) + print(f"Running dog model: {m}") + result = benchmark_videos(m, dog_video, output=str(out_dir), n_frames=n_frames, pixels=pixels) + print("Dog model result:", result) for m in mouse_models: - benchmark_videos(m, mouse_video, output=str(out_dir), n_frames=n_frames, pixels=pixels) + print(f"Running mouse model: {m}") + result = benchmark_videos(m, mouse_video, output=str(out_dir), n_frames=n_frames, pixels=pixels) + print("Mouse model result:", result) assert any(out_dir.iterdir()) From 70d82e01816e021f245eeb35fddfe0b9ec5fb1d0 Mon Sep 17 00:00:00 2001 From: Mackenzie Mathis Date: Mon, 16 Jun 2025 12:30:55 +0200 Subject: [PATCH 11/12] Update test_benchmark_script.py --- tests/test_benchmark_script.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_benchmark_script.py b/tests/test_benchmark_script.py index 8254db0..66f5015 100644 --- a/tests/test_benchmark_script.py +++ b/tests/test_benchmark_script.py @@ -21,9 +21,9 @@ def test_benchmark_script_runs(tmp_path): n_frames = 15 for m in dog_models: - print(f"Running dog model: {m}") - result = benchmark_videos(m, dog_video, output=str(out_dir), n_frames=n_frames, pixels=pixels) - print("Dog model result:", result) + print(f"Running dog model: {m}") + result = benchmark_videos(m, dog_video, output=str(out_dir), n_frames=n_frames, pixels=pixels) + print("Dog model result:", result) for m in mouse_models: print(f"Running mouse model: {m}") From cd0100d51fab22a824966bc9537aa9371f4f8c73 Mon Sep 17 00:00:00 2001 From: Alexander Mathis Date: Mon, 16 Jun 2025 13:45:51 +0200 Subject: [PATCH 12/12] I'd make the numbers even smaller for a unit-test! --- tests/test_benchmark_script.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_benchmark_script.py b/tests/test_benchmark_script.py index 66f5015..ff3a3c2 100644 --- a/tests/test_benchmark_script.py +++ b/tests/test_benchmark_script.py @@ -17,8 +17,8 @@ def test_benchmark_script_runs(tmp_path): out_dir = tmp_path / "results" out_dir.mkdir(exist_ok=True) - pixels = [2500, 10000] - n_frames = 15 + pixels = [100, 400] #[2500, 10000] + n_frames = 5 for m in dog_models: print(f"Running dog model: {m}")