From ed03d1fd1e2db97d6ce8f4c6892932719f4dc64a Mon Sep 17 00:00:00 2001 From: Padraig Gleeson Date: Thu, 20 Mar 2025 15:45:23 +0000 Subject: [PATCH 01/13] Test on experimental too --- .github/workflows/ci_python.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci_python.yml b/.github/workflows/ci_python.yml index 94f02dc..64e30ed 100644 --- a/.github/workflows/ci_python.yml +++ b/.github/workflows/ci_python.yml @@ -2,9 +2,9 @@ name: Test Python on: push: - branches: [ master, development ] + branches: [ master, development, experimental, test* ] pull_request: - branches: [ master, development ] + branches: [ master, development, experimental, test* ] jobs: build: @@ -13,13 +13,13 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] + python-version: ["3.9", "3.10", "3.11"] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} From b00c17166c9e06d6823d25e41814f75bad57da7c Mon Sep 17 00:00:00 2001 From: pgleeson Date: Thu, 20 Mar 2025 16:07:43 +0000 Subject: [PATCH 02/13] Test unpin scipy --- src/Python/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Python/setup.py b/src/Python/setup.py index 34be5cf..ba34cb1 100644 --- a/src/Python/setup.py +++ b/src/Python/setup.py @@ -52,7 +52,7 @@ keywords='C. elegans worm tracking', packages=['wcon'], package_data={'': ['../../wcon_schema.json']}, - install_requires=['jsonschema', 'six', 'numpy', 'scipy<=0.17.1'] + install_requires=['jsonschema', 'six', 'scipy', 'pandas'] # Actually also requires numpy, scipy and numpy but I don't want to force # pip to install these since pip is bad at that for those packages. ) From b19292bc388ac4b32cd7bb3ff7228424cfab6196 Mon Sep 17 00:00:00 2001 From: Padraig Gleeson Date: Wed, 23 Apr 2025 17:36:54 +0100 Subject: [PATCH 03/13] Update ci_python.yml --- .github/workflows/ci_python.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_python.yml b/.github/workflows/ci_python.yml index 64e30ed..c24ee59 100644 --- a/.github/workflows/ci_python.yml +++ b/.github/workflows/ci_python.yml @@ -13,7 +13,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.9", "3.10", "3.11"] + python-version: ["3.9", "3.10", "3.11", "3.12"] steps: From 0e4febd8336fb7045098fb09612ac05b9792a729 Mon Sep 17 00:00:00 2001 From: Padraig Gleeson Date: Tue, 28 Apr 2026 21:11:50 +0100 Subject: [PATCH 04/13] Update gha versions --- .github/workflows/ci_python.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci_python.yml b/.github/workflows/ci_python.yml index c24ee59..861f362 100644 --- a/.github/workflows/ci_python.yml +++ b/.github/workflows/ci_python.yml @@ -17,9 +17,9 @@ jobs: steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} From a86449ab640d329c2eb2ad3ccaebeccca583d1bb Mon Sep 17 00:00:00 2001 From: Padraig Gleeson Date: Tue, 28 Apr 2026 21:44:41 +0100 Subject: [PATCH 05/13] Some panda modifications --- .gitignore | 2 ++ src/Python/examples/view_wcon.py | 2 +- src/Python/setup.py | 12 ++++++++++-- src/Python/wcon/wcon_data.py | 32 +++++++++++++++++++++++--------- src/Python/wcon/wcon_parser.py | 23 ++++++++++++++++------- 5 files changed, 52 insertions(+), 19 deletions(-) diff --git a/.gitignore b/.gitignore index 807e483..72ba9b5 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,5 @@ src/Python/wcon.egg* # Scala src/scala/target/* +/src/Python/example_saved_file.WCON +/src/Python/wcon/wcon_schema.json diff --git a/src/Python/examples/view_wcon.py b/src/Python/examples/view_wcon.py index a5d836f..3d10c4a 100644 --- a/src/Python/examples/view_wcon.py +++ b/src/Python/examples/view_wcon.py @@ -4,5 +4,5 @@ sys.path.append('..') from wcon import WCONWorms, MeasurementUnit -file_name = 'asic-1 (ok415) on food L_2010_07_08__11_46_40___7___5.wcon' +file_name = '../../../tests/minimax.wcon' w = WCONWorms.load_from_file(file_name) diff --git a/src/Python/setup.py b/src/Python/setup.py index ba34cb1..e2bca5a 100644 --- a/src/Python/setup.py +++ b/src/Python/setup.py @@ -12,6 +12,7 @@ from codecs import open from os import path import os +import shutil exec(open('wcon/version.py').read()) here = path.abspath(path.dirname(__file__)) @@ -24,7 +25,13 @@ with open(readme_path, encoding='utf-8') as f: long_description += f.read() -print(os.listdir('.')) # DEBUG +# The canonical wcon_schema.json lives at the repository root so it can be +# shared by every language implementation. setuptools cannot package files +# from outside the package directory, so copy it into wcon/ at build time. +repo_schema = path.join(here, '..', '..', 'wcon_schema.json') +pkg_schema = path.join(here, 'wcon', 'wcon_schema.json') +if path.exists(repo_schema): + shutil.copyfile(repo_schema, pkg_schema) setup( name='wcon', @@ -51,7 +58,8 @@ ], keywords='C. elegans worm tracking', packages=['wcon'], - package_data={'': ['../../wcon_schema.json']}, + package_data={'wcon': ['wcon_schema.json']}, + include_package_data=True, install_requires=['jsonschema', 'six', 'scipy', 'pandas'] # Actually also requires numpy, scipy and numpy but I don't want to force # pip to install these since pip is bad at that for those packages. diff --git a/src/Python/wcon/wcon_data.py b/src/Python/wcon/wcon_data.py index 7ee574e..ebcf9a9 100644 --- a/src/Python/wcon/wcon_data.py +++ b/src/Python/wcon/wcon_data.py @@ -119,6 +119,13 @@ def df_upsert(src, dest): dest_sliced.sort_index(inplace=True) src_sliced.sort_index(inplace=True) + # Align src_sliced's row/column labels to dest_sliced. The two + # were built with independent .isin() masks so column order may + # differ; pandas >=1.x refuses to compare DataFrames whose + # labels are not identical. + src_sliced = src_sliced.reindex(index=dest_sliced.index, + columns=dest_sliced.columns) + # Obtain a mask of the conflicts in the current segment # as compared with all previously loaded data. That is: # NaN NaN = False @@ -189,24 +196,30 @@ def convert_origin(df): # `for` loop loops through both `x` and `y`. if offset in cur_worm.columns.get_level_values(0): - # Consider offset as 0 if not available in a certain frame - ox_column = cur_worm.loc[:, (offset)].fillna(0) + # Consider offset as 0 if not available in a certain frame. + # Coerce to numeric: the parser can leave the offset column + # with object dtype (mixed str/int entries) when offsets + # are present in some segments but not others. + ox_column = cur_worm.loc[:, (offset)].apply( + pd.to_numeric, errors='coerce').fillna(0) # Shift our 'x' values by offset - all_x_columns = cur_worm.loc[:, (coord)] - ox_affine_change = (np.array(ox_column) * + all_x_columns = cur_worm.loc[:, (coord)].apply( + pd.to_numeric, errors='coerce') + ox_affine_change = (np.array(ox_column, dtype=float) * np.ones(all_x_columns.shape)) all_x_columns += ox_affine_change if centroid in cur_worm.columns.get_level_values(0): - cx_column = cur_worm.loc[:, (centroid)] + cx_column = cur_worm.loc[:, (centroid)].apply( + pd.to_numeric, errors='coerce') # Shift the centroid by the offset cx_column += ox_column # Now make the centroid our new offset, since the rule # is that if the offset exists, the centroid is not # the offset, but we want it to be. - cx_affine_change = (np.array(cx_column) * + cx_affine_change = (np.array(cx_column, dtype=float) * np.ones(all_x_columns.shape)) all_x_columns -= cx_affine_change @@ -224,7 +237,8 @@ def convert_origin(df): # This is so DataFrames with and without offsets # will show as comparing identically. for offset_key in offset_keys: - df.drop(offset_key, axis=1, level='key', inplace=True) + df.drop(offset_key, axis=1, level='key', inplace=True, + errors='ignore') # Because of a known issue in Pandas # (https://github.com/pydata/pandas/issues/2770), the dropped columns @@ -402,7 +416,7 @@ def _obtain_time_series_data_frame(time_series_data): cur_df = pd.DataFrame(cur_data, columns=cur_columns) cur_df.index = cur_timeframes - cur_df.index.names = 't' + cur_df.index.names = ['t'] # We want the index (time) to be in order. cur_df.sort_index(axis=0, inplace=True) @@ -466,7 +480,7 @@ def _obtain_time_series_data_frame(time_series_data): with warnings.catch_warnings(): warnings.filterwarnings(action="ignore", category=FutureWarning) df_odict[worm_id] = \ - df_odict[worm_id].convert_objects(convert_numeric=True) + df_odict[worm_id].infer_objects() # If 'head' or 'ventral' is NaN, we must specify '?' since # otherwise, when saving this object, to specify "no value" we would diff --git a/src/Python/wcon/wcon_parser.py b/src/Python/wcon/wcon_parser.py index 13010f6..135320a 100644 --- a/src/Python/wcon/wcon_parser.py +++ b/src/Python/wcon/wcon_parser.py @@ -385,12 +385,8 @@ def to_canon(self): for data_key in self.units: mu = self.units[data_key] - # Don't bother to "convert" units that are already in their - # canonical form. - if mu.unit_string == mu.canonical_unit_string: - continue - tmu = self.units['t'] + already_canonical = (mu.unit_string == mu.canonical_unit_string) for worm_id in w.worm_ids: try: @@ -398,8 +394,21 @@ def to_canon(self): mu_slice = \ w._data[worm_id].loc[:, idx[:, data_key, :]].copy() - w._data[worm_id].loc[:, idx[:, data_key, :]] = \ - mu_slice.applymap(mu.to_canon) + # The parser can leave numeric columns with object + # dtype (e.g. mixed int/str entries from how segments + # are merged). Coerce so downstream arithmetic and + # JSON serialization treat them as numbers, even when + # the unit is already canonical and no conversion is + # otherwise required. + mu_slice = mu_slice.apply(pd.to_numeric, + errors='coerce') + + if already_canonical: + w._data[worm_id].loc[:, idx[:, data_key, :]] = \ + mu_slice + else: + w._data[worm_id].loc[:, idx[:, data_key, :]] = \ + mu_slice.applymap(mu.to_canon) except KeyError: # Just ignore cases where there are "units" entries but no # corresponding data From 7a4e2bf334e7af10771062f719a987668d05e1b1 Mon Sep 17 00:00:00 2001 From: Padraig Gleeson Date: Wed, 29 Apr 2026 08:53:00 +0100 Subject: [PATCH 06/13] More fixes for latest pandas --- src/Python/tests/tests.py | 2 +- src/Python/wcon/measurement_unit.py | 9 ++++++--- src/Python/wcon/wcon_data.py | 2 +- src/Python/wcon/wcon_parser.py | 29 +++++++++++++++-------------- 4 files changed, 23 insertions(+), 19 deletions(-) diff --git a/src/Python/tests/tests.py b/src/Python/tests/tests.py index 545fbb8..73c3ab6 100644 --- a/src/Python/tests/tests.py +++ b/src/Python/tests/tests.py @@ -41,7 +41,7 @@ def flatten(list_of_lists): for element in list_of_lists: # If it's iterable but not a string or bytes, then recurse, otherwise # we are at a "leaf" node of our traversal - if(isinstance(element, collections.Iterable) and + if(isinstance(element, collections.abc.Iterable) and not isinstance(element, (str, bytes))): for sub_element in flatten(element): yield sub_element diff --git a/src/Python/wcon/measurement_unit.py b/src/Python/wcon/measurement_unit.py index 675184c..237fac2 100644 --- a/src/Python/wcon/measurement_unit.py +++ b/src/Python/wcon/measurement_unit.py @@ -637,12 +637,15 @@ def _create_from_atomic(cls, unit_string): @classmethod def _create_from_node(cls, node): """ - node: is ast.Num or ast.BinOp or ast.UnaryOp or ast.Str or ast.Name + node: is ast.Constant (numeric) or ast.BinOp or ast.UnaryOp or ast.Name The expression to be transformed into a MeasurementUnit """ - if isinstance(node, ast.Num): # - n = node.n + # ast.Num was deprecated in Python 3.8 and removed in 3.14; + # ast.Constant now represents all literal values. + if isinstance(node, ast.Constant) and isinstance( + node.value, (int, float)): # + n = node.value assert(n != 0) # A unit cannot have zero in the expression u = cls() diff --git a/src/Python/wcon/wcon_data.py b/src/Python/wcon/wcon_data.py index ebcf9a9..198df74 100644 --- a/src/Python/wcon/wcon_data.py +++ b/src/Python/wcon/wcon_data.py @@ -403,7 +403,7 @@ def _obtain_time_series_data_frame(time_series_data): for i in range(len(cur_timeframes)): data_segment[k][i] = ( data_segment[k][i] + - [np.NaN] * (max_aspect_size - len(data_segment[k][i]))) + [np.nan] * (max_aspect_size - len(data_segment[k][i]))) num_timeframes = len(cur_timeframes) diff --git a/src/Python/wcon/wcon_parser.py b/src/Python/wcon/wcon_parser.py index 135320a..916daf8 100644 --- a/src/Python/wcon/wcon_parser.py +++ b/src/Python/wcon/wcon_parser.py @@ -390,25 +390,25 @@ def to_canon(self): for worm_id in w.worm_ids: try: - # Apply across all worm ids and all aspects - mu_slice = \ - w._data[worm_id].loc[:, idx[:, data_key, :]].copy() + df = w._data[worm_id] + target_cols = [c for c in df.columns + if c[1] == data_key] + if not target_cols: + raise KeyError(data_key) # The parser can leave numeric columns with object # dtype (e.g. mixed int/str entries from how segments # are merged). Coerce so downstream arithmetic and # JSON serialization treat them as numbers, even when # the unit is already canonical and no conversion is - # otherwise required. - mu_slice = mu_slice.apply(pd.to_numeric, - errors='coerce') - - if already_canonical: - w._data[worm_id].loc[:, idx[:, data_key, :]] = \ - mu_slice - else: - w._data[worm_id].loc[:, idx[:, data_key, :]] = \ - mu_slice.applymap(mu.to_canon) + # otherwise required. Replace each column whole rather + # than via .loc[] assignment, which would preserve the + # parent column's existing (object) dtype. + for col in target_cols: + new_col = pd.to_numeric(df[col], errors='coerce') + if not already_canonical: + new_col = new_col.apply(mu.to_canon) + df[col] = new_col except KeyError: # Just ignore cases where there are "units" entries but no # corresponding data @@ -831,7 +831,8 @@ def pd_equals(df1, df2): return False try: - pd.util.testing.assert_frame_equal(df1, df2) + # pd.util.testing was removed in pandas 2.0; use pd.testing. + pd.testing.assert_frame_equal(df1, df2) except AssertionError: return False From d46814f0e9058b74f0bff0f59081b2ae4ca07a00 Mon Sep 17 00:00:00 2001 From: Padraig Gleeson Date: Wed, 29 Apr 2026 08:53:08 +0100 Subject: [PATCH 07/13] To v1.2.1 --- src/Python/wcon/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Python/wcon/version.py b/src/Python/wcon/version.py index 4cd8613..d5b90cd 100644 --- a/src/Python/wcon/version.py +++ b/src/Python/wcon/version.py @@ -6,4 +6,4 @@ # 2) we can import it in setup.py for the same reason # 3) we can import it into your module module # (from http://stackoverflow.com/questions/458550/) -__version__ = '1.1.0' +__version__ = '1.2.1' From ed56ade0985056cff79f7b0ce4a15c2100dc5077 Mon Sep 17 00:00:00 2001 From: Padraig Gleeson Date: Wed, 29 Apr 2026 08:54:59 +0100 Subject: [PATCH 08/13] Run tests --- .github/workflows/ci_python.yml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci_python.yml b/.github/workflows/ci_python.yml index 861f362..7be361e 100644 --- a/.github/workflows/ci_python.yml +++ b/.github/workflows/ci_python.yml @@ -31,6 +31,16 @@ jobs: - name: Install Python package run: | cd src/Python - python setup.py install + pip install . + + - name: Run tests + run: | + cd src/Python/tests + python diagnostic_test.py ../../../tests/minimax.wcon + python tests.py + + - name: Final version info + run: | + pip list From 036fdeacd5870785863fdc585934216684df4e2c Mon Sep 17 00:00:00 2001 From: Padraig Gleeson Date: Wed, 29 Apr 2026 09:17:38 +0100 Subject: [PATCH 09/13] Specifying draft-07 json schema --- wcon_schema.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wcon_schema.json b/wcon_schema.json index 1e75d36..2912dfe 100644 --- a/wcon_schema.json +++ b/wcon_schema.json @@ -1,5 +1,5 @@ { - "$schema": "http://json-schema.org/schema", + "$schema": "http://json-schema.org/draft-07/schema", "title": "Worm tracker Commons Object Notation (WCON)", "description": "A text-based data interchange format for *C. elegans* trackers. It is a constrained subset of JSON. It is designed to be both human and machine readable and to facilitate data exchange and inter-operability for worm tracking data that is independent of platform and the language used for implementation.", "type": "object", From e4d130a56f4a329c65ed2514c586592926b3ed8d Mon Sep 17 00:00:00 2001 From: Padraig Gleeson Date: Wed, 29 Apr 2026 09:17:48 +0100 Subject: [PATCH 10/13] Zdd psutil dependency --- src/Python/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Python/setup.py b/src/Python/setup.py index e2bca5a..7e71e0a 100644 --- a/src/Python/setup.py +++ b/src/Python/setup.py @@ -60,7 +60,7 @@ packages=['wcon'], package_data={'wcon': ['wcon_schema.json']}, include_package_data=True, - install_requires=['jsonschema', 'six', 'scipy', 'pandas'] + install_requires=['jsonschema', 'six', 'scipy', 'pandas', 'psutil'], # Actually also requires numpy, scipy and numpy but I don't want to force # pip to install these since pip is bad at that for those packages. ) From 17d164bae8493a68d3f135655322f267d1156e60 Mon Sep 17 00:00:00 2001 From: Padraig Gleeson Date: Wed, 29 Apr 2026 09:22:38 +0100 Subject: [PATCH 11/13] Fix another pandas 2 issue... --- src/Python/wcon/wcon_data.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/Python/wcon/wcon_data.py b/src/Python/wcon/wcon_data.py index 198df74..4d02e3f 100644 --- a/src/Python/wcon/wcon_data.py +++ b/src/Python/wcon/wcon_data.py @@ -502,11 +502,14 @@ def _obtain_time_series_data_frame(time_series_data): df_odict[worm_id].loc[:, idx[:, k, :]] = \ cur_slice.fillna(value=np.nan) - # Make sure aspect_size is a float, since only floats are nullable: + # Make sure aspect_size is a float, since only floats are nullable. + # Replace the column whole rather than assigning via .loc[]; pandas + # 2.x preserves the parent column's existing (object/str) dtype on + # .loc[] assignment and raises TypeError on non-string values. if 'aspect_size' in df_keys: - df_odict[worm_id].loc[:, idx[:, 'aspect_size', :]] = \ - df_odict[worm_id].loc[:, idx[:, 'aspect_size', :]] \ - .astype(float) + df = df_odict[worm_id] + for col in [c for c in df.columns if c[1] == 'aspect_size']: + df[col] = df[col].astype(float) return sort_odict(df_odict) From 1c632158c10a124bc86404a0ff681882aac551fd Mon Sep 17 00:00:00 2001 From: Padraig Gleeson Date: Wed, 29 Apr 2026 09:35:40 +0100 Subject: [PATCH 12/13] More output in diagnostics --- .github/workflows/ci_python.yml | 2 +- src/Python/tests/diagnostic_test.py | 21 +++++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci_python.yml b/.github/workflows/ci_python.yml index 7be361e..a6bd9f4 100644 --- a/.github/workflows/ci_python.yml +++ b/.github/workflows/ci_python.yml @@ -36,8 +36,8 @@ jobs: - name: Run tests run: | cd src/Python/tests - python diagnostic_test.py ../../../tests/minimax.wcon python tests.py + python diagnostic_test.py ../../../tests/minimax.wcon - name: Final version info run: | diff --git a/src/Python/tests/diagnostic_test.py b/src/Python/tests/diagnostic_test.py index 17dc45b..8372536 100644 --- a/src/Python/tests/diagnostic_test.py +++ b/src/Python/tests/diagnostic_test.py @@ -18,6 +18,7 @@ idx = pd.IndexSlice import numpy as np import time +import pprint as pp sys.path.append('..') from wcon import WCONWorms, MeasurementUnit @@ -51,6 +52,12 @@ def timing_function(): validate_against_schema=False) print("Time to load w1: " + str(timing_function() - start_time)) + print(" ------- W1 has " + str(len(w1.data)) + " rows and " + + str(len(w1.data.columns)) + " columns") + + print(pp.pformat(w1.data_as_odict)) + print (' -------- ') + # Save these worm tracks to a file, then load that file test_path = 'test.wcon' start_time = timing_function() @@ -62,6 +69,12 @@ def timing_function(): validate_against_schema=False) print("Time to load w2: " + str(timing_function() - start_time)) + print(" ------- W2 has " + str(len(w2.data)) + " rows and " + + str(len(w2.data.columns)) + " columns") + + print(pp.pformat(w2.data_as_odict)) + print (' -------- ') + # x1 = w1.data.loc[:, idx[0, 'x', 0]].fillna(0) # x2 = w2.data.loc[:, idx[0, 'x', 0]].fillna(0) # cmm = np.flatnonzero(x1 != x2) @@ -76,6 +89,14 @@ def timing_function(): # "id" first in a data segment, etc.) w3 = WCONWorms.load_from_file(test_path, validate_against_schema=False) + + + print(" ------- W3 has " + str(len(w3.data)) + " rows and " + + str(len(w3.data.columns)) + " columns") + + print(pp.pformat(w3.data_as_odict)) + print (' -------- ') + assert(w2 == w3) assert(w1 == w2) assert(w1 == w3) From 0d2fc2d7878c5d286c11d48dcd1a43999df63319 Mon Sep 17 00:00:00 2001 From: Padraig Gleeson Date: Wed, 29 Apr 2026 12:51:42 +0100 Subject: [PATCH 13/13] More pandas 3 fixes --- src/Python/wcon/wcon_data.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/Python/wcon/wcon_data.py b/src/Python/wcon/wcon_data.py index 4d02e3f..8f23258 100644 --- a/src/Python/wcon/wcon_data.py +++ b/src/Python/wcon/wcon_data.py @@ -492,15 +492,18 @@ def _obtain_time_series_data_frame(time_series_data): # We must replace NaN with None, otherwise the JSON encoder will # save 'NaN' as the string and this will get rejected by our schema - # on any subsequent loads - # Note we can't use .fillna(None) due to this issue: - # https://github.com/pydata/pandas/issues/1972 + # on any subsequent loads. + # Pandas 3.0 infers 'str' dtype for these columns, and assigning + # NaN on a str-dtype column coerces to the string 'nan'. Force + # object dtype and map both real NaN and stringified 'nan' back + # to None so downstream JSON serialization writes null. df_keys = set(df_odict[worm_id].columns.get_level_values('key')) for k in ['head', 'ventral']: if k in df_keys: - cur_slice = df_odict[worm_id].loc[:, idx[:, k, :]] - df_odict[worm_id].loc[:, idx[:, k, :]] = \ - cur_slice.fillna(value=np.nan) + df = df_odict[worm_id] + for col in [c for c in df.columns if c[1] == k]: + s = df[col].astype(object) + df[col] = s.where(s.notna() & (s != 'nan'), None) # Make sure aspect_size is a float, since only floats are nullable. # Replace the column whole rather than assigning via .loc[]; pandas