From ed03d1fd1e2db97d6ce8f4c6892932719f4dc64a Mon Sep 17 00:00:00 2001 From: Padraig Gleeson Date: Thu, 20 Mar 2025 15:45:23 +0000 Subject: [PATCH 1/8] Test on experimental too --- .github/workflows/ci_python.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci_python.yml b/.github/workflows/ci_python.yml index 94f02dc..64e30ed 100644 --- a/.github/workflows/ci_python.yml +++ b/.github/workflows/ci_python.yml @@ -2,9 +2,9 @@ name: Test Python on: push: - branches: [ master, development ] + branches: [ master, development, experimental, test* ] pull_request: - branches: [ master, development ] + branches: [ master, development, experimental, test* ] jobs: build: @@ -13,13 +13,13 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] + python-version: ["3.9", "3.10", "3.11"] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} From b00c17166c9e06d6823d25e41814f75bad57da7c Mon Sep 17 00:00:00 2001 From: pgleeson Date: Thu, 20 Mar 2025 16:07:43 +0000 Subject: [PATCH 2/8] Test unpin scipy --- src/Python/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Python/setup.py b/src/Python/setup.py index 34be5cf..ba34cb1 100644 --- a/src/Python/setup.py +++ b/src/Python/setup.py @@ -52,7 +52,7 @@ keywords='C. elegans worm tracking', packages=['wcon'], package_data={'': ['../../wcon_schema.json']}, - install_requires=['jsonschema', 'six', 'numpy', 'scipy<=0.17.1'] + install_requires=['jsonschema', 'six', 'scipy', 'pandas'] # Actually also requires numpy, scipy and numpy but I don't want to force # pip to install these since pip is bad at that for those packages. ) From b19292bc388ac4b32cd7bb3ff7228424cfab6196 Mon Sep 17 00:00:00 2001 From: Padraig Gleeson Date: Wed, 23 Apr 2025 17:36:54 +0100 Subject: [PATCH 3/8] Update ci_python.yml --- .github/workflows/ci_python.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_python.yml b/.github/workflows/ci_python.yml index 64e30ed..c24ee59 100644 --- a/.github/workflows/ci_python.yml +++ b/.github/workflows/ci_python.yml @@ -13,7 +13,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.9", "3.10", "3.11"] + python-version: ["3.9", "3.10", "3.11", "3.12"] steps: From 0e4febd8336fb7045098fb09612ac05b9792a729 Mon Sep 17 00:00:00 2001 From: Padraig Gleeson Date: Tue, 28 Apr 2026 21:11:50 +0100 Subject: [PATCH 4/8] Update gha versions --- .github/workflows/ci_python.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci_python.yml b/.github/workflows/ci_python.yml index c24ee59..861f362 100644 --- a/.github/workflows/ci_python.yml +++ b/.github/workflows/ci_python.yml @@ -17,9 +17,9 @@ jobs: steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} From a86449ab640d329c2eb2ad3ccaebeccca583d1bb Mon Sep 17 00:00:00 2001 From: Padraig Gleeson Date: Tue, 28 Apr 2026 21:44:41 +0100 Subject: [PATCH 5/8] Some panda modifications --- .gitignore | 2 ++ src/Python/examples/view_wcon.py | 2 +- src/Python/setup.py | 12 ++++++++++-- src/Python/wcon/wcon_data.py | 32 +++++++++++++++++++++++--------- src/Python/wcon/wcon_parser.py | 23 ++++++++++++++++------- 5 files changed, 52 insertions(+), 19 deletions(-) diff --git a/.gitignore b/.gitignore index 807e483..72ba9b5 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,5 @@ src/Python/wcon.egg* # Scala src/scala/target/* +/src/Python/example_saved_file.WCON +/src/Python/wcon/wcon_schema.json diff --git a/src/Python/examples/view_wcon.py b/src/Python/examples/view_wcon.py index a5d836f..3d10c4a 100644 --- a/src/Python/examples/view_wcon.py +++ b/src/Python/examples/view_wcon.py @@ -4,5 +4,5 @@ sys.path.append('..') from wcon import WCONWorms, MeasurementUnit -file_name = 'asic-1 (ok415) on food L_2010_07_08__11_46_40___7___5.wcon' +file_name = '../../../tests/minimax.wcon' w = WCONWorms.load_from_file(file_name) diff --git a/src/Python/setup.py b/src/Python/setup.py index ba34cb1..e2bca5a 100644 --- a/src/Python/setup.py +++ b/src/Python/setup.py @@ -12,6 +12,7 @@ from codecs import open from os import path import os +import shutil exec(open('wcon/version.py').read()) here = path.abspath(path.dirname(__file__)) @@ -24,7 +25,13 @@ with open(readme_path, encoding='utf-8') as f: long_description += f.read() -print(os.listdir('.')) # DEBUG +# The canonical wcon_schema.json lives at the repository root so it can be +# shared by every language implementation. setuptools cannot package files +# from outside the package directory, so copy it into wcon/ at build time. +repo_schema = path.join(here, '..', '..', 'wcon_schema.json') +pkg_schema = path.join(here, 'wcon', 'wcon_schema.json') +if path.exists(repo_schema): + shutil.copyfile(repo_schema, pkg_schema) setup( name='wcon', @@ -51,7 +58,8 @@ ], keywords='C. elegans worm tracking', packages=['wcon'], - package_data={'': ['../../wcon_schema.json']}, + package_data={'wcon': ['wcon_schema.json']}, + include_package_data=True, install_requires=['jsonschema', 'six', 'scipy', 'pandas'] # Actually also requires numpy, scipy and numpy but I don't want to force # pip to install these since pip is bad at that for those packages. diff --git a/src/Python/wcon/wcon_data.py b/src/Python/wcon/wcon_data.py index 7ee574e..ebcf9a9 100644 --- a/src/Python/wcon/wcon_data.py +++ b/src/Python/wcon/wcon_data.py @@ -119,6 +119,13 @@ def df_upsert(src, dest): dest_sliced.sort_index(inplace=True) src_sliced.sort_index(inplace=True) + # Align src_sliced's row/column labels to dest_sliced. The two + # were built with independent .isin() masks so column order may + # differ; pandas >=1.x refuses to compare DataFrames whose + # labels are not identical. + src_sliced = src_sliced.reindex(index=dest_sliced.index, + columns=dest_sliced.columns) + # Obtain a mask of the conflicts in the current segment # as compared with all previously loaded data. That is: # NaN NaN = False @@ -189,24 +196,30 @@ def convert_origin(df): # `for` loop loops through both `x` and `y`. if offset in cur_worm.columns.get_level_values(0): - # Consider offset as 0 if not available in a certain frame - ox_column = cur_worm.loc[:, (offset)].fillna(0) + # Consider offset as 0 if not available in a certain frame. + # Coerce to numeric: the parser can leave the offset column + # with object dtype (mixed str/int entries) when offsets + # are present in some segments but not others. + ox_column = cur_worm.loc[:, (offset)].apply( + pd.to_numeric, errors='coerce').fillna(0) # Shift our 'x' values by offset - all_x_columns = cur_worm.loc[:, (coord)] - ox_affine_change = (np.array(ox_column) * + all_x_columns = cur_worm.loc[:, (coord)].apply( + pd.to_numeric, errors='coerce') + ox_affine_change = (np.array(ox_column, dtype=float) * np.ones(all_x_columns.shape)) all_x_columns += ox_affine_change if centroid in cur_worm.columns.get_level_values(0): - cx_column = cur_worm.loc[:, (centroid)] + cx_column = cur_worm.loc[:, (centroid)].apply( + pd.to_numeric, errors='coerce') # Shift the centroid by the offset cx_column += ox_column # Now make the centroid our new offset, since the rule # is that if the offset exists, the centroid is not # the offset, but we want it to be. - cx_affine_change = (np.array(cx_column) * + cx_affine_change = (np.array(cx_column, dtype=float) * np.ones(all_x_columns.shape)) all_x_columns -= cx_affine_change @@ -224,7 +237,8 @@ def convert_origin(df): # This is so DataFrames with and without offsets # will show as comparing identically. for offset_key in offset_keys: - df.drop(offset_key, axis=1, level='key', inplace=True) + df.drop(offset_key, axis=1, level='key', inplace=True, + errors='ignore') # Because of a known issue in Pandas # (https://github.com/pydata/pandas/issues/2770), the dropped columns @@ -402,7 +416,7 @@ def _obtain_time_series_data_frame(time_series_data): cur_df = pd.DataFrame(cur_data, columns=cur_columns) cur_df.index = cur_timeframes - cur_df.index.names = 't' + cur_df.index.names = ['t'] # We want the index (time) to be in order. cur_df.sort_index(axis=0, inplace=True) @@ -466,7 +480,7 @@ def _obtain_time_series_data_frame(time_series_data): with warnings.catch_warnings(): warnings.filterwarnings(action="ignore", category=FutureWarning) df_odict[worm_id] = \ - df_odict[worm_id].convert_objects(convert_numeric=True) + df_odict[worm_id].infer_objects() # If 'head' or 'ventral' is NaN, we must specify '?' since # otherwise, when saving this object, to specify "no value" we would diff --git a/src/Python/wcon/wcon_parser.py b/src/Python/wcon/wcon_parser.py index 13010f6..135320a 100644 --- a/src/Python/wcon/wcon_parser.py +++ b/src/Python/wcon/wcon_parser.py @@ -385,12 +385,8 @@ def to_canon(self): for data_key in self.units: mu = self.units[data_key] - # Don't bother to "convert" units that are already in their - # canonical form. - if mu.unit_string == mu.canonical_unit_string: - continue - tmu = self.units['t'] + already_canonical = (mu.unit_string == mu.canonical_unit_string) for worm_id in w.worm_ids: try: @@ -398,8 +394,21 @@ def to_canon(self): mu_slice = \ w._data[worm_id].loc[:, idx[:, data_key, :]].copy() - w._data[worm_id].loc[:, idx[:, data_key, :]] = \ - mu_slice.applymap(mu.to_canon) + # The parser can leave numeric columns with object + # dtype (e.g. mixed int/str entries from how segments + # are merged). Coerce so downstream arithmetic and + # JSON serialization treat them as numbers, even when + # the unit is already canonical and no conversion is + # otherwise required. + mu_slice = mu_slice.apply(pd.to_numeric, + errors='coerce') + + if already_canonical: + w._data[worm_id].loc[:, idx[:, data_key, :]] = \ + mu_slice + else: + w._data[worm_id].loc[:, idx[:, data_key, :]] = \ + mu_slice.applymap(mu.to_canon) except KeyError: # Just ignore cases where there are "units" entries but no # corresponding data From 7a4e2bf334e7af10771062f719a987668d05e1b1 Mon Sep 17 00:00:00 2001 From: Padraig Gleeson Date: Wed, 29 Apr 2026 08:53:00 +0100 Subject: [PATCH 6/8] More fixes for latest pandas --- src/Python/tests/tests.py | 2 +- src/Python/wcon/measurement_unit.py | 9 ++++++--- src/Python/wcon/wcon_data.py | 2 +- src/Python/wcon/wcon_parser.py | 29 +++++++++++++++-------------- 4 files changed, 23 insertions(+), 19 deletions(-) diff --git a/src/Python/tests/tests.py b/src/Python/tests/tests.py index 545fbb8..73c3ab6 100644 --- a/src/Python/tests/tests.py +++ b/src/Python/tests/tests.py @@ -41,7 +41,7 @@ def flatten(list_of_lists): for element in list_of_lists: # If it's iterable but not a string or bytes, then recurse, otherwise # we are at a "leaf" node of our traversal - if(isinstance(element, collections.Iterable) and + if(isinstance(element, collections.abc.Iterable) and not isinstance(element, (str, bytes))): for sub_element in flatten(element): yield sub_element diff --git a/src/Python/wcon/measurement_unit.py b/src/Python/wcon/measurement_unit.py index 675184c..237fac2 100644 --- a/src/Python/wcon/measurement_unit.py +++ b/src/Python/wcon/measurement_unit.py @@ -637,12 +637,15 @@ def _create_from_atomic(cls, unit_string): @classmethod def _create_from_node(cls, node): """ - node: is ast.Num or ast.BinOp or ast.UnaryOp or ast.Str or ast.Name + node: is ast.Constant (numeric) or ast.BinOp or ast.UnaryOp or ast.Name The expression to be transformed into a MeasurementUnit """ - if isinstance(node, ast.Num): # - n = node.n + # ast.Num was deprecated in Python 3.8 and removed in 3.14; + # ast.Constant now represents all literal values. + if isinstance(node, ast.Constant) and isinstance( + node.value, (int, float)): # + n = node.value assert(n != 0) # A unit cannot have zero in the expression u = cls() diff --git a/src/Python/wcon/wcon_data.py b/src/Python/wcon/wcon_data.py index ebcf9a9..198df74 100644 --- a/src/Python/wcon/wcon_data.py +++ b/src/Python/wcon/wcon_data.py @@ -403,7 +403,7 @@ def _obtain_time_series_data_frame(time_series_data): for i in range(len(cur_timeframes)): data_segment[k][i] = ( data_segment[k][i] + - [np.NaN] * (max_aspect_size - len(data_segment[k][i]))) + [np.nan] * (max_aspect_size - len(data_segment[k][i]))) num_timeframes = len(cur_timeframes) diff --git a/src/Python/wcon/wcon_parser.py b/src/Python/wcon/wcon_parser.py index 135320a..916daf8 100644 --- a/src/Python/wcon/wcon_parser.py +++ b/src/Python/wcon/wcon_parser.py @@ -390,25 +390,25 @@ def to_canon(self): for worm_id in w.worm_ids: try: - # Apply across all worm ids and all aspects - mu_slice = \ - w._data[worm_id].loc[:, idx[:, data_key, :]].copy() + df = w._data[worm_id] + target_cols = [c for c in df.columns + if c[1] == data_key] + if not target_cols: + raise KeyError(data_key) # The parser can leave numeric columns with object # dtype (e.g. mixed int/str entries from how segments # are merged). Coerce so downstream arithmetic and # JSON serialization treat them as numbers, even when # the unit is already canonical and no conversion is - # otherwise required. - mu_slice = mu_slice.apply(pd.to_numeric, - errors='coerce') - - if already_canonical: - w._data[worm_id].loc[:, idx[:, data_key, :]] = \ - mu_slice - else: - w._data[worm_id].loc[:, idx[:, data_key, :]] = \ - mu_slice.applymap(mu.to_canon) + # otherwise required. Replace each column whole rather + # than via .loc[] assignment, which would preserve the + # parent column's existing (object) dtype. + for col in target_cols: + new_col = pd.to_numeric(df[col], errors='coerce') + if not already_canonical: + new_col = new_col.apply(mu.to_canon) + df[col] = new_col except KeyError: # Just ignore cases where there are "units" entries but no # corresponding data @@ -831,7 +831,8 @@ def pd_equals(df1, df2): return False try: - pd.util.testing.assert_frame_equal(df1, df2) + # pd.util.testing was removed in pandas 2.0; use pd.testing. + pd.testing.assert_frame_equal(df1, df2) except AssertionError: return False From d46814f0e9058b74f0bff0f59081b2ae4ca07a00 Mon Sep 17 00:00:00 2001 From: Padraig Gleeson Date: Wed, 29 Apr 2026 08:53:08 +0100 Subject: [PATCH 7/8] To v1.2.1 --- src/Python/wcon/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Python/wcon/version.py b/src/Python/wcon/version.py index 4cd8613..d5b90cd 100644 --- a/src/Python/wcon/version.py +++ b/src/Python/wcon/version.py @@ -6,4 +6,4 @@ # 2) we can import it in setup.py for the same reason # 3) we can import it into your module module # (from http://stackoverflow.com/questions/458550/) -__version__ = '1.1.0' +__version__ = '1.2.1' From ed56ade0985056cff79f7b0ce4a15c2100dc5077 Mon Sep 17 00:00:00 2001 From: Padraig Gleeson Date: Wed, 29 Apr 2026 08:54:59 +0100 Subject: [PATCH 8/8] Run tests --- .github/workflows/ci_python.yml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci_python.yml b/.github/workflows/ci_python.yml index 861f362..7be361e 100644 --- a/.github/workflows/ci_python.yml +++ b/.github/workflows/ci_python.yml @@ -31,6 +31,16 @@ jobs: - name: Install Python package run: | cd src/Python - python setup.py install + pip install . + + - name: Run tests + run: | + cd src/Python/tests + python diagnostic_test.py ../../../tests/minimax.wcon + python tests.py + + - name: Final version info + run: | + pip list