From 5e14cc83bc1e9f4a2c9cb9f45f8562a7d72a8d04 Mon Sep 17 00:00:00 2001 From: Danila Pechenev Date: Thu, 3 Sep 2026 23:09:19 +0200 Subject: [PATCH 1/2] test: cover pandas sparse density in from_df --- tests/test_constructor.py | 116 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 116 insertions(+) diff --git a/tests/test_constructor.py b/tests/test_constructor.py index 6260f1b9..e1b4801c 100644 --- a/tests/test_constructor.py +++ b/tests/test_constructor.py @@ -68,6 +68,122 @@ def test_pandas_to_matrix(): assert df["ds"].dtype == original_dtypes["ds"] +@pytest.mark.parametrize( + "values,sparse_threshold,expected_type", + [ + pytest.param( + [1.0] + [0.0] * 9, + 0.2, + tm.SparseMatrix, + id="density-below-threshold", + ), + pytest.param( + [1.0] * 9 + [0.0], + 0.5, + tm.DenseMatrix, + id="density-above-threshold", + ), + pytest.param( + [1.0, 2.0, 3.0] + [0.0] * 7, + 0.3, + tm.SparseMatrix, + id="density-equals-threshold", + ), + ], +) +def test_pandas_sparse_dtype_respects_sparse_threshold( + values, sparse_threshold, expected_type +): + df = pd.DataFrame( + {"sparse": pd.Series(values, dtype=pd.SparseDtype("float", fill_value=0.0))} + ) + + mat = tm.from_df(df, sparse_threshold=sparse_threshold) + + assert isinstance(mat, expected_type) + np.testing.assert_array_equal(mat.toarray(), df.to_numpy()) + + +def test_pandas_sparse_dtype_with_explicit_zeros(): + df = pd.DataFrame( + { + "col_a": [1.0, 0.0] * 5, + "col_b": [0.0, 1.0] * 5, + } + ).astype(pd.SparseDtype("float", fill_value=0.0)) + col_c = df["col_a"] * df["col_b"] + df = pd.DataFrame({"col_c": col_c}) + sparse_threshold = 0.5 + + assert col_c.sparse.density > sparse_threshold + assert (col_c != 0).mean() == 0 + + mat = tm.from_df(df, sparse_threshold=sparse_threshold) + + assert isinstance(mat, tm.SparseMatrix) + np.testing.assert_array_equal(mat.toarray(), df.to_numpy()) + + +@pytest.mark.parametrize( + "values,expected_storage_density,expected_logical_density,expected_type", + [ + pytest.param( + [5.0, 5.0, 5.0, 0.0], + 0.25, + 0.75, + tm.DenseMatrix, + id="storage-density-below-logical-density-above", + ), + pytest.param( + [0.0, 0.0, 0.0, 5.0], + 0.75, + 0.25, + tm.SparseMatrix, + id="storage-density-above-logical-density-below", + ), + ], +) +def test_pandas_sparse_dtype_with_nonzero_fill_value( + values, + expected_storage_density, + expected_logical_density, + expected_type, +): + series = pd.Series(values, dtype=pd.SparseDtype("float", fill_value=5.0)) + df = pd.DataFrame({"sparse": series}) + sparse_threshold = 0.5 + + assert series.sparse.density == expected_storage_density + assert (series != 0).mean() == expected_logical_density + + mat = tm.from_df(df, sparse_threshold=sparse_threshold) + + assert isinstance(mat, expected_type) + np.testing.assert_array_equal(mat.toarray(), df.to_numpy()) + + +@pytest.mark.parametrize("fill_value", [pd.NA, np.nan]) +def test_pandas_sparse_dtype_with_missing_fill_value(fill_value): + series = pd.Series([0.0, 1.0], dtype=pd.SparseDtype("float", fill_value)) + df = pd.DataFrame({"sparse": series}) + + mat = tm.from_df(df, sparse_threshold=0.1) + + assert isinstance(mat, tm.SparseMatrix) + np.testing.assert_array_equal(mat.toarray(), df.to_numpy()) + + +def test_empty_pandas_sparse_dtype(): + series = pd.Series([], dtype=pd.SparseDtype("float", fill_value=0.0)) + df = pd.DataFrame({"sparse": series}) + + mat = tm.from_df(df, sparse_threshold=0.1) + + assert isinstance(mat, tm.SparseMatrix) + assert mat.shape == (0, 1) + np.testing.assert_array_equal(mat.toarray(), df.to_numpy()) + + @pytest.mark.parametrize("categorical_dtype", [pl.Categorical, pl.Enum(["a", "b"])]) def test_polars_to_matrix(categorical_dtype): df = construct_data("polars").with_columns(cl=pl.col("cl").cast(categorical_dtype)) From cd7383a327e9456f3371a02ed5c93c6d81e16cfc Mon Sep 17 00:00:00 2001 From: Danila Pechenev Date: Fri, 4 Sep 2026 15:04:58 +0200 Subject: [PATCH 2/2] fix: respect sparse threshold for pandas sparse columns --- CHANGELOG.rst | 4 ++++ src/tabmat/constructor.py | 25 ++++++++++++++++++++++--- 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 9de6fbaa..b7608e82 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -10,6 +10,10 @@ Changelog 4.2.2 - unreleased ------------------ +**Bug fixes:** + +- :func:`tabmat.from_df` now respects ``sparse_threshold`` when classifying pandas ``SparseDtype`` columns, including columns with non-zero fill values. + **Other changes:** - We disabled fast math to avoid invalid results (e.g., when dividing by zero). diff --git a/src/tabmat/constructor.py b/src/tabmat/constructor.py index 6dfca30a..890b4098 100644 --- a/src/tabmat/constructor.py +++ b/src/tabmat/constructor.py @@ -105,9 +105,28 @@ def from_df( # deal with Pandas sparse dtype (not supported by narwhals) if pd is not None: - if isinstance(nw.to_native(coldata).dtype, pd.SparseDtype): - sparse_dfidx.append(dfcolidx) - sparse_tmidx.append(mxcolidx) + native_coldata = nw.to_native(coldata) + if isinstance(native_coldata.dtype, pd.SparseDtype): + fill_value = native_coldata.sparse.fill_value + if len(native_coldata) == 0 or pd.isna(fill_value): + sparse_dfidx.append(dfcolidx) + sparse_tmidx.append(mxcolidx) + else: + if fill_value == 0: + # With zero fill, logical density cannot exceed storage density. + density = native_coldata.sparse.density + if density > sparse_threshold: + # Explicitly stored zeros may lower the logical density. + density = (native_coldata != 0).mean() + else: + density = (native_coldata != 0).mean() + + if density <= sparse_threshold: + sparse_dfidx.append(dfcolidx) + sparse_tmidx.append(mxcolidx) + else: + dense_dfidx.append(dfcolidx) + dense_tmidx.append(mxcolidx) mxcolidx += 1 continue