Respect sparse_threshold for pandas SparseDtype columns

CHANGELOG.rst

@@ -10,6 +10,10 @@ Changelog
 4.2.2 - unreleased
 ------------------
 
+**Bug fixes:**
+
+- :func:`tabmat.from_df` now respects ``sparse_threshold`` when classifying pandas ``SparseDtype`` columns, including columns with non-zero fill values.
+
 **Other changes:**
 
 - We disabled fast math to avoid invalid results (e.g., when dividing by zero).

src/tabmat/constructor.py

@@ -105,9 +105,28 @@ def from_df(
 
         # deal with Pandas sparse dtype (not supported by narwhals)
         if pd is not None:
-            if isinstance(nw.to_native(coldata).dtype, pd.SparseDtype):
-                sparse_dfidx.append(dfcolidx)
-                sparse_tmidx.append(mxcolidx)
+            native_coldata = nw.to_native(coldata)
+            if isinstance(native_coldata.dtype, pd.SparseDtype):
+                fill_value = native_coldata.sparse.fill_value
+                if len(native_coldata) == 0 or pd.isna(fill_value):
+                    sparse_dfidx.append(dfcolidx)
+                    sparse_tmidx.append(mxcolidx)
+                else:
+                    if fill_value == 0:
+                        # With zero fill, logical density cannot exceed storage density.
+                        density = native_coldata.sparse.density
+                        if density > sparse_threshold:
+                            # Explicitly stored zeros may lower the logical density.
+                            density = (native_coldata != 0).mean()
+                    else:
+                        density = (native_coldata != 0).mean()
+
+                    if density <= sparse_threshold:
+                        sparse_dfidx.append(dfcolidx)
+                        sparse_tmidx.append(mxcolidx)
+                    else:
+                        dense_dfidx.append(dfcolidx)
+                        dense_tmidx.append(mxcolidx)
                 mxcolidx += 1
                 continue
 

tests/test_constructor.py

@@ -68,6 +68,122 @@ def test_pandas_to_matrix():
     assert df["ds"].dtype == original_dtypes["ds"]
 
 
+@pytest.mark.parametrize(
+    "values,sparse_threshold,expected_type",
+    [
+        pytest.param(
+            [1.0] + [0.0] * 9,
+            0.2,
+            tm.SparseMatrix,
+            id="density-below-threshold",
+        ),
+        pytest.param(
+            [1.0] * 9 + [0.0],
+            0.5,
+            tm.DenseMatrix,
+            id="density-above-threshold",
+        ),
+        pytest.param(
+            [1.0, 2.0, 3.0] + [0.0] * 7,
+            0.3,
+            tm.SparseMatrix,
+            id="density-equals-threshold",
+        ),
+    ],
+)
+def test_pandas_sparse_dtype_respects_sparse_threshold(
+    values, sparse_threshold, expected_type
+):
+    df = pd.DataFrame(
+        {"sparse": pd.Series(values, dtype=pd.SparseDtype("float", fill_value=0.0))}
+    )
+
+    mat = tm.from_df(df, sparse_threshold=sparse_threshold)
+
+    assert isinstance(mat, expected_type)
+    np.testing.assert_array_equal(mat.toarray(), df.to_numpy())
+
+
+def test_pandas_sparse_dtype_with_explicit_zeros():
+    df = pd.DataFrame(
+        {
+            "col_a": [1.0, 0.0] * 5,
+            "col_b": [0.0, 1.0] * 5,
+        }
+    ).astype(pd.SparseDtype("float", fill_value=0.0))
+    col_c = df["col_a"] * df["col_b"]
+    df = pd.DataFrame({"col_c": col_c})
+    sparse_threshold = 0.5
+
+    assert col_c.sparse.density > sparse_threshold
+    assert (col_c != 0).mean() == 0
+
+    mat = tm.from_df(df, sparse_threshold=sparse_threshold)
+
+    assert isinstance(mat, tm.SparseMatrix)
+    np.testing.assert_array_equal(mat.toarray(), df.to_numpy())
+
+
+@pytest.mark.parametrize(
+    "values,expected_storage_density,expected_logical_density,expected_type",
+    [
+        pytest.param(
+            [5.0, 5.0, 5.0, 0.0],
+            0.25,
+            0.75,
+            tm.DenseMatrix,
+            id="storage-density-below-logical-density-above",
+        ),
+        pytest.param(
+            [0.0, 0.0, 0.0, 5.0],
+            0.75,
+            0.25,
+            tm.SparseMatrix,
+            id="storage-density-above-logical-density-below",
+        ),
+    ],
+)
+def test_pandas_sparse_dtype_with_nonzero_fill_value(
+    values,
+    expected_storage_density,
+    expected_logical_density,
+    expected_type,
+):
+    series = pd.Series(values, dtype=pd.SparseDtype("float", fill_value=5.0))
+    df = pd.DataFrame({"sparse": series})
+    sparse_threshold = 0.5
+
+    assert series.sparse.density == expected_storage_density
+    assert (series != 0).mean() == expected_logical_density
+
+    mat = tm.from_df(df, sparse_threshold=sparse_threshold)
+
+    assert isinstance(mat, expected_type)
+    np.testing.assert_array_equal(mat.toarray(), df.to_numpy())
+
+
+@pytest.mark.parametrize("fill_value", [pd.NA, np.nan])
+def test_pandas_sparse_dtype_with_missing_fill_value(fill_value):
+    series = pd.Series([0.0, 1.0], dtype=pd.SparseDtype("float", fill_value))
+    df = pd.DataFrame({"sparse": series})
+
+    mat = tm.from_df(df, sparse_threshold=0.1)
+
+    assert isinstance(mat, tm.SparseMatrix)
+    np.testing.assert_array_equal(mat.toarray(), df.to_numpy())
+
+
+def test_empty_pandas_sparse_dtype():
+    series = pd.Series([], dtype=pd.SparseDtype("float", fill_value=0.0))
+    df = pd.DataFrame({"sparse": series})
+
+    mat = tm.from_df(df, sparse_threshold=0.1)
+
+    assert isinstance(mat, tm.SparseMatrix)
+    assert mat.shape == (0, 1)
+    np.testing.assert_array_equal(mat.toarray(), df.to_numpy())
+
+
 @pytest.mark.parametrize("categorical_dtype", [pl.Categorical, pl.Enum(["a", "b"])])
 def test_polars_to_matrix(categorical_dtype):
     df = construct_data("polars").with_columns(cl=pl.col("cl").cast(categorical_dtype))