From ef8a8820666ebd1bc1a52440f3414b9bccb1a65f Mon Sep 17 00:00:00 2001 From: Aaron Meyer Date: Fri, 18 Sep 2026 08:37:50 -0700 Subject: [PATCH 1/3] Add CLAUDE.md maintainability-check guidance and fix flagged code smells Adapted from cccRISE's CLAUDE.md: adds vulture/radon/xenon to the dev dependency group and documents how to run dead-code, duplication, and cyclomatic-complexity checks against src/vsparse/. Running those checks surfaced and fixed: - _base.py: removed _VCSBase._major_range, a dead method with no callers anywhere in the repo (including tests). - _ivcsc.py/_rapid_load.py: extracted the repeated varint-decode loop into a shared _decode_varint njit helper, reused across _unpack, _decode_chunks, and _decode_selected_rows. - _anndata_class.py: extracted the duplicated X/raw_X coercion logic in VCSCAnnData's setters into a shared _coerce_vcs helper. The remaining jscpd-flagged duplication is the major=row/major=col pair of numba kernels repeated throughout _norm_common.py, _vcs_matmul.py, and _ops.py -- an intentional, documented pattern for cache-friendly parallel iteration over each array's own memory layout, not a code smell. Co-Authored-By: Claude Sonnet 5 --- CLAUDE.md | 58 +++++++++++++++++++++++++++++++++++ pyproject.toml | 3 ++ src/vsparse/_anndata_class.py | 26 ++++++++-------- src/vsparse/_base.py | 18 ----------- src/vsparse/_ivcsc.py | 35 ++++++++++----------- src/vsparse/_rapid_load.py | 10 +----- uv.lock | 54 ++++++++++++++++++++++++++++++++ 7 files changed, 146 insertions(+), 58 deletions(-) create mode 100644 CLAUDE.md diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..d9f8191 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,58 @@ +# CLAUDE.md + +Guidance for AI coding agents (and human contributors) working in this repository. + +## Maintainability checks for new/changed code + +This project's dev dependency group (`uv sync --group dev`) includes several +static-analysis tools that are not wired into pre-commit/CI, so they won't +run automatically. Whenever you add or substantially modify Python code in +`src/vsparse/`, run the relevant tool(s) below yourself and address anything +they flag before considering the change done. + +### Dead code -- vulture + +```sh +uv run vulture src/vsparse/ +``` + +Flags functions, variables, and imports that appear to be unused. Before +deleting a reported item, confirm with `grep`/`git grep` that it really has +no callers (vulture's confidence score is a heuristic, and a few things -- +e.g. manually-invoked one-off scripts -- are meant to have no in-repo +callers). If something is a deliberate exception, prefer leaving a short +comment explaining why over silencing the tool. + +### Duplicated code -- jscpd + +```sh +npx jscpd src/vsparse/ --min-lines 5 --min-tokens 50 +``` + +(No install needed beyond Node/npx; jscpd is not a Python dependency.) Flags +copy-pasted blocks. If a new module duplicates an existing block of 10+ +lines, prefer extracting a shared helper instead of copy-pasting. + +### Cyclomatic complexity -- radon + xenon + +```sh +uv run radon cc src/vsparse/ -n C -s # list functions ranked C or worse +uv run radon mi src/vsparse/ -s # maintainability index per file +uv run xenon --max-absolute B --max-modules A --max-average A src/vsparse/ +``` + +`xenon` exits non-zero and prints every function/module exceeding the given +rank thresholds (A best -- F worst). Treat a new function ranked C or worse +as a signal to break it up: extract the branchy/loop-heavy interior into one +or more named helper functions (as opposed to introducing more parameters or +flags to the same function). A handful of pre-existing functions still +exceed these thresholds; it's fine to leave those alone unless you're +already modifying them, but don't add new ones. + +## Why these aren't pre-commit hooks + +`ruff` and `codespell` are fast and have an unambiguous pass/fail; they run +on every commit. `vulture`, `jscpd`, and `xenon` are noisier and require +judgment calls (a flagged function may be an intentional exception), so +they're kept as tools to run and reason about manually rather than hard +commit gates. diff --git a/pyproject.toml b/pyproject.toml index 3b8a2e4..7050c08 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,6 +47,9 @@ dev = [ "hypothesis>=6.100", "codespell>=2.3", "pytest-memray>=1.11.0", + "vulture>=2.16", + "radon>=6.0.1", + "xenon>=0.9.3", ] [build-system] diff --git a/src/vsparse/_anndata_class.py b/src/vsparse/_anndata_class.py index ee284b0..f063945 100644 --- a/src/vsparse/_anndata_class.py +++ b/src/vsparse/_anndata_class.py @@ -70,6 +70,17 @@ def _subset_2d(v: Any, oidx: Any, vidx: Any) -> Any: return np.asarray(v)[oidx][:, vidx] +def _coerce_vcs(value: Any, name: str) -> Any: + """Coerce a raw scipy/ndarray ``X``/``raw_X`` value to VCSC/VCSR, validating anything else.""" + if value is not None and not isinstance(value, _VCS_TYPES): + if sp.issparse(value) or isinstance(value, np.ndarray): + vcls = VCSCArray if isinstance(value, sp.csc_array | sp.csc_matrix) else VCSRArray + value = vcls.from_scipy(value) + else: + _check_vcs_type(value, name) + return value + + def _copy_value(v: Any) -> Any: """A deep-enough copy of one obs/var/obsm/varm/obsp/varp/layers value. @@ -146,12 +157,7 @@ def X(self) -> _AnyVCS | None: @X.setter def X(self, value: Any) -> None: - if value is not None and not isinstance(value, _VCS_TYPES): - if sp.issparse(value) or isinstance(value, np.ndarray): - vcls = VCSCArray if isinstance(value, sp.csc_array | sp.csc_matrix) else VCSRArray - value = vcls.from_scipy(value) - else: - _check_vcs_type(value, "X") + value = _coerce_vcs(value, "X") if ( value is not None and hasattr(self, "_obs") @@ -168,13 +174,7 @@ def raw_X(self) -> _AnyVCS | None: @raw_X.setter def raw_X(self, value: Any) -> None: - if value is not None and not isinstance(value, _VCS_TYPES): - if sp.issparse(value) or isinstance(value, np.ndarray): - vcls = VCSCArray if isinstance(value, sp.csc_array | sp.csc_matrix) else VCSRArray - value = vcls.from_scipy(value) - else: - _check_vcs_type(value, "raw_X") - self._vcs_raw_X = value + self._vcs_raw_X = _coerce_vcs(value, "raw_X") # -- indexing / view creation --------------------------------------------- diff --git a/src/vsparse/_base.py b/src/vsparse/_base.py index 001266c..c6cde21 100644 --- a/src/vsparse/_base.py +++ b/src/vsparse/_base.py @@ -551,24 +551,6 @@ def _select_major(self, key: Any) -> _VCSBase: new_shape = (n_minor, idx.shape[0]) if self._format == "csc" else (idx.shape[0], n_minor) return type(self)(new_shape, new_major_ptr, new_values, new_value_ptr, new_indices) - def _major_range(self, start: int, stop: int) -> _VCSBase: - """The contiguous major-slice range ``[start, stop)``, without copying values. - - ``values``/``indices`` come back as views; only the two pointer - arrays are rebuilt, rebased to the new start. - """ - u0, u1 = int(self.major_ptr[start]), int(self.major_ptr[stop]) - k0, k1 = int(self.value_ptr[u0]), int(self.value_ptr[u1]) - n_sel = stop - start - new_shape = (self.n_minor, n_sel) if self._format == "csc" else (n_sel, self.n_minor) - return type(self)( - new_shape, - self.major_ptr[start : stop + 1] - u0, - self.values[u0:u1], - self.value_ptr[u0 : u1 + 1] - k0, - self.indices[k0:k1], - ) - def _select_minor(self, key: Any) -> _VCSBase: """Select along the minor axis (rows for VCSC, columns for VCSR). diff --git a/src/vsparse/_ivcsc.py b/src/vsparse/_ivcsc.py index c779d94..43859b7 100644 --- a/src/vsparse/_ivcsc.py +++ b/src/vsparse/_ivcsc.py @@ -77,6 +77,21 @@ def _pack(value_ptr: np.ndarray, indices: np.ndarray) -> np.ndarray: return buf[:pos].copy() +@numba.njit(cache=True) +def _decode_varint(buf: np.ndarray, pos: np.int64) -> tuple[np.uint64, np.int64]: + """Decode one varint starting at ``pos``; returns ``(value, next_pos)``.""" + shift = np.uint64(0) + result = np.uint64(0) + while True: + b = buf[pos] + pos += 1 + result |= np.uint64(b & 0x7F) << shift + if b & 0x80 == 0: + break + shift += np.uint64(7) + return result, pos + + @numba.njit(cache=True) def _unpack(value_ptr: np.ndarray, buf: np.ndarray, out: np.ndarray) -> None: n_groups = value_ptr.shape[0] - 1 @@ -85,15 +100,7 @@ def _unpack(value_ptr: np.ndarray, buf: np.ndarray, out: np.ndarray) -> None: start, end = value_ptr[g], value_ptr[g + 1] prev = np.int64(-1) for k in range(start, end): - shift = np.uint64(0) - result = np.uint64(0) - while True: - b = buf[pos] - pos += 1 - result |= np.uint64(b & 0x7F) << shift - if b & 0x80 == 0: - break - shift += np.uint64(7) + result, pos = _decode_varint(buf, pos) prev = prev + 1 + np.int64(result) out[k] = prev @@ -177,15 +184,7 @@ def _decode_chunks( start, end = value_ptr[g], value_ptr[g + 1] prev = np.int64(-1) for kk in range(start, end): - shift = np.uint64(0) - result = np.uint64(0) - while True: - b = buf[pos] - pos += 1 - result |= np.uint64(b & 0x7F) << shift - if b & 0x80 == 0: - break - shift += np.uint64(7) + result, pos = _decode_varint(buf, pos) prev = prev + 1 + np.int64(result) out[kk] = prev diff --git a/src/vsparse/_rapid_load.py b/src/vsparse/_rapid_load.py index b5e12c0..e8b7b12 100644 --- a/src/vsparse/_rapid_load.py +++ b/src/vsparse/_rapid_load.py @@ -130,15 +130,7 @@ def _decode_selected_rows( prev = np.int64(-1) value = values[g] for _ in range(value_ptr[g], value_ptr[g + 1]): - shift = np.uint64(0) - result = np.uint64(0) - while True: - b = packed[pos] - pos += 1 - result |= np.uint64(b & 0x7F) << shift - if b & 0x80 == 0: - break - shift += np.uint64(7) + result, pos = _ivcsc._decode_varint(packed, pos) prev = prev + 1 + np.int64(result) out_indices[out_pos] = prev out_data[out_pos] = value diff --git a/uv.lock b/uv.lock index ea0f251..5593e93 100644 --- a/uv.lock +++ b/uv.lock @@ -624,6 +624,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d8/c6/32d68bfbf1d0c36888530ef6fd72864861af23dc546302b41033471a8c3a/llvmlite-0.49.0-cp314-cp314t-win_amd64.whl", hash = "sha256:be637e465010bc9c50f070468f7f1cf5385e92fee364d192dd5e6cea790ecba9", size = 42986602, upload-time = "2026-08-11T16:25:57.69Z" }, ] +[[package]] +name = "mando" +version = "0.7.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "six" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/35/24/cd70d5ae6d35962be752feccb7dca80b5e0c2d450e995b16abd6275f3296/mando-0.7.1.tar.gz", hash = "sha256:18baa999b4b613faefb00eac4efadcf14f510b59b924b66e08289aa1de8c3500", size = 37868, upload-time = "2022-02-24T08:12:27.316Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d2/f0/834e479e47e499b6478e807fb57b31cc2db696c4db30557bb6f5aea4a90b/mando-0.7.1-py2.py3-none-any.whl", hash = "sha256:26ef1d70928b6057ee3ca12583d73c63e05c49de8972d620c278a7b206581a8a", size = 28149, upload-time = "2022-02-24T08:12:25.24Z" }, +] + [[package]] name = "markdown-it-py" version = "4.2.0" @@ -1231,6 +1243,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" }, ] +[[package]] +name = "radon" +version = "6.0.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama" }, + { name = "mando" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b1/6d/98e61600febf6bd929cf04154537c39dc577ce414bafbfc24a286c4fa76d/radon-6.0.1.tar.gz", hash = "sha256:d1ac0053943a893878940fedc8b19ace70386fc9c9bf0a09229a44125ebf45b5", size = 1874992, upload-time = "2023-03-26T06:24:38.868Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/93/f7/d00d9b4a0313a6be3a3e0818e6375e15da6d7076f4ae47d1324e7ca986a1/radon-6.0.1-py2.py3-none-any.whl", hash = "sha256:632cc032364a6f8bb1010a2f6a12d0f14bc7e5ede76585ef29dc0cecf4cd8859", size = 52784, upload-time = "2023-03-26T06:24:33.949Z" }, +] + [[package]] name = "requests" version = "2.34.2" @@ -1642,8 +1667,11 @@ dev = [ { name = "pytest" }, { name = "pytest-cov" }, { name = "pytest-memray" }, + { name = "radon" }, { name = "ruff" }, { name = "ty" }, + { name = "vulture" }, + { name = "xenon" }, ] [package.metadata] @@ -1667,8 +1695,34 @@ dev = [ { name = "pytest", specifier = ">=8.0" }, { name = "pytest-cov", specifier = ">=5.0" }, { name = "pytest-memray", specifier = ">=1.11.0" }, + { name = "radon", specifier = ">=6.0.1" }, { name = "ruff", specifier = ">=0.6" }, { name = "ty", specifier = ">=0.0.1a1" }, + { name = "vulture", specifier = ">=2.16" }, + { name = "xenon", specifier = ">=0.9.3" }, +] + +[[package]] +name = "vulture" +version = "2.16" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/66/3e/4d08c5903b2c0c70cad583c170cc4a663fc6a61e2ad00b711fcda61358cd/vulture-2.16.tar.gz", hash = "sha256:f8d9f6e2af03011664a3c6c240c9765b3f392917d3135fddca6d6a68d359f717", size = 52680, upload-time = "2026-03-25T14:41:27.141Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f5/be/f935130312330614811dae2ea9df3f395f6d63889eb6c2e68c14507152ee/vulture-2.16-py3-none-any.whl", hash = "sha256:6e0f1c312cef1c87856957e5c2ca9608834a7c794c2180477f30bf0e4cc58eee", size = 26993, upload-time = "2026-03-25T14:41:26.21Z" }, +] + +[[package]] +name = "xenon" +version = "0.9.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pyyaml" }, + { name = "radon" }, + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c4/7c/2b341eaeec69d514b635ea18481885a956d196a74322a4b0942ef0c31691/xenon-0.9.3.tar.gz", hash = "sha256:4a7538d8ba08aa5d79055fb3e0b2393c0bd6d7d16a4ab0fcdef02ef1f10a43fa", size = 9883, upload-time = "2024-10-21T10:27:53.722Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6f/5d/29ff8665b129cafd147d90b86e92babee32e116e3c84447107da3e77f8fb/xenon-0.9.3-py2.py3-none-any.whl", hash = "sha256:6e2c2c251cc5e9d01fe984e623499b13b2140fcbf74d6c03a613fa43a9347097", size = 8966, upload-time = "2024-10-21T10:27:51.121Z" }, ] [[package]] From 1e76a68fe273e09c87819e67f9e0cd7916b86d97 Mon Sep 17 00:00:00 2001 From: Aaron Meyer Date: Fri, 18 Sep 2026 08:51:46 -0700 Subject: [PATCH 2/3] Break up the remaining C-rank functions flagged by radon/xenon Pulls apart every plain-Python function radon/xenon ranked C (cyclomatic complexity >= 11), all in _anndata_class.py/_norm_common.py/_rapid_load.py, by extracting cohesive pieces into named helpers: - _anndata_class.py: __getitem__/copy/from_anndata/to_anndata/_write_group shared a repeated "filter the None key, transform each value" comprehension -- factored into _filtered/_map_subset_1d/_map_subset_2d/_map_copy. normalized()'s "reuse stats stored in obs/varm/uns" branch is now _normalized_from_stored(). X/raw_X's setter coercion was already shared via _coerce_vcs (see previous commit). - _norm_common.py: NormalizedViewBase.__init__'s VCSC/VCSR statistics dispatch and mean/scale finishing are now _column_gstats/_column_mean_scale. - _rapid_load.py: load_and_normalize's obs_filter/no-filter branches (each with their own gene-mask logic) are now _rows_without_obs_filter/ _rows_with_obs_filter/_compute_gene_mask; the h5py group read and the min_cells validation are now _read_ivcsr_group/_validate_min_cells. All of these were plain Python orchestration -- no behavior changes, and no numba kernels touched. The one function still ranked C, _column_stats_major_is_col in _norm_common.py, is a numba-parallel kernel whose branchiness is the documented, intentional cost of fusing three statistics passes into one over each column (see its docstring) -- left alone, consistent with CLAUDE.md's guidance not to force-fix pre-existing complexity that reflects a deliberate performance tradeoff. uv run pytest: 1352 passed, 61 skipped (unchanged). ruff/vulture/jscpd results unchanged from the previous commit. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_016wKWZn2btQ3Uo4grjuxxd2 --- src/vsparse/_anndata_class.py | 112 +++++++++++--------- src/vsparse/_norm_common.py | 128 +++++++++++----------- src/vsparse/_rapid_load.py | 193 ++++++++++++++++++++++------------ 3 files changed, 252 insertions(+), 181 deletions(-) diff --git a/src/vsparse/_anndata_class.py b/src/vsparse/_anndata_class.py index f063945..ef2bea1 100644 --- a/src/vsparse/_anndata_class.py +++ b/src/vsparse/_anndata_class.py @@ -91,6 +91,23 @@ def _copy_value(v: Any) -> Any: return None if v is None else v.copy() +def _filtered(mapping: Mapping[Any, Any]) -> dict[Any, Any]: + """A plain dict of ``mapping``, dropping the ``None`` key anndata sometimes carries.""" + return {k: v for k, v in mapping.items() if k is not None} + + +def _map_subset_1d(mapping: Mapping[Any, Any], idx: Any) -> dict[Any, Any]: + return {k: _subset_1d(v, idx) for k, v in mapping.items() if k is not None} + + +def _map_subset_2d(mapping: Mapping[Any, Any], oidx: Any, vidx: Any) -> dict[Any, Any]: + return {k: _subset_2d(v, oidx, vidx) for k, v in mapping.items() if k is not None} + + +def _map_copy(mapping: Mapping[Any, Any]) -> dict[Any, Any]: + return {k: _copy_value(v) for k, v in mapping.items() if k is not None} + + def _check_vcs_type(value: Any, name: str) -> None: if value is not None and not isinstance(value, _VCS_TYPES): raise TypeError( @@ -219,11 +236,11 @@ def __getitem__(self, index: Any) -> VCSCAnnData: # ty: ignore[invalid-method-o obs=obs, var=var, uns=uns, - obsm={k: _subset_1d(v, oidx) for k, v in self.obsm.items() if k is not None}, - varm={k: _subset_1d(v, vidx) for k, v in self.varm.items() if k is not None}, - obsp={k: _subset_2d(v, oidx, oidx) for k, v in self.obsp.items() if k is not None}, - varp={k: _subset_2d(v, vidx, vidx) for k, v in self.varp.items() if k is not None}, - layers={k: _subset_2d(v, oidx, vidx) for k, v in self.layers.items() if k is not None}, + obsm=_map_subset_1d(self.obsm, oidx), + varm=_map_subset_1d(self.varm, vidx), + obsp=_map_subset_2d(self.obsp, oidx, oidx), + varp=_map_subset_2d(self.varp, vidx, vidx), + layers=_map_subset_2d(self.layers, oidx, vidx), ) def copy(self) -> VCSCAnnData: # ty: ignore[invalid-method-override] @@ -244,11 +261,11 @@ def copy(self) -> VCSCAnnData: # ty: ignore[invalid-method-override] obs=cast(pd.DataFrame, self.obs).copy(), var=cast(pd.DataFrame, self.var).copy(), uns=_copy.deepcopy(dict(self.uns)), - obsm={k: _copy_value(v) for k, v in self.obsm.items() if k is not None}, - varm={k: _copy_value(v) for k, v in self.varm.items() if k is not None}, - obsp={k: _copy_value(v) for k, v in self.obsp.items() if k is not None}, - varp={k: _copy_value(v) for k, v in self.varp.items() if k is not None}, - layers={k: _copy_value(v) for k, v in self.layers.items() if k is not None}, + obsm=_map_copy(self.obsm), + varm=_map_copy(self.varm), + obsp=_map_copy(self.obsp), + varp=_map_copy(self.varp), + layers=_map_copy(self.layers), ) def to_memory(self, *, copy: bool = False) -> VCSCAnnData: @@ -271,6 +288,31 @@ def to_memory(self, *, copy: bool = False) -> VCSCAnnData: # -- normalization ---------------------------------------------------------- + def _normalized_from_stored(self, recipe: Recipe) -> Any: + """Rebuild a normalized view from ``obs``/``varm``/``uns`` if they match ``recipe``, else ``None``.""" + stored = self.uns.get(_VSPARSE_UNS_KEY) + if not ( + stored is not None + and stored.get("recipe") == recipe.name + and _VSPARSE_OBS_A in self.obs + and len(self.obs[_VSPARSE_OBS_A]) == self.n_obs + and _VSPARSE_VARM_B in self.varm + and _VSPARSE_VARM_C in self.varm + and _VSPARSE_VARM_S in self.varm + and len(self.varm[_VSPARSE_VARM_B]) == self.n_vars + ): + return None + nview_cls = VCSCArrayNormalized if isinstance(self._vcs_X, VCSCArray) else VCSRArrayNormalized + return nview_cls.from_stats( + self._vcs_X, + recipe, + a=np.asarray(self.obs[_VSPARSE_OBS_A], dtype=np.float64), + b=np.asarray(self.varm[_VSPARSE_VARM_B], dtype=np.float64).reshape(-1), + c=np.asarray(self.varm[_VSPARSE_VARM_C], dtype=np.float64).reshape(-1), + s=np.asarray(self.varm[_VSPARSE_VARM_S], dtype=np.float64).reshape(-1), + stale=bool(stored.get("stale", False)), + ) + def normalized(self, view: str | Recipe = DEFAULT_RECIPE, *, recalculate: bool = True) -> Any: """A normalized view of ``X`` -- see :meth:`vsparse._base._VCSBase.normalized`. @@ -299,31 +341,8 @@ def normalized(self, view: str | Recipe = DEFAULT_RECIPE, *, recalculate: bool = cached = cache.get(recipe, self._vcs_X) if cached is not None: return cached - stored = self.uns.get(_VSPARSE_UNS_KEY) - if ( - stored is not None - and stored.get("recipe") == recipe.name - and _VSPARSE_OBS_A in self.obs - and len(self.obs[_VSPARSE_OBS_A]) == self.n_obs - and _VSPARSE_VARM_B in self.varm - and _VSPARSE_VARM_C in self.varm - and _VSPARSE_VARM_S in self.varm - and len(self.varm[_VSPARSE_VARM_B]) == self.n_vars - ): - nview_cls = ( - VCSCArrayNormalized - if isinstance(self._vcs_X, VCSCArray) - else VCSRArrayNormalized - ) - nview = nview_cls.from_stats( - self._vcs_X, - recipe, - a=np.asarray(self.obs[_VSPARSE_OBS_A], dtype=np.float64), - b=np.asarray(self.varm[_VSPARSE_VARM_B], dtype=np.float64).reshape(-1), - c=np.asarray(self.varm[_VSPARSE_VARM_C], dtype=np.float64).reshape(-1), - s=np.asarray(self.varm[_VSPARSE_VARM_S], dtype=np.float64).reshape(-1), - stale=bool(stored.get("stale", False)), - ) + nview = self._normalized_from_stored(recipe) + if nview is not None: cache.put(recipe, nview) return nview @@ -360,11 +379,11 @@ def from_anndata( obs=cast(pd.DataFrame, adata.obs).copy(), var=cast(pd.DataFrame, adata.var).copy(), uns=adata.uns, - obsm={k: v for k, v in adata.obsm.items() if k is not None}, - varm={k: v for k, v in adata.varm.items() if k is not None}, - obsp={k: v for k, v in adata.obsp.items() if k is not None}, - varp={k: v for k, v in adata.varp.items() if k is not None}, - layers={k: v for k, v in adata.layers.items() if k is not None}, + obsm=_filtered(adata.obsm), + varm=_filtered(adata.varm), + obsp=_filtered(adata.obsp), + varp=_filtered(adata.varp), + layers=_filtered(adata.layers), ) def to_anndata(self) -> ad.AnnData: @@ -376,11 +395,11 @@ def to_anndata(self) -> ad.AnnData: obs=obs.copy(), var=var.copy(), uns=self.uns, - obsm=cast(Any, {k: v for k, v in self.obsm.items() if k is not None}), - varm=cast(Any, {k: v for k, v in self.varm.items() if k is not None}), - obsp=cast(Any, {k: v for k, v in self.obsp.items() if k is not None}), - varp=cast(Any, {k: v for k, v in self.varp.items() if k is not None}), - layers=cast(Any, {k: v for k, v in self.layers.items() if k is not None}), + obsm=cast(Any, _filtered(self.obsm)), + varm=cast(Any, _filtered(self.varm)), + obsp=cast(Any, _filtered(self.obsp)), + varp=cast(Any, _filtered(self.varp)), + layers=cast(Any, _filtered(self.layers)), ) if self._vcs_raw_X is not None: out.raw = ad.AnnData(X=self._vcs_raw_X.to_scipy(), obs=obs.copy(), var=var.copy()) @@ -416,8 +435,7 @@ def _write_group( for key in _DF_KEYS: ad.io.write_elem(g, key, getattr(self, key), dataset_kwargs=dataset_kwargs) for key in _MAPPING_KEYS: - mapping = {k: v for k, v in getattr(self, key).items() if k is not None} - ad.io.write_elem(g, key, mapping, dataset_kwargs=dataset_kwargs) + ad.io.write_elem(g, key, _filtered(getattr(self, key)), dataset_kwargs=dataset_kwargs) g.attrs["encoding-type"] = "anndata" g.attrs["encoding-version"] = "0.1.0" diff --git a/src/vsparse/_norm_common.py b/src/vsparse/_norm_common.py index 97b2d1a..4a8e93f 100644 --- a/src/vsparse/_norm_common.py +++ b/src/vsparse/_norm_common.py @@ -679,6 +679,61 @@ def _is_constant_column(variance: np.ndarray, mean: np.ndarray, n_rows: int) -> return variance <= n_rows * eps * variance + (n_rows * mean * eps) ** 2 +def _column_gstats(arr, row_scale, need_b, need_gstats, g_code, n_rows, n_cols): + """``gene_scale`` plus the stored-only sum/m2/corr/nnz feeding :func:`_finish_variance`. + + Dispatches on ``arr._format``: VCSC fuses both statistics into one pass + per column (see :func:`_column_stats_major_is_col`); VCSR needs two + genuinely separate passes, since a column's ``gene_scale`` isn't final + until every row has scattered into it. + """ + indices = arr.indices # decode once; shared by both statistics passes below + zeros = np.zeros(n_cols, dtype=np.float64) + ones = np.ones(n_cols, dtype=np.float64) + + if arr._format == "csc": + if not (need_b or need_gstats): + return ones, zeros, zeros, zeros, zeros + return _column_stats_major_is_col( + arr.major_ptr, arr.values, arr.value_ptr, indices, row_scale, need_b, need_gstats, g_code, n_rows + ) + + gene_scale = ones + if need_b: + nthreads = numba.get_num_threads() + gene_scale = _scaled_col_sums_vcs( + arr.major_ptr, arr.values, arr.value_ptr, indices, row_scale, n_cols, nthreads + ) + if not need_gstats: + return gene_scale, zeros, zeros, zeros, zeros + + nthreads = numba.get_num_threads() + kernel_args = (arr.major_ptr, arr.values, arr.value_ptr, indices, row_scale, gene_scale, g_code) + col_sum, col_nnz = _gstats_col_sums_vcs(*kernel_args, n_cols, nthreads) + col_m2, col_corr = _gstats_col_deviations_vcs( + *kernel_args, col_sum / n_rows if n_rows > 0 else zeros, n_cols, nthreads + ) + return gene_scale, col_sum, col_m2, col_corr, col_nnz + + +def _column_mean_scale(recipe, col_sum, col_m2, col_corr, col_nnz, n_rows, n_cols, *, need_gstats): + """Per-column ``col_mean``/``col_post_scale``, or the recipe's no-op defaults.""" + zeros = np.zeros(n_cols, dtype=np.float64) + ones = np.ones(n_cols, dtype=np.float64) + if not need_gstats: + return zeros, ones + + mean, variance = _finish_variance(col_sum, col_m2, col_corr, col_nnz, n_rows) + col_mean = mean if recipe.center else zeros + if not recipe.post_scale: + return col_mean, ones + + std = np.sqrt(variance) + with np.errstate(divide="ignore", invalid="ignore"): + col_post_scale = np.where(_is_constant_column(variance, mean, n_rows), 1.0, 1.0 / std) + return col_mean, col_post_scale + + class NormalizedViewBase: """Shared implementation for the normalized VCSC/VCSR views. @@ -722,76 +777,15 @@ def __init__( row_scale = _compute_row_scale(arr, self.recipe) self.row_scale = row_scale - indices = arr.indices # decode once; shared by both statistics passes below need_b = self.recipe.gene_scale need_gstats = self.recipe.center or self.recipe.post_scale - - if self._format == "csc": - # One fused pass per column for both -- see _column_stats_major_is_col. - if need_b or need_gstats: - gene_scale, col_sum, col_m2, col_corr, col_nnz = _column_stats_major_is_col( - arr.major_ptr, - arr.values, - arr.value_ptr, - indices, - row_scale, - need_b, - need_gstats, - self.recipe.g_code, - n_rows, - ) - else: - gene_scale = np.ones(n_cols, dtype=np.float64) - col_sum = col_m2 = col_corr = col_nnz = np.zeros(n_cols, dtype=np.float64) - else: - # VCSR can't fuse these: gene_scale[c] isn't final until every row - # has been scattered into it, so the g-transform pass has to wait - # for the whole first pass to finish -- two genuinely separate passes. - if need_b: - nthreads = numba.get_num_threads() - gene_scale = _scaled_col_sums_vcs( - arr.major_ptr, arr.values, arr.value_ptr, indices, row_scale, n_cols, nthreads - ) - else: - gene_scale = np.ones(n_cols, dtype=np.float64) - if need_gstats: - nthreads = numba.get_num_threads() - kernel_args = ( - arr.major_ptr, - arr.values, - arr.value_ptr, - indices, - row_scale, - gene_scale, - self.recipe.g_code, - ) - col_sum, col_nnz = _gstats_col_sums_vcs(*kernel_args, n_cols, nthreads) - col_m2, col_corr = _gstats_col_deviations_vcs( - *kernel_args, - col_sum / n_rows if n_rows > 0 else np.zeros(n_cols, dtype=np.float64), - n_cols, - nthreads, - ) - else: - col_sum = col_m2 = col_corr = col_nnz = np.zeros(n_cols, dtype=np.float64) + gene_scale, col_sum, col_m2, col_corr, col_nnz = _column_gstats( + arr, row_scale, need_b, need_gstats, self.recipe.g_code, n_rows, n_cols + ) self.gene_scale = gene_scale - - if need_gstats: - mean, variance = _finish_variance(col_sum, col_m2, col_corr, col_nnz, n_rows) - std = np.sqrt(variance) - col_mean = mean if self.recipe.center else np.zeros(n_cols, dtype=np.float64) - if self.recipe.post_scale: - with np.errstate(divide="ignore", invalid="ignore"): - col_post_scale = np.where( - _is_constant_column(variance, mean, n_rows), 1.0, 1.0 / std - ) - else: - col_post_scale = np.ones(n_cols, dtype=np.float64) - else: - col_mean = np.zeros(n_cols, dtype=np.float64) - col_post_scale = np.ones(n_cols, dtype=np.float64) - self.col_mean = col_mean - self.col_post_scale = col_post_scale + self.col_mean, self.col_post_scale = _column_mean_scale( + self.recipe, col_sum, col_m2, col_corr, col_nnz, n_rows, n_cols, need_gstats=need_gstats + ) @classmethod def from_stats( diff --git a/src/vsparse/_rapid_load.py b/src/vsparse/_rapid_load.py index e8b7b12..03af785 100644 --- a/src/vsparse/_rapid_load.py +++ b/src/vsparse/_rapid_load.py @@ -51,7 +51,7 @@ from collections.abc import Callable from pathlib import Path -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any import anndata as ad import numba @@ -356,9 +356,112 @@ def _normalize_and_transform( # -- top-level entry point --------------------------------------------------- +def _compute_gene_mask( + indices: np.ndarray, + data: np.ndarray, + n_genes: int, + denom: int, + gene_threshold: float, + min_cells: int | None, +) -> np.ndarray: + """Genes with total raw counts above ``gene_threshold * denom``, and detected in >= ``min_cells``.""" + gene_totals_raw = _weighted_bincount(indices, data, n_genes, accumulator_threads(n_genes)) + gene_mask = gene_totals_raw > (gene_threshold * denom) + if min_cells is not None: + gene_detection_counts = _gene_detection_counts( + indices, data, n_genes, accumulator_threads(n_genes) + ) + gene_mask &= gene_detection_counts >= min_cells + return gene_mask + + +def _rows_without_obs_filter( + major_ptr: np.ndarray, + values: np.ndarray, + value_ptr: np.ndarray, + packed: np.ndarray, + indices_dtype: np.dtype, + cell_totals: np.ndarray, + min_cell_counts: float, +) -> tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray]: + """Decode every row; ``metadata_cell_mask`` is just ``cell_mask`` since none were dropped upfront.""" + cell_mask = cell_totals > min_cell_counts + indices = _ivcsc.unpack_indices(value_ptr, packed, indices_dtype) + data = _build_data(values, value_ptr, indices.shape[0]) + row_indptr = value_ptr[major_ptr] + return cell_mask, row_indptr, indices, data, cell_mask + + +def _rows_with_obs_filter( + major_ptr: np.ndarray, + values: np.ndarray, + value_ptr: np.ndarray, + packed: np.ndarray, + indices_dtype: np.dtype, + cell_totals: np.ndarray, + obs: Any, + obs_filter: Callable[[pd.DataFrame], object], + min_cell_counts: float, + n_cells: int, +) -> tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray, int]: + """Validate ``obs_filter``, then decode only the rows it selects.""" + if not isinstance(obs, pd.DataFrame): + raise ValueError("obs_filter requires an obs table in the h5ad file") + if not callable(obs_filter): + raise TypeError("obs_filter must be callable or None") + + obs_mask = np.asarray(obs_filter(obs)) + if obs_mask.ndim != 1 or obs_mask.shape[0] != n_cells: + raise ValueError(f"obs_filter must return a one-dimensional mask of length {n_cells}") + if obs_mask.dtype != np.bool_: + raise ValueError("obs_filter must return a boolean mask") + if not np.any(obs_mask): + raise ValueError("obs_filter selected no cells") + obs_mask = np.ascontiguousarray(obs_mask) + + selected_rows = np.nonzero(obs_mask)[0] + cell_mask = cell_totals[obs_mask] > min_cell_counts + row_indptr, indices, data = _build_selected_rows( + major_ptr, values, value_ptr, packed, indices_dtype, obs_mask + ) + + metadata_cell_mask = np.zeros(n_cells, dtype=np.bool_) + metadata_cell_mask[selected_rows[cell_mask]] = True + return cell_mask, row_indptr, indices, data, metadata_cell_mask, selected_rows.shape[0] + + _FIELD_KEYS = ("obs", "var", "obsm", "varm", "obsp", "varp", "layers", "uns") +def _validate_min_cells(min_cells: int | None) -> None: + if min_cells is None: + return + if isinstance(min_cells, bool) or not isinstance(min_cells, int): + raise TypeError("min_cells must be an integer or None") + if min_cells < 0: + raise ValueError("min_cells must be non-negative") + + +def _read_ivcsr_group( + f: Any, x_key: str +) -> tuple[ + tuple[int, int], np.dtype, np.ndarray, np.ndarray, np.ndarray, np.ndarray, dict[str, Any] +]: + """Read the packed IVCSR array at ``f[x_key]`` plus the surrounding obs/var/etc. fields.""" + g = f[x_key] + shape = (int(g.attrs["shape"][0]), int(g.attrs["shape"][1])) + indices_dtype = np.dtype(g.attrs["indices_dtype"]) + major_ptr = g["major_ptr"][...] + values = g["values"][...] + value_ptr = g["value_ptr"][...] + packed = g["packed_indices"][...] + + kwargs = {k: ad.io.read_elem(f[k]) for k in _FIELD_KEYS if k in f} + if "raw" in f: + kwargs["raw"] = ad.io.read_elem(f["raw"]) + return shape, indices_dtype, major_ptr, values, value_ptr, packed, kwargs + + def load_and_normalize( path: str | PathLike[str], *, @@ -426,24 +529,12 @@ def load_and_normalize( import h5py import hdf5plugin # noqa: F401 -- registers the Blosc2 HDF5 filter - if min_cells is not None: - if isinstance(min_cells, bool) or not isinstance(min_cells, int): - raise TypeError("min_cells must be an integer or None") - if min_cells < 0: - raise ValueError("min_cells must be non-negative") + _validate_min_cells(min_cells) with h5py.File(Path(path), "r") as f: - g = f[x_key] - shape = (int(g.attrs["shape"][0]), int(g.attrs["shape"][1])) - indices_dtype = np.dtype(g.attrs["indices_dtype"]) - major_ptr = g["major_ptr"][...] - values = g["values"][...] - value_ptr = g["value_ptr"][...] - packed = g["packed_indices"][...] - - kwargs = {k: ad.io.read_elem(f[k]) for k in _FIELD_KEYS if k in f} - if "raw" in f: - kwargs["raw"] = ad.io.read_elem(f["raw"]) + shape, indices_dtype, major_ptr, values, value_ptr, packed, kwargs = _read_ivcsr_group( + f, x_key + ) n_cells, n_genes = shape @@ -451,58 +542,26 @@ def load_and_normalize( cell_totals = _cell_totals(major_ptr, values, value_ptr) if obs_filter is None: - cell_mask = cell_totals > min_cell_counts - - # Everything past this point needs every nonzero visited at least once. - # Each array below is only kept alive as long as something still needs - # it -- at nnz-billions scale, an un-`del`ed stale reference is a real - # multi-GB cost, not housekeeping. - indices = _ivcsc.unpack_indices(value_ptr, packed, indices_dtype) - del packed - data = _build_data(values, value_ptr, indices.shape[0]) - row_indptr = value_ptr[major_ptr] - - gene_totals_raw = _weighted_bincount(indices, data, n_genes, accumulator_threads(n_genes)) - gene_mask = gene_totals_raw > (gene_threshold * n_cells) - if min_cells is not None: - gene_detection_counts = _gene_detection_counts( - indices, data, n_genes, accumulator_threads(n_genes) - ) - gene_mask &= gene_detection_counts >= min_cells - metadata_cell_mask = cell_mask + cell_mask, row_indptr, indices, data, metadata_cell_mask = _rows_without_obs_filter( + major_ptr, values, value_ptr, packed, indices_dtype, cell_totals, min_cell_counts + ) + gene_denom = n_cells else: - obs = kwargs.get("obs") - if not isinstance(obs, pd.DataFrame): - raise ValueError("obs_filter requires an obs table in the h5ad file") - if not callable(obs_filter): - raise TypeError("obs_filter must be callable or None") - - obs_mask = np.asarray(obs_filter(obs)) - if obs_mask.ndim != 1 or obs_mask.shape[0] != n_cells: - raise ValueError(f"obs_filter must return a one-dimensional mask of length {n_cells}") - if obs_mask.dtype != np.bool_: - raise ValueError("obs_filter must return a boolean mask") - if not np.any(obs_mask): - raise ValueError("obs_filter selected no cells") - obs_mask = np.ascontiguousarray(obs_mask) - - selected_rows = np.nonzero(obs_mask)[0] - cell_mask = cell_totals[obs_mask] > min_cell_counts - row_indptr, indices, data = _build_selected_rows( - major_ptr, values, value_ptr, packed, indices_dtype, obs_mask + cell_mask, row_indptr, indices, data, metadata_cell_mask, gene_denom = _rows_with_obs_filter( + major_ptr, + values, + value_ptr, + packed, + indices_dtype, + cell_totals, + kwargs.get("obs"), + obs_filter, + min_cell_counts, + n_cells, ) - del packed - - gene_totals_raw = _weighted_bincount(indices, data, n_genes, accumulator_threads(n_genes)) - gene_mask = gene_totals_raw > (gene_threshold * selected_rows.shape[0]) - if min_cells is not None: - gene_detection_counts = _gene_detection_counts( - indices, data, n_genes, accumulator_threads(n_genes) - ) - gene_mask &= gene_detection_counts >= min_cells - - metadata_cell_mask = np.zeros(n_cells, dtype=np.bool_) - metadata_cell_mask[selected_rows[cell_mask]] = True + del packed + + gene_mask = _compute_gene_mask(indices, data, n_genes, gene_denom, gene_threshold, min_cells) new_indptr, out_indices, out_data, kept_rows, n_kept_genes = _filter_and_compact( row_indptr, indices, data, cell_mask, gene_mask From e5b2a870025371de1b767e919be70c07b1909e03 Mon Sep 17 00:00:00 2001 From: Aaron Meyer Date: Fri, 18 Sep 2026 09:25:54 -0700 Subject: [PATCH 3/3] Fix CI: benchmark gate, ty check, and formatting - benchmarks/cases.py: misaligned_matmat_vs_scipy/misaligned_rmatmat_vs_scipy still called peak_alloc_mb() after #56 dropped that helper entirely from benchmarks/harness.py, so every benchmark run failed with NameError. This was already broken on main (confirmed: main's own CI is red for the same reason), not something introduced by this branch, but it blocks this PR's gate too. Drops the two dead peak_alloc_mb entries (case output and baselines.json), matching how #56 already removed the other five memory-metric cases. - _ivcsc.py/_rapid_load.py: the shared _decode_varint helper from an earlier commit here expects a np.int64 position, but both callers seeded `pos` as a plain `0`, which `ty check` flagged as a real (if numba-tolerated) type mismatch. Seed `pos = np.int64(0)` instead. - _rapid_load.py: the _read_ivcsr_group extraction from the same earlier commit gives `kwargs` an explicit `dict[str, Any]` return type, which resolves the invalid-argument-type ty previously needed ignore comments for on `ad.AnnData(**kwargs)` -- removed the two now-unused ignores. - ruff format on the three files touched by the complexity refactor. uv run pytest: 1352 passed, 61 skipped. uv run python -m benchmarks.run --set fast: no regressions. ruff check/format, ty check, codespell all pass. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_016wKWZn2btQ3Uo4grjuxxd2 --- benchmarks/baselines.json | 6 ++---- benchmarks/cases.py | 10 ++-------- src/vsparse/_anndata_class.py | 4 +++- src/vsparse/_ivcsc.py | 2 +- src/vsparse/_norm_common.py | 10 +++++++++- src/vsparse/_rapid_load.py | 30 ++++++++++++++++-------------- 6 files changed, 33 insertions(+), 29 deletions(-) diff --git a/benchmarks/baselines.json b/benchmarks/baselines.json index 3431493..3dfd1bb 100644 --- a/benchmarks/baselines.json +++ b/benchmarks/baselines.json @@ -95,12 +95,10 @@ "time_ratio_vs_scipy": 1.2909 }, "misaligned_matmat_vs_scipy": { - "time_ratio_vs_scipy": 2.0436, - "peak_alloc_mb": 38.4009 + "time_ratio_vs_scipy": 2.0436 }, "misaligned_rmatmat_vs_scipy": { - "time_ratio_vs_scipy": 1.0817, - "peak_alloc_mb": 20.2251 + "time_ratio_vs_scipy": 1.0817 } } } diff --git a/benchmarks/cases.py b/benchmarks/cases.py index 9da2687..ef2b68f 100644 --- a/benchmarks/cases.py +++ b/benchmarks/cases.py @@ -308,10 +308,7 @@ def misaligned_matmat_vs_scipy() -> dict[str, float]: v = VCSCArray.from_scipy(mat) csc = sp.csc_array(mat) B = np.random.default_rng(0).normal(size=(mat.shape[1], 8)) - return { - "time_ratio_vs_scipy": ratio_vs_scipy(lambda: v @ B, lambda: csc @ B), - "peak_alloc_mb": peak_alloc_mb(lambda: v @ B), - } + return {"time_ratio_vs_scipy": ratio_vs_scipy(lambda: v @ B, lambda: csc @ B)} @fast @@ -325,10 +322,7 @@ def misaligned_rmatmat_vs_scipy() -> dict[str, float]: v = VCSRArray.from_scipy(mat) csr = sp.csr_array(mat) B = np.random.default_rng(0).normal(size=(8, mat.shape[0])) - return { - "time_ratio_vs_scipy": ratio_vs_scipy(lambda: B @ v, lambda: B @ csr), - "peak_alloc_mb": peak_alloc_mb(lambda: B @ v), - } + return {"time_ratio_vs_scipy": ratio_vs_scipy(lambda: B @ v, lambda: B @ csr)} # -- larger, for the scheduled job ------------------------------------------- diff --git a/src/vsparse/_anndata_class.py b/src/vsparse/_anndata_class.py index ef2bea1..4519b31 100644 --- a/src/vsparse/_anndata_class.py +++ b/src/vsparse/_anndata_class.py @@ -302,7 +302,9 @@ def _normalized_from_stored(self, recipe: Recipe) -> Any: and len(self.varm[_VSPARSE_VARM_B]) == self.n_vars ): return None - nview_cls = VCSCArrayNormalized if isinstance(self._vcs_X, VCSCArray) else VCSRArrayNormalized + nview_cls = ( + VCSCArrayNormalized if isinstance(self._vcs_X, VCSCArray) else VCSRArrayNormalized + ) return nview_cls.from_stats( self._vcs_X, recipe, diff --git a/src/vsparse/_ivcsc.py b/src/vsparse/_ivcsc.py index 43859b7..a7b3a2e 100644 --- a/src/vsparse/_ivcsc.py +++ b/src/vsparse/_ivcsc.py @@ -95,7 +95,7 @@ def _decode_varint(buf: np.ndarray, pos: np.int64) -> tuple[np.uint64, np.int64] @numba.njit(cache=True) def _unpack(value_ptr: np.ndarray, buf: np.ndarray, out: np.ndarray) -> None: n_groups = value_ptr.shape[0] - 1 - pos = 0 + pos = np.int64(0) for g in range(n_groups): start, end = value_ptr[g], value_ptr[g + 1] prev = np.int64(-1) diff --git a/src/vsparse/_norm_common.py b/src/vsparse/_norm_common.py index 4a8e93f..63bd638 100644 --- a/src/vsparse/_norm_common.py +++ b/src/vsparse/_norm_common.py @@ -695,7 +695,15 @@ def _column_gstats(arr, row_scale, need_b, need_gstats, g_code, n_rows, n_cols): if not (need_b or need_gstats): return ones, zeros, zeros, zeros, zeros return _column_stats_major_is_col( - arr.major_ptr, arr.values, arr.value_ptr, indices, row_scale, need_b, need_gstats, g_code, n_rows + arr.major_ptr, + arr.values, + arr.value_ptr, + indices, + row_scale, + need_b, + need_gstats, + g_code, + n_rows, ) gene_scale = ones diff --git a/src/vsparse/_rapid_load.py b/src/vsparse/_rapid_load.py index 03af785..19d8a3e 100644 --- a/src/vsparse/_rapid_load.py +++ b/src/vsparse/_rapid_load.py @@ -119,7 +119,7 @@ def _decode_selected_rows( out_data: np.ndarray, ) -> None: """Decode only selected IVCSR rows while scanning past excluded rows.""" - pos = 0 + pos = np.int64(0) out_pos = 0 n_rows = major_ptr.shape[0] - 1 @@ -547,17 +547,19 @@ def load_and_normalize( ) gene_denom = n_cells else: - cell_mask, row_indptr, indices, data, metadata_cell_mask, gene_denom = _rows_with_obs_filter( - major_ptr, - values, - value_ptr, - packed, - indices_dtype, - cell_totals, - kwargs.get("obs"), - obs_filter, - min_cell_counts, - n_cells, + cell_mask, row_indptr, indices, data, metadata_cell_mask, gene_denom = ( + _rows_with_obs_filter( + major_ptr, + values, + value_ptr, + packed, + indices_dtype, + cell_totals, + kwargs.get("obs"), + obs_filter, + min_cell_counts, + n_cells, + ) ) del packed @@ -572,9 +574,9 @@ def load_and_normalize( X = csr_array((normalized, out_indices, new_indptr), shape=(kept_rows.shape[0], n_kept_genes)) if "obs" in kwargs and "var" in kwargs: - adata = ad.AnnData(**kwargs) # ty: ignore[invalid-argument-type] + adata = ad.AnnData(**kwargs) else: - adata = ad.AnnData(shape=shape, **kwargs) # ty: ignore[invalid-argument-type] + adata = ad.AnnData(shape=shape, **kwargs) adata = adata[metadata_cell_mask, gene_mask].copy() adata.X = X