Add options to omit empty or default columns from v2 tables

Converting PEtab v2 tables to DataFrames or TSV files currently always
includes all standard columns, even if they are empty in every row.

Add `TableColumns` modes `all` (default, current behavior), `non_empty`,
and `non_default` to select the columns to include. Required columns are
always included. The mode can be passed to `to_df`, `to_tsv`, and
`Problem.to_files`, or set via `petab.v2.set_options` or temporarily via
`petab.v2.option_context`, which also applies to `Problem.*_df`.

Also add the `parameterName` column as `Parameter.name` and make
`to_df` work for empty parameter, observable, and measurement tables.

Closes #523.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

Author: dweindl

doc/modules.rst

@@ -38,4 +38,5 @@ API Reference
    petab.v2.extensions
    petab.v2.lint
    petab.v2.models
+   petab.v2.options
    petab.v2.petab1to2

petab/v2/__init__.py

@@ -37,3 +37,4 @@
 )
 from .lint import lint_problem  # noqa: F401
 from .models import MODEL_TYPE_PYSB, MODEL_TYPE_SBML, Model  # noqa: F401
+from .options import *

petab/v2/core.py

@@ -8,10 +8,10 @@
 import tempfile
 import traceback
 from abc import abstractmethod
-from collections.abc import Sequence
+from collections.abc import Iterable, Sequence
 from enum import StrEnum
 from itertools import chain
-from math import nan
+from math import isnan, nan
 from numbers import Number
 from pathlib import Path
 from typing import (
@@ -54,6 +54,7 @@
 from ..versions import parse_version
 from . import C, get_observable_df
 from .extensions import ExtensionConfig, parse_extension_config
+from .options import TableColumns, get_options
 
 if TYPE_CHECKING:
     from ..v2.lint import ValidationResultList, ValidationTask
@@ -123,6 +124,24 @@ def _reset_index(df: pd.DataFrame) -> pd.DataFrame:
     return df.reset_index(drop=all(name is None for name in df.index.names))
 
 
+class _RequiredColumn:
+    """Marks a field whose column is required, even if it has a default."""
+
+
+#: Field annotation for columns that are required, even if they have a
+#: default value or may contain empty values.
+_REQUIRED_COLUMN = _RequiredColumn()
+
+
+def _is_empty(v) -> bool:
+    """Check if a serialized table value is empty."""
+    return (
+        v is None
+        or (isinstance(v, str) and v == "")
+        or (isinstance(v, float) and isnan(v))
+    )
+
+
 def _valid_petab_id(v: str) -> str:
     """Field validator for PEtab IDs."""
     if not v:
@@ -270,9 +289,81 @@ def __getitem__(self, id_: str) -> T:
     def from_df(cls, df: pd.DataFrame, **kwargs) -> BaseTable[T]:
         """Create a table from a DataFrame."""
 
+    def to_df(
+        self, *, columns: TableColumns | str | None = None
+    ) -> pd.DataFrame:
+        """Convert the table to a DataFrame.
+
+        :param columns:
+            The columns to include. See :class:`TableColumns`.
+            If ``None``, :attr:`Options.table_columns` is used
+            (see :func:`get_options`).
+        """
+        if columns is None:
+            columns = get_options().table_columns
+        columns = TableColumns(columns)
+
+        df = self._to_df()
+        if columns == TableColumns.ALL:
+            return df
+
+        required = set(self.required_columns())
+        drop = [
+            col
+            for col in df.columns
+            if col not in required and df[col].map(_is_empty).all()
+        ]
+        if columns == TableColumns.NON_DEFAULT:
+            drop.extend(
+                col
+                for col in self._default_columns()
+                if col in df.columns
+                and col not in required
+                and col not in drop
+            )
+        return df.drop(columns=drop)
+
     @abstractmethod
-    def to_df(self) -> pd.DataFrame:
-        """Convert the table to a DataFrame."""
+    def _to_df(self) -> pd.DataFrame:
+        """Convert the table to a DataFrame with all columns."""
+
+    @classmethod
+    def standard_columns(cls) -> list[str]:
+        """The standard columns of this table, in the default order."""
+        return [
+            field.alias or name
+            for name, field in cls._row_class().model_fields.items()
+        ]
+
+    @classmethod
+    def required_columns(cls) -> list[str]:
+        """The columns that are always included in this table."""
+        return [
+            field.alias or name
+            for name, field in cls._row_class().model_fields.items()
+            if field.is_required() or _REQUIRED_COLUMN in field.metadata
+        ]
+
+    @classmethod
+    def _row_class(cls) -> type[BaseModel]:
+        """The class corresponding to a single table row."""
+        return cls._element_class()
+
+    def _rows(self) -> Iterable[BaseModel]:
+        """The objects corresponding to the individual table rows."""
+        return self.elements
+
+    def _default_columns(self) -> list[str]:
+        """Get the standard columns that have the default value in all rows."""
+        rows = list(self._rows())
+        result = []
+        for name, field in self._row_class().model_fields.items():
+            if field.is_required():
+                continue
+            default = field.get_default(call_default_factory=True)
+            if all(getattr(row, name) == default for row in rows):
+                result.append(field.alias or name)
+        return result
 
     @classmethod
     def from_tsv(
@@ -282,9 +373,21 @@ def from_tsv(
         df = pd.read_csv(_generate_path(file_path, base_path), sep="\t")
         return cls.from_df(df, rel_path=file_path, base_path=base_path)
 
-    def to_tsv(self, file_path: str | Path | None = None) -> None:
-        """Write the table to a TSV file."""
-        df = self.to_df()
+    def to_tsv(
+        self,
+        file_path: str | Path | None = None,
+        *,
+        columns: TableColumns | str | None = None,
+    ) -> None:
+        """Write the table to a TSV file.
+
+        :param file_path:
+            The file to write to. If ``None``, the path is generated from
+            :attr:`rel_path` and :attr:`base_path`.
+        :param columns:
+            The columns to include. See :meth:`to_df`.
+        """
+        df = self.to_df(columns=columns)
         df.to_csv(
             file_path or _generate_path(self.rel_path, self.base_path),
             sep="\t",
@@ -426,8 +529,13 @@ def from_df(cls, df: pd.DataFrame, **kwargs) -> ObservableTable:
         ]
         return cls(observables, **kwargs)
 
-    def to_df(self) -> pd.DataFrame:
+    def _to_df(self) -> pd.DataFrame:
         """Convert the ObservableTable to a DataFrame."""
+        if not self.observables:
+            return pd.DataFrame(columns=self.standard_columns()).set_index(
+                [C.OBSERVABLE_ID]
+            )
+
         records = self.model_dump(by_alias=True)["elements"]
         for record in records:
             obs = record[C.OBSERVABLE_FORMULA]
@@ -550,7 +658,26 @@ def from_df(cls, df: pd.DataFrame, **kwargs) -> ConditionTable:
 
         return cls(conditions, **kwargs)
 
-    def to_df(self) -> pd.DataFrame:
+    @classmethod
+    def standard_columns(cls) -> list[str]:
+        """The standard columns of this table, in the default order."""
+        return [C.CONDITION_ID, *super().standard_columns()]
+
+    @classmethod
+    def required_columns(cls) -> list[str]:
+        """The columns that are always included in this table."""
+        return [C.CONDITION_ID, *super().required_columns()]
+
+    @classmethod
+    def _row_class(cls) -> type[BaseModel]:
+        return Change
+
+    def _rows(self) -> Iterable[BaseModel]:
+        return chain.from_iterable(
+            condition.changes for condition in self.conditions
+        )
+
+    def _to_df(self) -> pd.DataFrame:
         """Convert the ConditionTable to a DataFrame."""
         records = [
             {C.CONDITION_ID: condition.id, **change.model_dump(by_alias=True)}
@@ -566,7 +693,7 @@ def to_df(self) -> pd.DataFrame:
         return (
             pd.DataFrame(records)
             if records
-            else pd.DataFrame(columns=C.CONDITION_DF_REQUIRED_COLS)
+            else pd.DataFrame(columns=self.standard_columns())
         )
 
     @property
@@ -714,7 +841,26 @@ def from_df(cls, df: pd.DataFrame, **kwargs) -> ExperimentTable:
 
         return cls(experiments, **kwargs)
 
-    def to_df(self) -> pd.DataFrame:
+    @classmethod
+    def standard_columns(cls) -> list[str]:
+        """The standard columns of this table, in the default order."""
+        return [C.EXPERIMENT_ID, C.TIME, C.CONDITION_ID]
+
+    @classmethod
+    def required_columns(cls) -> list[str]:
+        """The columns that are always included in this table."""
+        return cls.standard_columns()
+
+    @classmethod
+    def _row_class(cls) -> type[BaseModel]:
+        return ExperimentPeriod
+
+    def _rows(self) -> Iterable[BaseModel]:
+        return chain.from_iterable(
+            experiment.periods for experiment in self.experiments
+        )
+
+    def _to_df(self) -> pd.DataFrame:
         """Convert the ExperimentTable to a DataFrame."""
         records = [
             {
@@ -729,7 +875,7 @@ def to_df(self) -> pd.DataFrame:
         return (
             pd.DataFrame(records)
             if records
-            else pd.DataFrame(columns=C.EXPERIMENT_DF_REQUIRED_COLS)
+            else pd.DataFrame(columns=self.standard_columns())
         )
 
 
@@ -750,7 +896,9 @@ class Measurement(BaseModel):
     )
     #: The experiment ID.
     experiment_id: Annotated[
-        str | None, BeforeValidator(_valid_petab_id_or_none)
+        str | None,
+        BeforeValidator(_valid_petab_id_or_none),
+        _REQUIRED_COLUMN,
     ] = Field(alias=C.EXPERIMENT_ID, default=None)
     #: The time point of the measurement in time units as defined in the model.
     time: Annotated[float, AfterValidator(_is_finite_or_pos_inf)] = Field(
@@ -835,8 +983,11 @@ def from_df(cls, df: pd.DataFrame, **kwargs) -> MeasurementTable:
 
         return cls(measurements, **kwargs)
 
-    def to_df(self) -> pd.DataFrame:
+    def _to_df(self) -> pd.DataFrame:
         """Convert the MeasurementTable to a DataFrame."""
+        if not self.measurements:
+            return pd.DataFrame(columns=self.standard_columns())
+
         records = self.model_dump(by_alias=True)["elements"]
         for record in records:
             record[C.OBSERVABLE_PARAMETERS] = C.PARAMETER_SEPARATOR.join(
@@ -857,9 +1008,9 @@ class Mapping(BaseModel):
         alias=C.PETAB_ENTITY_ID
     )
     #: Model entity ID.
-    model_id: Annotated[str | None, BeforeValidator(_convert_nan_to_none)] = (
-        Field(alias=C.MODEL_ENTITY_ID, default=None)
-    )
+    model_id: Annotated[
+        str | None, BeforeValidator(_convert_nan_to_none), _REQUIRED_COLUMN
+    ] = Field(alias=C.MODEL_ENTITY_ID, default=None)
     #: Arbitrary name
     name: Annotated[str | None, BeforeValidator(_convert_nan_to_none)] = Field(
         alias=C.NAME, default=None
@@ -906,12 +1057,12 @@ def from_df(cls, df: pd.DataFrame, **kwargs) -> MappingTable:
         ]
         return cls(mappings, **kwargs)
 
-    def to_df(self) -> pd.DataFrame:
+    def _to_df(self) -> pd.DataFrame:
         """Convert the MappingTable to a DataFrame."""
         res = (
             pd.DataFrame(self.model_dump(by_alias=True)["elements"])
             if self.mappings
-            else pd.DataFrame(columns=C.MAPPING_DF_REQUIRED_COLS)
+            else pd.DataFrame(columns=self.standard_columns())
         )
         return res.set_index([C.PETAB_ENTITY_ID])
 
@@ -937,6 +1088,10 @@ class Parameter(BaseModel):
     id: Annotated[str, BeforeValidator(_valid_petab_id)] = Field(
         alias=C.PARAMETER_ID
     )
+    #: Parameter name.
+    name: Annotated[str | None, BeforeValidator(_convert_nan_to_none)] = Field(
+        alias=C.PARAMETER_NAME, default=None
+    )
     #: Lower bound.
     lb: Annotated[float | None, BeforeValidator(_convert_nan_to_none)] = Field(
         alias=C.LOWER_BOUND, default=None
@@ -951,7 +1106,9 @@ class Parameter(BaseModel):
         float | Literal["array"] | None, BeforeValidator(_convert_nan_to_none)
     ] = Field(alias=C.NOMINAL_VALUE, default=None)
     #: Is the parameter to be estimated?
-    estimate: bool = Field(alias=C.ESTIMATE, default=True)
+    estimate: Annotated[bool, _REQUIRED_COLUMN] = Field(
+        alias=C.ESTIMATE, default=True
+    )
     #: Type of parameter prior distribution.
     prior_distribution: Annotated[
         PriorDistribution | None, BeforeValidator(_convert_nan_to_none)
@@ -1117,11 +1274,14 @@ def from_df(cls, df: pd.DataFrame, **kwargs) -> ParameterTable:
 
         return cls(parameters, **kwargs)
 
-    def to_df(self) -> pd.DataFrame:
+    def _to_df(self) -> pd.DataFrame:
         """Convert the ParameterTable to a DataFrame."""
-        return pd.DataFrame(
-            self.model_dump(by_alias=True)["elements"]
-        ).set_index([C.PARAMETER_ID])
+        res = (
+            pd.DataFrame(self.model_dump(by_alias=True)["elements"])
+            if self.parameters
+            else pd.DataFrame(columns=self.standard_columns())
+        )
+        return res.set_index([C.PARAMETER_ID])
 
     @property
     def n_estimated(self) -> int:
@@ -1431,7 +1591,12 @@ def get_problem(problem: str | Path | Problem) -> Problem:
             "or a PEtab problem object."
         )
 
-    def to_files(self, base_path: str | Path | None) -> None:
+    def to_files(
+        self,
+        base_path: str | Path | None,
+        *,
+        columns: TableColumns | str | None = None,
+    ) -> None:
         """Write the PEtab problem to files.
 
         Writes the model, condition, experiment, measurement, parameter,
@@ -1445,6 +1610,9 @@ def to_files(self, base_path: str | Path | None) -> None:
             The base path the yaml file and tables will be written to.
             If ``None``, the `base_path` of the individual tables and
             :obj:`Problem.config.base_path` will be used.
+        :param columns:
+            The columns to include in the tables.
+            See :meth:`BaseTable.to_df`.
         """
         config = copy.deepcopy(self.config) or ProblemConfig(
             format_version="2.0.0"
@@ -1499,7 +1667,8 @@ def to_files(self, base_path: str | Path | None) -> None:
                 table.to_tsv(
                     _generate_path(
                         table.rel_path, base_path or table.base_path
-                    )
+                    ),
+                    columns=columns,
                 )
 
         config.to_yaml(
@@ -2578,6 +2747,7 @@ def model_dump(self, **kwargs) -> dict[str, Any]:
          'parameters': [{'estimate': 'true',
                          'id': 'par',
                          'lb': 0.0,
+                         'name': None,
                          'nominal_value': None,
                          'prior_distribution': '',
                          'prior_parameters': '',