diff --git a/feature_engine/_base_transformers/base_numerical.py b/feature_engine/_base_transformers/base_numerical.py index 60212f3d6..10b24e99a 100644 --- a/feature_engine/_base_transformers/base_numerical.py +++ b/feature_engine/_base_transformers/base_numerical.py @@ -1,4 +1,4 @@ -""" The base transformer provides functionality that is shared by most transformer +"""The base transformer provides functionality that is shared by most transformer classes. Provides the base functionality within the fit() and transform() methods shared by most transformers, like checking that input is a df, the size, NA, etc. """ @@ -60,7 +60,9 @@ def fit(self, X: pd.DataFrame) -> pd.DataFrame: # find or check for numerical variables if self.variables is None: - self.variables_ = find_numerical_variables(X) + self.variables_ = find_numerical_variables( + X, return_empty=self.return_empty + ) else: self.variables_ = check_numerical_variables(X, self.variables) diff --git a/feature_engine/_check_init_parameters/check_init_input_params.py b/feature_engine/_check_init_parameters/check_init_input_params.py index f48c46e6c..e1000accc 100644 --- a/feature_engine/_check_init_parameters/check_init_input_params.py +++ b/feature_engine/_check_init_parameters/check_init_input_params.py @@ -12,3 +12,11 @@ def _check_param_drop_original(drop_original): "drop_original takes only boolean values True and False. " f"Got {drop_original} instead." ) + + +def _check_return_empty_is_bool(return_empty): + if not isinstance(return_empty, bool): + raise ValueError( + "return_empty takes only boolean values True and False. " + f"Got {return_empty} instead." + ) diff --git a/feature_engine/_docstrings/init_parameters/all_trasnformers.py b/feature_engine/_docstrings/init_parameters/all_transformers.py similarity index 68% rename from feature_engine/_docstrings/init_parameters/all_trasnformers.py rename to feature_engine/_docstrings/init_parameters/all_transformers.py index 5c699d3de..898106fd1 100644 --- a/feature_engine/_docstrings/init_parameters/all_trasnformers.py +++ b/feature_engine/_docstrings/init_parameters/all_transformers.py @@ -22,3 +22,15 @@ contain missing values. If `'ignore'`, missing data will be ignored when learning parameters or performing the transformation. """.rstrip() + +_return_empty_docstring = """return_empty : bool, default=False + Whether to return an empty list when no variables of the required type are + found. If False, the transformer raises an error. This parameter is only + used when `variables` is `None`. + + .. versionadded:: 2.0 + `return_empty` currently defaults to False. The default will change to + True in version 2.1. To keep the current behaviour and silence the + warning, explicitly set `return_empty=False` instead of relying on the + default. + """.rstrip() diff --git a/feature_engine/_docstrings/substitute.py b/feature_engine/_docstrings/substitute.py index 490fc6151..b2a1be5e3 100644 --- a/feature_engine/_docstrings/substitute.py +++ b/feature_engine/_docstrings/substitute.py @@ -1,6 +1,6 @@ """Utilities for docstring in Feature-engine. -Taken from the project imbalanced-learn: +Adapted from the project imbalanced-learn: https://github.com/scikit-learn-contrib/imbalanced-learn/blob/ imblearn/utils/_docstring.py#L7 @@ -10,16 +10,22 @@ class Substitution: """Decorate a function's or a class' docstring to perform string substitution on it. + + Only the placeholders whose names are passed as keyword arguments are + replaced, so literal braces in the docstring (e.g., in code examples) + are left untouched. + This decorator should be robust even if obj.__doc__ is None (for example, if -OO was passed to the interpreter). """ - def __init__(self, *args, **kwargs): - if args and kwargs: - raise AssertionError("Only positional or keyword args are allowed") - - self.params = args or kwargs + def __init__(self, **kwargs): + self.params = kwargs def __call__(self, obj): - obj.__doc__ = obj.__doc__.format(**self.params) + doc = obj.__doc__ + if doc: + for key, value in self.params.items(): + doc = doc.replace("{" + key + "}", value) + obj.__doc__ = doc return obj diff --git a/feature_engine/creation/__init__.py b/feature_engine/creation/__init__.py index ede28f4e3..9ac285890 100644 --- a/feature_engine/creation/__init__.py +++ b/feature_engine/creation/__init__.py @@ -2,6 +2,7 @@ The module creation includes classes to create new variables by combination of existing variables in the dataframe. """ + from .cyclical_features import CyclicalFeatures from .decision_tree_features import DecisionTreeFeatures from .geo_features import GeoDistanceFeatures diff --git a/feature_engine/creation/cyclical_features.py b/feature_engine/creation/cyclical_features.py index 40e96cab7..109e452ce 100644 --- a/feature_engine/creation/cyclical_features.py +++ b/feature_engine/creation/cyclical_features.py @@ -10,6 +10,7 @@ ) from feature_engine._check_init_parameters.check_init_input_params import ( _check_param_drop_original, + _check_return_empty_is_bool, ) from feature_engine._check_init_parameters.check_input_dictionary import ( _check_numerical_dict, @@ -22,8 +23,9 @@ _n_features_in_docstring, _variables_attribute_docstring, ) -from feature_engine._docstrings.init_parameters.all_trasnformers import ( +from feature_engine._docstrings.init_parameters.all_transformers import ( _drop_original_docstring, + _return_empty_docstring, _variables_numerical_docstring, ) from feature_engine._docstrings.methods import ( @@ -36,6 +38,7 @@ @Substitution( variables=_variables_numerical_docstring, drop_original=_drop_original_docstring, + return_empty=_return_empty_docstring, variables_=_variables_attribute_docstring, feature_names_in_=_feature_names_in_docstring, n_features_in_=_n_features_in_docstring, @@ -66,6 +69,8 @@ class CyclicalFeatures( ---------- {variables} + {return_empty} + max_values: dict, default=None A dictionary with the maximum value of each variable to transform. Useful when the maximum value is not present in the dataset. If None, the transformer will @@ -122,14 +127,17 @@ class CyclicalFeatures( def __init__( self, variables: Union[None, int, str, List[Union[str, int]]] = None, + return_empty: bool = False, max_values: Optional[Dict[str, Union[int, float]]] = None, drop_original: Optional[bool] = False, ) -> None: _check_numerical_dict(max_values) _check_param_drop_original(drop_original) + _check_return_empty_is_bool(return_empty) self.variables = _check_variables_input_value(variables) + self.return_empty = return_empty self.max_values = max_values self.drop_original = drop_original diff --git a/feature_engine/creation/decision_tree_features.py b/feature_engine/creation/decision_tree_features.py index 8ec2030aa..266c9700e 100644 --- a/feature_engine/creation/decision_tree_features.py +++ b/feature_engine/creation/decision_tree_features.py @@ -13,6 +13,7 @@ from feature_engine._check_init_parameters.check_init_input_params import ( _check_param_drop_original, _check_param_missing_values, + _check_return_empty_is_bool, ) from feature_engine._check_init_parameters.check_variables import ( _check_variables_input_value, @@ -22,9 +23,10 @@ _n_features_in_docstring, _variables_attribute_docstring, ) -from feature_engine._docstrings.init_parameters.all_trasnformers import ( +from feature_engine._docstrings.init_parameters.all_transformers import ( _drop_original_docstring, _missing_values_docstring, + _return_empty_docstring, _variables_numerical_docstring, ) from feature_engine._docstrings.init_parameters.creation import _features_to_combine @@ -52,6 +54,7 @@ features_to_combine=_features_to_combine, missing_values=_missing_values_docstring, drop_original=_drop_original_docstring, + return_empty=_return_empty_docstring, variables_=_variables_attribute_docstring, feature_names_in_=_feature_names_in_docstring, n_features_in_=_n_features_in_docstring, @@ -80,6 +83,8 @@ class DecisionTreeFeatures(TransformerMixin, BaseEstimator, GetFeatureNamesOutMi ---------- {variables} + {return_empty} + {features_to_combine} precision: int, default=None @@ -210,6 +215,7 @@ class DecisionTreeFeatures(TransformerMixin, BaseEstimator, GetFeatureNamesOutMi def __init__( self, variables: Union[None, int, str, List[Union[str, int]]] = None, + return_empty: bool = False, features_to_combine: Optional[Union[Iterable[Any], int]] = None, precision: Union[int, None] = None, cv=3, @@ -232,10 +238,12 @@ def __init__( f"regression must be a boolean value. Got {regression} instead." ) + _check_return_empty_is_bool(return_empty) _check_param_missing_values(missing_values) _check_param_drop_original(drop_original) self.variables = _check_variables_input_value(variables) + self.return_empty = return_empty self.features_to_combine = features_to_combine self.precision = precision self.cv = cv @@ -276,7 +284,7 @@ def fit(self, X: pd.DataFrame, y: pd.Series): # find or check for numerical variables if self.variables is None: - variables_ = find_numerical_variables(X) + variables_ = find_numerical_variables(X, return_empty=self.return_empty) else: variables_ = check_numerical_variables(X, self.variables) diff --git a/feature_engine/creation/math_features.py b/feature_engine/creation/math_features.py index 35cbe73aa..7013568fd 100644 --- a/feature_engine/creation/math_features.py +++ b/feature_engine/creation/math_features.py @@ -1,5 +1,6 @@ from typing import Any, List, Optional, Union +import numpy as np import pandas as pd from feature_engine._docstrings.fit_attributes import ( @@ -7,7 +8,7 @@ _n_features_in_docstring, _variables_attribute_docstring, ) -from feature_engine._docstrings.init_parameters.all_trasnformers import ( +from feature_engine._docstrings.init_parameters.all_transformers import ( _drop_original_docstring, _missing_values_docstring, ) @@ -19,6 +20,26 @@ from feature_engine._docstrings.substitute import Substitution from feature_engine.creation.base_creation import BaseCreation +_PANDAS_LT_3 = int(pd.__version__.split(".")[0]) < 3 + +# In pandas < 3, agg() maps these callables to the pandas methods and warns that +# this will change; the string alias keeps that behavior (e.g., np.std -> +# Series.std with ddof=1) without the warning. In pandas >= 3 the callables are +# used directly (np.std applies ddof=0), so they must not be aliased. +_FUNC_TO_STRING_ALIAS = { + sum: "sum", + min: "min", + max: "max", + np.sum: "sum", + np.mean: "mean", + np.std: "std", + np.var: "var", + np.median: "median", + np.min: "min", + np.max: "max", + np.prod: "prod", +} + @Substitution( missing_values=_missing_values_docstring, @@ -206,10 +227,17 @@ def transform(self, X: pd.DataFrame) -> pd.DataFrame: new_variable_names = self._get_new_features_name() + func = self.func + if _PANDAS_LT_3: + if isinstance(func, list): + func = [_FUNC_TO_STRING_ALIAS.get(fun, fun) for fun in func] + else: + func = _FUNC_TO_STRING_ALIAS.get(func, func) + if len(new_variable_names) == 1: - X[new_variable_names[0]] = X[self.variables].agg(self.func, axis=1) + X[new_variable_names[0]] = X[self.variables].agg(func, axis=1) else: - X[new_variable_names] = X[self.variables].agg(self.func, axis=1) + X[new_variable_names] = X[self.variables].agg(func, axis=1) if self.drop_original: X.drop(columns=self.variables, inplace=True) diff --git a/feature_engine/creation/relative_features.py b/feature_engine/creation/relative_features.py index 54608962d..70d541c11 100644 --- a/feature_engine/creation/relative_features.py +++ b/feature_engine/creation/relative_features.py @@ -7,7 +7,7 @@ _n_features_in_docstring, _variables_attribute_docstring, ) -from feature_engine._docstrings.init_parameters.all_trasnformers import ( +from feature_engine._docstrings.init_parameters.all_transformers import ( _drop_original_docstring, _missing_values_docstring, _variables_numerical_docstring, diff --git a/feature_engine/datetime/datetime.py b/feature_engine/datetime/datetime.py index acb096fb3..b0b1591bd 100644 --- a/feature_engine/datetime/datetime.py +++ b/feature_engine/datetime/datetime.py @@ -9,6 +9,9 @@ from sklearn.utils.validation import check_is_fitted from feature_engine._base_transformers.mixins import GetFeatureNamesOutMixin +from feature_engine._check_init_parameters.check_init_input_params import ( + _check_return_empty_is_bool, +) from feature_engine._check_init_parameters.check_variables import ( _check_variables_input_value, ) @@ -16,6 +19,9 @@ _feature_names_in_docstring, _n_features_in_docstring, ) +from feature_engine._docstrings.init_parameters.all_transformers import ( + _return_empty_docstring, +) from feature_engine._docstrings.methods import ( _fit_not_learn_docstring, _fit_transform_docstring, @@ -40,6 +46,7 @@ @Substitution( + return_empty=_return_empty_docstring, feature_names_in_=_feature_names_in_docstring, n_features_in_=_n_features_in_docstring, fit=_fit_not_learn_docstring, @@ -88,6 +95,8 @@ class DatetimeFeatures(TransformerMixin, BaseEstimator, GetFeatureNamesOutMixin) If "index", the transformer will extract datetime features from the index of the dataframe. + {return_empty} + features_to_extract: list, default=None The list of date features to extract. If None, the following features will be extracted: "month", "year", "day_of_week", "day_of_month", "hour", @@ -178,6 +187,7 @@ class DatetimeFeatures(TransformerMixin, BaseEstimator, GetFeatureNamesOutMixin) def __init__( self, variables: Union[None, int, str, List[Union[str, int]]] = None, + return_empty: bool = False, features_to_extract: Union[None, str, List[str]] = None, drop_original: bool = True, missing_values: str = "raise", @@ -218,7 +228,10 @@ def __init__( if utc is not None and not isinstance(utc, bool): raise ValueError("utc takes only booleans or None. " f"Got {utc} instead.") + _check_return_empty_is_bool(return_empty) + self.variables = _check_variables_input_value(variables) + self.return_empty = return_empty self.drop_original = drop_original self.missing_values = missing_values self.dayfirst = dayfirst @@ -260,7 +273,9 @@ def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): self.variables_ = [] elif self.variables is None: - self.variables_ = find_datetime_variables(X) + self.variables_ = find_datetime_variables( + X, return_empty=self.return_empty + ) else: self.variables_ = check_datetime_variables(X, self.variables) @@ -341,6 +356,9 @@ def transform(self, X: pd.DataFrame) -> pd.DataFrame: if self.missing_values == "raise": _check_contains_na(X, self.variables_) + if len(self.variables_) == 0: + return X + # convert datetime variables datetime_df = pd.concat( [ diff --git a/feature_engine/datetime/datetime_ordinal.py b/feature_engine/datetime/datetime_ordinal.py index 981251488..4717cd2fa 100644 --- a/feature_engine/datetime/datetime_ordinal.py +++ b/feature_engine/datetime/datetime_ordinal.py @@ -6,6 +6,9 @@ from sklearn.utils.validation import check_is_fitted from feature_engine._base_transformers.mixins import GetFeatureNamesOutMixin +from feature_engine._check_init_parameters.check_init_input_params import ( + _check_return_empty_is_bool, +) from feature_engine._check_init_parameters.check_variables import ( _check_variables_input_value, ) @@ -13,6 +16,9 @@ _feature_names_in_docstring, _n_features_in_docstring, ) +from feature_engine._docstrings.init_parameters.all_transformers import ( + _return_empty_docstring, +) from feature_engine._docstrings.methods import ( _fit_not_learn_docstring, _fit_transform_docstring, @@ -28,6 +34,7 @@ @Substitution( + return_empty=_return_empty_docstring, feature_names_in_=_feature_names_in_docstring, n_features_in_=_n_features_in_docstring, fit=_fit_not_learn_docstring, @@ -51,6 +58,8 @@ class DatetimeOrdinal(TransformerMixin, BaseEstimator, GetFeatureNamesOutMixin): find and select all datetime variables, including variables of type object that can be converted to datetime. + {return_empty} + missing_values: string, default='raise' Indicates if missing values should be ignored or raised. If 'raise' the transformer will return an error if the datasets passed to `fit` or `transform` @@ -112,6 +121,7 @@ class DatetimeOrdinal(TransformerMixin, BaseEstimator, GetFeatureNamesOutMixin): def __init__( self, variables: Union[None, int, str, List[Union[str, int]]] = None, + return_empty: bool = False, missing_values: str = "raise", start_date: Union[None, str, datetime.datetime] = None, drop_original: bool = True, @@ -140,7 +150,10 @@ def __init__( f"Got {drop_original} instead." ) + _check_return_empty_is_bool(return_empty) + self.variables = _check_variables_input_value(variables) + self.return_empty = return_empty self.missing_values = missing_values self.drop_original = drop_original @@ -164,7 +177,9 @@ def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): X = check_X(X) if self.variables is None: - self.variables_ = find_datetime_variables(X) + self.variables_ = find_datetime_variables( + X, return_empty=self.return_empty + ) else: self.variables_ = check_datetime_variables(X, self.variables) @@ -213,6 +228,9 @@ def transform(self, X: pd.DataFrame) -> pd.DataFrame: # reorder variables to match train set X = X[self.feature_names_in_] + if len(self.variables_) == 0: + return X + # create a copy(to protect original data) X_new = X.copy() diff --git a/feature_engine/datetime/datetime_subtraction.py b/feature_engine/datetime/datetime_subtraction.py index cd4472cca..0286d91e4 100644 --- a/feature_engine/datetime/datetime_subtraction.py +++ b/feature_engine/datetime/datetime_subtraction.py @@ -5,6 +5,9 @@ from pandas.api.types import is_datetime64_any_dtype as is_datetime from sklearn.utils.validation import check_is_fitted +from feature_engine._check_init_parameters.check_init_input_params import ( + _check_return_empty_is_bool, +) from feature_engine._check_init_parameters.check_variables import ( _check_variables_input_value, ) @@ -12,8 +15,9 @@ _feature_names_in_docstring, _n_features_in_docstring, ) -from feature_engine._docstrings.init_parameters.all_trasnformers import ( +from feature_engine._docstrings.init_parameters.all_transformers import ( _missing_values_docstring, + _return_empty_docstring, ) from feature_engine._docstrings.methods import ( _fit_not_learn_docstring, @@ -48,6 +52,7 @@ @Substitution( missing_values=_missing_values_docstring, + return_empty=_return_empty_docstring, feature_names_in_=_feature_names_in_docstring, n_features_in_=_n_features_in_docstring, fit=_fit_not_learn_docstring, @@ -77,6 +82,8 @@ class DatetimeSubtraction(BaseCreation): The list of datetime reference variables that will be subtracted from `variables` (right side of the subtraction operation). + {return_empty} + new_variables_names: list, default=None Names of the new variables. You have the option to pass a list with the names you'd like to assing to the new variables. If `None`, the transformer will @@ -154,6 +161,7 @@ def __init__( self, variables: Union[None, int, str, List[Union[str, int]]] = None, reference: Union[None, int, str, List[Union[str, int]]] = None, + return_empty: bool = False, new_variables_names: Union[None, List[str], str] = None, output_unit: str = "D", missing_values: str = "ignore", @@ -198,9 +206,12 @@ def __init__( f"Got {new_variables_names} instead." ) + _check_return_empty_is_bool(return_empty) + super().__init__(missing_values, drop_original) self.variables = _check_variables_input_value(variables) self.reference = _check_variables_input_value(reference) + self.return_empty = return_empty self.new_variables_names = new_variables_names self.output_unit = output_unit self.dayfirst = dayfirst @@ -226,12 +237,16 @@ def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): # check variables are datetime if self.variables is None: - self.variables_ = find_datetime_variables(X) + self.variables_ = find_datetime_variables( + X, return_empty=self.return_empty + ) else: self.variables_ = check_datetime_variables(X, self.variables) if self.reference is None: - self.reference_ = find_datetime_variables(X) + self.reference_ = find_datetime_variables( + X, return_empty=self.return_empty + ) else: self.reference_ = check_datetime_variables(X, self.reference) diff --git a/feature_engine/discretisation/decision_tree.py b/feature_engine/discretisation/decision_tree.py index af691e4aa..b92ab37f2 100644 --- a/feature_engine/discretisation/decision_tree.py +++ b/feature_engine/discretisation/decision_tree.py @@ -10,6 +10,9 @@ from sklearn.utils.multiclass import check_classification_targets, type_of_target from feature_engine._base_transformers.base_numerical import BaseNumericalTransformer +from feature_engine._check_init_parameters.check_init_input_params import ( + _check_return_empty_is_bool, +) from feature_engine._check_init_parameters.check_variables import ( _check_variables_input_value, ) @@ -18,7 +21,8 @@ _n_features_in_docstring, _variables_attribute_docstring, ) -from feature_engine._docstrings.init_parameters.all_trasnformers import ( +from feature_engine._docstrings.init_parameters.all_transformers import ( + _return_empty_docstring, _variables_numerical_docstring, ) from feature_engine._docstrings.methods import _fit_transform_docstring @@ -32,6 +36,7 @@ feature_names_in_=_feature_names_in_docstring, n_features_in_=_n_features_in_docstring, fit_transform=_fit_transform_docstring, + return_empty=_return_empty_docstring, ) class DecisionTreeDiscretiser(BaseNumericalTransformer): """ @@ -57,6 +62,8 @@ class DecisionTreeDiscretiser(BaseNumericalTransformer): ---------- {variables} + {return_empty} + bin_output: str, default = "prediction" Whether to return the predictions of the tree, the bin number, or the interval boundaries. Takes values "prediction", "bin_number" and "boundaries", @@ -174,6 +181,7 @@ class DecisionTreeDiscretiser(BaseNumericalTransformer): def __init__( self, variables: Union[None, int, str, List[Union[str, int]]] = None, + return_empty: bool = False, bin_output: str = "prediction", precision: Union[int, None] = None, cv=3, @@ -205,6 +213,8 @@ def __init__( f"regression can only take True or False. Got {regression} instead." ) + _check_return_empty_is_bool(return_empty) + self.bin_output = bin_output self.precision = precision self.cv = cv @@ -213,6 +223,7 @@ def __init__( self.variables = _check_variables_input_value(variables) self.param_grid = param_grid self.random_state = random_state + self.return_empty = return_empty def fit(self, X: pd.DataFrame, y: pd.Series): # type: ignore """ diff --git a/feature_engine/discretisation/equal_frequency.py b/feature_engine/discretisation/equal_frequency.py index 9060f1d49..4993e5814 100644 --- a/feature_engine/discretisation/equal_frequency.py +++ b/feature_engine/discretisation/equal_frequency.py @@ -5,6 +5,9 @@ import pandas as pd +from feature_engine._check_init_parameters.check_init_input_params import ( + _check_return_empty_is_bool, +) from feature_engine._check_init_parameters.check_variables import ( _check_variables_input_value, ) @@ -14,7 +17,8 @@ _n_features_in_docstring, _variables_attribute_docstring, ) -from feature_engine._docstrings.init_parameters.all_trasnformers import ( +from feature_engine._docstrings.init_parameters.all_transformers import ( + _return_empty_docstring, _variables_numerical_docstring, ) from feature_engine._docstrings.init_parameters.discretisers import ( @@ -43,6 +47,7 @@ feature_names_in_=_feature_names_in_docstring, n_features_in_=_n_features_in_docstring, fit_transform=_fit_transform_docstring, + return_empty=_return_empty_docstring, ) class EqualFrequencyDiscretiser(BaseDiscretiser): """ @@ -64,6 +69,8 @@ class EqualFrequencyDiscretiser(BaseDiscretiser): ---------- {variables} + {return_empty} + q: int, default=10 Desired number of equal frequency intervals / bins. @@ -131,6 +138,7 @@ class EqualFrequencyDiscretiser(BaseDiscretiser): def __init__( self, variables: Union[None, int, str, List[Union[str, int]]] = None, + return_empty: bool = False, q: int = 10, return_object: bool = False, return_boundaries: bool = False, @@ -140,10 +148,13 @@ def __init__( if not isinstance(q, int): raise ValueError(f"q must be an integer. Got {q} instead.") + _check_return_empty_is_bool(return_empty) + super().__init__(return_object, return_boundaries, precision) - self.q = q self.variables = _check_variables_input_value(variables) + self.return_empty = return_empty + self.q = q def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): """ diff --git a/feature_engine/discretisation/equal_width.py b/feature_engine/discretisation/equal_width.py index 03787835d..2a7b9c02a 100644 --- a/feature_engine/discretisation/equal_width.py +++ b/feature_engine/discretisation/equal_width.py @@ -5,6 +5,9 @@ import pandas as pd +from feature_engine._check_init_parameters.check_init_input_params import ( + _check_return_empty_is_bool, +) from feature_engine._check_init_parameters.check_variables import ( _check_variables_input_value, ) @@ -14,7 +17,8 @@ _n_features_in_docstring, _variables_attribute_docstring, ) -from feature_engine._docstrings.init_parameters.all_trasnformers import ( +from feature_engine._docstrings.init_parameters.all_transformers import ( + _return_empty_docstring, _variables_numerical_docstring, ) from feature_engine._docstrings.init_parameters.discretisers import ( @@ -43,6 +47,7 @@ feature_names_in_=_feature_names_in_docstring, n_features_in_=_n_features_in_docstring, fit_transform=_fit_transform_docstring, + return_empty=_return_empty_docstring, ) class EqualWidthDiscretiser(BaseDiscretiser): """ @@ -72,6 +77,8 @@ class EqualWidthDiscretiser(BaseDiscretiser): ---------- {variables} + {return_empty} + bins: int, default=10 Desired number of equal width intervals / bins. @@ -139,6 +146,7 @@ class EqualWidthDiscretiser(BaseDiscretiser): def __init__( self, variables: Union[None, int, str, List[Union[str, int]]] = None, + return_empty: bool = False, bins: int = 10, return_object: bool = False, return_boundaries: bool = False, @@ -148,10 +156,13 @@ def __init__( if not isinstance(bins, int): raise ValueError(f"bins must be an integer. Got {bins} instead.") + _check_return_empty_is_bool(return_empty) + super().__init__(return_object, return_boundaries, precision) - self.bins = bins self.variables = _check_variables_input_value(variables) + self.return_empty = return_empty + self.bins = bins def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): """ diff --git a/feature_engine/discretisation/geometric_width.py b/feature_engine/discretisation/geometric_width.py index 9f7c37d21..e20139aac 100644 --- a/feature_engine/discretisation/geometric_width.py +++ b/feature_engine/discretisation/geometric_width.py @@ -3,6 +3,9 @@ import numpy as np import pandas as pd +from feature_engine._check_init_parameters.check_init_input_params import ( + _check_return_empty_is_bool, +) from feature_engine._check_init_parameters.check_variables import ( _check_variables_input_value, ) @@ -12,7 +15,8 @@ _n_features_in_docstring, _variables_attribute_docstring, ) -from feature_engine._docstrings.init_parameters.all_trasnformers import ( +from feature_engine._docstrings.init_parameters.all_transformers import ( + _return_empty_docstring, _variables_numerical_docstring, ) from feature_engine._docstrings.init_parameters.discretisers import ( @@ -43,6 +47,7 @@ fit_transform=_fit_transform_docstring, power="{1/n}", subindex="{i+1}", + return_empty=_return_empty_docstring, ) class GeometricWidthDiscretiser(BaseDiscretiser): """ @@ -87,6 +92,8 @@ class GeometricWidthDiscretiser(BaseDiscretiser): ---------- {variables} + {return_empty} + bins: int, default=10 Desired number of intervals / bins. @@ -130,6 +137,7 @@ class GeometricWidthDiscretiser(BaseDiscretiser): def __init__( self, variables: Union[None, int, str, List[Union[str, int]]] = None, + return_empty: bool = False, bins: int = 10, return_object: bool = False, return_boundaries: bool = False, @@ -139,10 +147,13 @@ def __init__( if not isinstance(bins, int): raise ValueError(f"bins must be an integer. Got {bins} instead.") + _check_return_empty_is_bool(return_empty) + super().__init__(return_object, return_boundaries, precision) - self.bins = bins self.variables = _check_variables_input_value(variables) + self.return_empty = return_empty + self.bins = bins def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): """ diff --git a/feature_engine/encoding/base_encoder.py b/feature_engine/encoding/base_encoder.py index b4ae3478f..35427f260 100644 --- a/feature_engine/encoding/base_encoder.py +++ b/feature_engine/encoding/base_encoder.py @@ -6,11 +6,15 @@ from sklearn.utils.validation import check_is_fitted from feature_engine._base_transformers.mixins import GetFeatureNamesOutMixin +from feature_engine._check_init_parameters.check_init_input_params import ( + _check_return_empty_is_bool, +) from feature_engine._check_init_parameters.check_variables import ( _check_variables_input_value, ) -from feature_engine._docstrings.init_parameters.all_trasnformers import ( +from feature_engine._docstrings.init_parameters.all_transformers import ( _missing_values_docstring, + _return_empty_docstring, _variables_categorical_docstring, ) from feature_engine._docstrings.init_parameters.encoders import _ignore_format_docstring @@ -32,6 +36,7 @@ @Substitution( ignore_format=_ignore_format_docstring, variables=_variables_categorical_docstring, + return_empty=_return_empty_docstring, ) class CategoricalInitMixin: """Shared initialization parameters across transformers. Sets and checks init @@ -39,7 +44,9 @@ class CategoricalInitMixin: Parameters ---------- - {variables}. + {variables} + + {return_empty} {ignore_format} """ @@ -47,6 +54,7 @@ class CategoricalInitMixin: def __init__( self, variables: Union[None, int, str, List[Union[str, int]]] = None, + return_empty: bool = False, ignore_format: bool = False, ) -> None: @@ -56,7 +64,10 @@ def __init__( f"Got {ignore_format} instead." ) + _check_return_empty_is_bool(return_empty) + self.variables = _check_variables_input_value(variables) + self.return_empty = return_empty self.ignore_format = ignore_format @@ -140,7 +151,9 @@ def _check_or_select_variables(self, X: pd.DataFrame): variables_ = check_all_variables(X, self.variables) else: if self.variables is None: - variables_ = find_categorical_variables(X) + variables_ = find_categorical_variables( + X, return_empty=self.return_empty + ) else: variables_ = check_categorical_variables(X, self.variables) diff --git a/feature_engine/encoding/count_frequency.py b/feature_engine/encoding/count_frequency.py index ae6507627..114781f29 100644 --- a/feature_engine/encoding/count_frequency.py +++ b/feature_engine/encoding/count_frequency.py @@ -5,13 +5,17 @@ import pandas as pd +from feature_engine._check_init_parameters.check_init_input_params import ( + _check_return_empty_is_bool, +) from feature_engine._docstrings.fit_attributes import ( _feature_names_in_docstring, _n_features_in_docstring, _variables_attribute_docstring, ) -from feature_engine._docstrings.init_parameters.all_trasnformers import ( +from feature_engine._docstrings.init_parameters.all_transformers import ( _missing_values_docstring, + _return_empty_docstring, _variables_categorical_docstring, ) from feature_engine._docstrings.init_parameters.encoders import ( @@ -41,6 +45,7 @@ ignore_format=_ignore_format_docstring, missing_values=_missing_values_docstring, variables=_variables_categorical_docstring, + return_empty=_return_empty_docstring, unseen=_unseen_docstring, variables_=_variables_attribute_docstring, feature_names_in_=_feature_names_in_docstring, @@ -84,6 +89,8 @@ class CountFrequencyEncoder(CategoricalMethodsMixin, CategoricalInitMixinNA): {variables} + {return_empty} + {missing_values} {ignore_format} @@ -155,6 +162,7 @@ def __init__( self, encoding_method: str = "count", variables: Union[None, int, str, List[Union[str, int]]] = None, + return_empty: bool = False, missing_values: str = "raise", ignore_format: bool = False, unseen: str = "ignore", @@ -167,9 +175,12 @@ def __init__( ) check_parameter_unseen(unseen, ["ignore", "raise", "encode"]) + _check_return_empty_is_bool(return_empty) + super().__init__(variables, missing_values, ignore_format) self.encoding_method = encoding_method self.unseen = unseen + self.return_empty = return_empty def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): """ diff --git a/feature_engine/encoding/decision_tree.py b/feature_engine/encoding/decision_tree.py index 63b5edbac..1cc45b40b 100644 --- a/feature_engine/encoding/decision_tree.py +++ b/feature_engine/encoding/decision_tree.py @@ -13,7 +13,8 @@ _n_features_in_docstring, _variables_attribute_docstring, ) -from feature_engine._docstrings.init_parameters.all_trasnformers import ( +from feature_engine._docstrings.init_parameters.all_transformers import ( + _return_empty_docstring, _variables_categorical_docstring, ) from feature_engine._docstrings.init_parameters.encoders import ( @@ -35,7 +36,6 @@ from feature_engine.encoding.ordinal import OrdinalEncoder from feature_engine.tags import _return_tags - _unseen_docstring = ( _unseen_docstring + """ If `'encode'` unseen categories will be encoded as `fill_value`.""" @@ -45,6 +45,7 @@ @Substitution( ignore_format=_ignore_format_docstring, variables=_variables_categorical_docstring, + return_empty=_return_empty_docstring, variables_=_variables_attribute_docstring, unseen=_unseen_docstring, feature_names_in_=_feature_names_in_docstring, @@ -125,6 +126,8 @@ class DecisionTreeEncoder(CategoricalMethodsMixin, CategoricalInitMixin): {variables} + {return_empty} + {ignore_format} precision: int, default=None @@ -220,6 +223,7 @@ def __init__( regression: bool = True, random_state: Optional[int] = None, variables: Union[None, int, str, List[Union[str, int]]] = None, + return_empty: bool = False, ignore_format: bool = False, precision: Optional[int] = None, unseen: str = "ignore", @@ -247,7 +251,7 @@ def __init__( ) check_parameter_unseen(unseen, ["ignore", "raise", "encode"]) - super().__init__(variables, ignore_format) + super().__init__(variables, return_empty, ignore_format) self.encoding_method = encoding_method self.cv = cv self.scoring = scoring @@ -291,6 +295,14 @@ def fit(self, X: pd.DataFrame, y: pd.Series): param_grid = self._assign_param_grid() + # if the list of variables to transform is empty, we + # stop the logic. + if isinstance(variables_, list) and len(variables_) == 0: + self.encoder_dict_ = {} + self.variables_ = variables_ + self._get_feature_names_in(X) + return self + encoder = OrdinalEncoder( encoding_method=self.encoding_method, variables=variables_, diff --git a/feature_engine/encoding/mean_encoding.py b/feature_engine/encoding/mean_encoding.py index bdcf160d4..bb90b7e2c 100644 --- a/feature_engine/encoding/mean_encoding.py +++ b/feature_engine/encoding/mean_encoding.py @@ -4,13 +4,17 @@ import pandas as pd +from feature_engine._check_init_parameters.check_init_input_params import ( + _check_return_empty_is_bool, +) from feature_engine._docstrings.fit_attributes import ( _feature_names_in_docstring, _n_features_in_docstring, _variables_attribute_docstring, ) -from feature_engine._docstrings.init_parameters.all_trasnformers import ( +from feature_engine._docstrings.init_parameters.all_transformers import ( _missing_values_docstring, + _return_empty_docstring, _variables_categorical_docstring, ) from feature_engine._docstrings.init_parameters.encoders import ( @@ -40,6 +44,7 @@ missing_values=_missing_values_docstring, ignore_format=_ignore_format_docstring, variables=_variables_categorical_docstring, + return_empty=_return_empty_docstring, unseen=_unseen_docstring, variables_=_variables_attribute_docstring, feature_names_in_=_feature_names_in_docstring, @@ -95,6 +100,8 @@ class MeanEncoder(CategoricalMethodsMixin, CategoricalInitMixinNA): ---------- {variables} + {return_empty} + {missing_values} {ignore_format} @@ -173,12 +180,16 @@ class MeanEncoder(CategoricalMethodsMixin, CategoricalInitMixinNA): def __init__( self, variables: Union[None, int, str, List[Union[str, int]]] = None, + return_empty: bool = False, missing_values: str = "raise", ignore_format: bool = False, unseen: str = "ignore", smoothing: Union[int, float, str] = 0.0, ) -> None: + _check_return_empty_is_bool(return_empty) + super().__init__(variables, missing_values, ignore_format) + self.return_empty = return_empty if ( not isinstance(smoothing, (str, float, int)) or isinstance(smoothing, str) diff --git a/feature_engine/encoding/one_hot.py b/feature_engine/encoding/one_hot.py index e94432a3d..cb61dc503 100644 --- a/feature_engine/encoding/one_hot.py +++ b/feature_engine/encoding/one_hot.py @@ -11,7 +11,8 @@ _n_features_in_docstring, _variables_attribute_docstring, ) -from feature_engine._docstrings.init_parameters.all_trasnformers import ( +from feature_engine._docstrings.init_parameters.all_transformers import ( + _return_empty_docstring, _variables_categorical_docstring, ) from feature_engine._docstrings.init_parameters.encoders import _ignore_format_docstring @@ -27,6 +28,7 @@ @Substitution( ignore_format=_ignore_format_docstring, variables=_variables_categorical_docstring, + return_empty=_return_empty_docstring, variables_=_variables_attribute_docstring, feature_names_in_=_feature_names_in_docstring, n_features_in_=_n_features_in_docstring, @@ -96,8 +98,11 @@ class OneHotEncoder(CategoricalMethodsMixin, CategoricalInitMixin): {variables} + {return_empty} + {ignore_format} + Attributes ---------- encoder_dict_: @@ -163,6 +168,7 @@ def __init__( drop_last: bool = False, drop_last_binary: bool = False, variables: Union[None, int, str, List[Union[str, int]]] = None, + return_empty: bool = False, ignore_format: bool = False, ) -> None: @@ -185,7 +191,7 @@ def __init__( f"Got {drop_last_binary} instead." ) - super().__init__(variables, ignore_format) + super().__init__(variables, return_empty, ignore_format) self.top_categories = top_categories self.drop_last = drop_last self.drop_last_binary = drop_last_binary diff --git a/feature_engine/encoding/ordinal.py b/feature_engine/encoding/ordinal.py index bff179e22..156857016 100644 --- a/feature_engine/encoding/ordinal.py +++ b/feature_engine/encoding/ordinal.py @@ -5,13 +5,17 @@ import pandas as pd +from feature_engine._check_init_parameters.check_init_input_params import ( + _check_return_empty_is_bool, +) from feature_engine._docstrings.fit_attributes import ( _feature_names_in_docstring, _n_features_in_docstring, _variables_attribute_docstring, ) -from feature_engine._docstrings.init_parameters.all_trasnformers import ( +from feature_engine._docstrings.init_parameters.all_transformers import ( _missing_values_docstring, + _return_empty_docstring, _variables_categorical_docstring, ) from feature_engine._docstrings.init_parameters.encoders import ( @@ -40,6 +44,7 @@ missing_values=_missing_values_docstring, ignore_format=_ignore_format_docstring, variables=_variables_categorical_docstring, + return_empty=_return_empty_docstring, unseen=_unseen_docstring, variables_=_variables_attribute_docstring, feature_names_in_=_feature_names_in_docstring, @@ -80,10 +85,13 @@ class OrdinalEncoder(CategoricalMethodsMixin, CategoricalInitMixinNA): {variables} + {return_empty} + {missing_values} {ignore_format} + {unseen} Attributes @@ -163,6 +171,7 @@ def __init__( self, encoding_method: str = "ordered", variables: Union[None, int, str, List[Union[str, int]]] = None, + return_empty: bool = False, missing_values: str = "raise", ignore_format: bool = False, unseen: str = "ignore", @@ -174,9 +183,12 @@ def __init__( ) check_parameter_unseen(unseen, ["ignore", "raise", "encode"]) + _check_return_empty_is_bool(return_empty) + super().__init__(variables, missing_values, ignore_format) self.encoding_method = encoding_method self.unseen = unseen + self.return_empty = return_empty def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): """Learn the numbers to be used to replace the categories in each diff --git a/feature_engine/encoding/rare_label.py b/feature_engine/encoding/rare_label.py index 8a57f9fa2..49fad22e1 100644 --- a/feature_engine/encoding/rare_label.py +++ b/feature_engine/encoding/rare_label.py @@ -7,13 +7,17 @@ import numpy as np import pandas as pd +from feature_engine._check_init_parameters.check_init_input_params import ( + _check_return_empty_is_bool, +) from feature_engine._docstrings.fit_attributes import ( _feature_names_in_docstring, _n_features_in_docstring, _variables_attribute_docstring, ) -from feature_engine._docstrings.init_parameters.all_trasnformers import ( +from feature_engine._docstrings.init_parameters.all_transformers import ( _missing_values_docstring, + _return_empty_docstring, _variables_categorical_docstring, ) from feature_engine._docstrings.init_parameters.encoders import _ignore_format_docstring @@ -30,6 +34,7 @@ missing_values=_missing_values_docstring, ignore_format=_ignore_format_docstring, variables=_variables_categorical_docstring, + return_empty=_return_empty_docstring, variables_=_variables_attribute_docstring, feature_names_in_=_feature_names_in_docstring, n_features_in_=_n_features_in_docstring, @@ -88,6 +93,8 @@ class RareLabelEncoder(CategoricalMethodsMixin, CategoricalInitMixinNA): {variables} + {return_empty} + {missing_values} {ignore_format} @@ -139,6 +146,7 @@ def __init__( max_n_categories: Optional[int] = None, replace_with: Union[str, int, float] = "Rare", variables: Union[None, int, str, List[Union[str, int]]] = None, + return_empty: bool = False, missing_values: str = "raise", ignore_format: bool = False, ) -> None: @@ -169,11 +177,14 @@ def __init__( f"Got {replace_with} instead." ) + _check_return_empty_is_bool(return_empty) + super().__init__(variables, missing_values, ignore_format) self.tol = tol self.n_categories = n_categories self.max_n_categories = max_n_categories self.replace_with = replace_with + self.return_empty = return_empty def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): """ diff --git a/feature_engine/encoding/similarity_encoder.py b/feature_engine/encoding/similarity_encoder.py index b6aa1b249..0e1df945c 100644 --- a/feature_engine/encoding/similarity_encoder.py +++ b/feature_engine/encoding/similarity_encoder.py @@ -10,7 +10,8 @@ _n_features_in_docstring, _variables_attribute_docstring, ) -from feature_engine._docstrings.init_parameters.all_trasnformers import ( +from feature_engine._docstrings.init_parameters.all_transformers import ( + _return_empty_docstring, _variables_categorical_docstring, ) from feature_engine._docstrings.init_parameters.encoders import _ignore_format_docstring @@ -33,6 +34,7 @@ def _gpm_fast(x1: str, x2: str) -> float: @Substitution( ignore_format=_ignore_format_docstring, variables=_variables_categorical_docstring, + return_empty=_return_empty_docstring, variables_=_variables_attribute_docstring, feature_names_in_=_feature_names_in_docstring, n_features_in_=_n_features_in_docstring, @@ -120,6 +122,8 @@ class StringSimilarityEncoder(CategoricalMethodsMixin, CategoricalInitMixin): {variables} + {return_empty} + {ignore_format} Attributes @@ -187,6 +191,7 @@ def __init__( keywords: Optional[dict] = None, missing_values: str = "impute", variables: Union[None, int, str, List[Union[str, int]]] = None, + return_empty: bool = False, ignore_format: bool = False, ): if top_categories and not isinstance(top_categories, int): @@ -207,7 +212,7 @@ def __init__( "The items in keywords should be lists." f" Got {keywords.values()!r} instead." ) - super().__init__(variables, ignore_format) + super().__init__(variables, return_empty, ignore_format) self.top_categories = top_categories self.missing_values = missing_values self.keywords = keywords @@ -314,6 +319,9 @@ def transform(self, X: pd.DataFrame) -> pd.DataFrame: if self.missing_values == "raise": _check_optional_contains_na(X, self.variables_) + if len(self.variables_) == 0: + return X + new_values = [] for var in self.variables_: if self.missing_values == "impute": diff --git a/feature_engine/encoding/woe.py b/feature_engine/encoding/woe.py index 2a803eebc..2837ed9c8 100644 --- a/feature_engine/encoding/woe.py +++ b/feature_engine/encoding/woe.py @@ -11,7 +11,8 @@ _n_features_in_docstring, _variables_attribute_docstring, ) -from feature_engine._docstrings.init_parameters.all_trasnformers import ( +from feature_engine._docstrings.init_parameters.all_transformers import ( + _return_empty_docstring, _variables_categorical_docstring, ) from feature_engine._docstrings.init_parameters.encoders import ( @@ -86,6 +87,7 @@ def _calculate_woe( @Substitution( ignore_format=_ignore_format_docstring, variables=_variables_categorical_docstring, + return_empty=_return_empty_docstring, unseen=_unseen_docstring, variables_=_variables_attribute_docstring, feature_names_in_=_feature_names_in_docstring, @@ -128,6 +130,8 @@ class WoEEncoder(CategoricalMethodsMixin, CategoricalInitMixin, WoE): ---------- {variables} + {return_empty} + {ignore_format} {unseen} @@ -199,12 +203,13 @@ class WoEEncoder(CategoricalMethodsMixin, CategoricalInitMixin, WoE): def __init__( self, variables: Union[None, int, str, List[Union[str, int]]] = None, + return_empty: bool = False, ignore_format: bool = False, unseen: str = "ignore", fill_value: Union[int, float, None] = None, ) -> None: - super().__init__(variables, ignore_format) + super().__init__(variables, return_empty, ignore_format) check_parameter_unseen(unseen, ["ignore", "raise"]) if fill_value is not None and not isinstance(fill_value, (int, float)): raise ValueError( diff --git a/feature_engine/imputation/arbitrary_number.py b/feature_engine/imputation/arbitrary_number.py index 668f391b0..69d7c624b 100644 --- a/feature_engine/imputation/arbitrary_number.py +++ b/feature_engine/imputation/arbitrary_number.py @@ -11,6 +11,9 @@ from feature_engine._check_init_parameters.check_variables import ( _check_variables_input_value, ) +from feature_engine._check_init_parameters.check_init_input_params import ( + _check_return_empty_is_bool +) from feature_engine._docstrings.fit_attributes import ( _feature_names_in_docstring, _imputer_dict_docstring, @@ -22,6 +25,9 @@ _fit_transform_docstring, _transform_imputers_docstring, ) +from feature_engine._docstrings.init_parameters.all_transformers import ( + _return_empty_docstring +) from feature_engine._docstrings.substitute import Substitution from feature_engine.dataframe_checks import check_X from feature_engine.imputation.base_imputer import BaseImputer @@ -34,6 +40,7 @@ @Substitution( imputer_dict_=_imputer_dict_docstring, variables_=_variables_attribute_docstring, + return_empty=_return_empty_docstring, feature_names_in_=_feature_names_in_docstring, n_features_in_=_n_features_in_docstring, fit=_fit_not_learn_docstring, @@ -63,6 +70,8 @@ class ArbitraryNumberImputer(BaseImputer): select all numerical variables. This parameter is used only if `imputer_dict` is None. + {return_empty} + imputer_dict: dict, default=None The dictionary of variables and the arbitrary numbers for their imputation. If specified, it overrides the above parameters. @@ -70,7 +79,6 @@ class ArbitraryNumberImputer(BaseImputer): Attributes ---------- - {imputer_dict_} {variables_} @@ -116,6 +124,7 @@ def __init__( self, arbitrary_number: Union[int, float] = 999, variables: Union[None, int, str, List[Union[str, int]]] = None, + return_empty: bool = False, imputer_dict: Optional[dict] = None, ) -> None: @@ -128,6 +137,9 @@ def __init__( self.variables = _check_variables_input_value(variables) + _check_return_empty_is_bool(return_empty) + self.return_empty = return_empty + self.imputer_dict = imputer_dict def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): @@ -155,7 +167,7 @@ def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): self.imputer_dict_ = self.imputer_dict else: if self.variables is None: - self.variables_ = find_numerical_variables(X) + self.variables_ = find_numerical_variables(X, self.return_empty) else: self.variables_ = check_numerical_variables(X, self.variables) self.imputer_dict_ = {var: self.arbitrary_number for var in self.variables_} diff --git a/feature_engine/imputation/base_imputer.py b/feature_engine/imputation/base_imputer.py index a1389adb8..f9c3a2fea 100644 --- a/feature_engine/imputation/base_imputer.py +++ b/feature_engine/imputation/base_imputer.py @@ -6,6 +6,8 @@ from feature_engine.dataframe_checks import _check_X_matches_training_df, check_X from feature_engine.tags import _return_tags +_PANDAS_LT_3 = int(pd.__version__.split(".")[0]) < 3 + class BaseImputer(TransformerMixin, BaseEstimator, GetFeatureNamesOutMixin): """shared set-up checks and methods across imputers""" @@ -59,9 +61,15 @@ def transform(self, X: pd.DataFrame) -> pd.DataFrame: X = self._transform(X) - # Replace missing data with learned parameters - X = X.fillna(value=self.imputer_dict_).infer_objects() - return X + # Replace missing data with learned parameters. In pandas < 3, fillna + # downcasts object columns and warns; the option applies the pandas 3 + # behavior: no downcasting, and infer_objects restores numeric dtypes. + if _PANDAS_LT_3: + with pd.option_context("future.no_silent_downcasting", True): + X = X.fillna(value=self.imputer_dict_) + else: + X = X.fillna(value=self.imputer_dict_) + return X.infer_objects() def _get_feature_names_in(self, X): """Get the names and number of features in the train set (the dataframe diff --git a/feature_engine/imputation/categorical.py b/feature_engine/imputation/categorical.py index 8c4000a0c..9c615ae47 100644 --- a/feature_engine/imputation/categorical.py +++ b/feature_engine/imputation/categorical.py @@ -8,6 +8,9 @@ from feature_engine._check_init_parameters.check_variables import ( _check_variables_input_value, ) +from feature_engine._check_init_parameters.check_init_input_params import ( + _check_return_empty_is_bool +) from feature_engine._docstrings.fit_attributes import ( _feature_names_in_docstring, _imputer_dict_docstring, @@ -18,6 +21,9 @@ _fit_transform_docstring, _transform_imputers_docstring, ) +from feature_engine._docstrings.init_parameters.all_transformers import ( + _return_empty_docstring +) from feature_engine._docstrings.substitute import Substitution from feature_engine.dataframe_checks import check_X from feature_engine.imputation.base_imputer import BaseImputer @@ -33,6 +39,7 @@ @Substitution( imputer_dict_=_imputer_dict_docstring, variables_=_variables_attribute_docstring, + return_empty=_return_empty_docstring, feature_names_in_=_feature_names_in_docstring, n_features_in_=_n_features_in_docstring, transform=_transform_imputers_docstring, @@ -75,6 +82,8 @@ class CategoricalImputer(BaseImputer): default. You can also make the transformer accept numerical variables, see the parameter `ignore_format` below. + {return_empty} + return_object: bool, default=False If working with numerical variables cast as object, decide whether to return the variables as numeric or re-cast them as object. @@ -133,6 +142,7 @@ def __init__( imputation_method: str = "missing", fill_value: Union[str, int, float] = "Missing", variables: Union[None, int, str, List[Union[str, int]]] = None, + return_empty: bool = False, return_object: bool = False, ignore_format: bool = False, ) -> None: @@ -149,6 +159,8 @@ def __init__( self.variables = _check_variables_input_value(variables) self.return_object = return_object self.ignore_format = ignore_format + _check_return_empty_is_bool(return_empty) + self.return_empty = return_empty def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): """ @@ -169,12 +181,12 @@ def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): # select variables to encode if self.ignore_format is True: if self.variables is None: - self.variables_ = find_all_variables(X) + self.variables_ = find_all_variables(X, self.return_empty) else: self.variables_ = check_all_variables(X, self.variables) else: if self.variables is None: - self.variables_ = find_categorical_variables(X) + self.variables_ = find_categorical_variables(X, self.return_empty) else: self.variables_ = check_categorical_variables(X, self.variables) @@ -230,18 +242,13 @@ def transform(self, X: pd.DataFrame) -> pd.DataFrame: # if variable is of type category, we need to add the new # category, before filling in the nan - add_cats = {} for variable in self.variables_: if X[variable].dtype.name == "category": - add_cats.update( - { - variable: X[variable].cat.add_categories( - self.imputer_dict_[variable] - ) - } + X[variable] = X[variable].cat.add_categories( + self.imputer_dict_[variable] ) - X = X.assign(**add_cats).fillna(self.imputer_dict_) + X = X.fillna(self.imputer_dict_) # add additional step to return variables cast as object if self.return_object: diff --git a/feature_engine/imputation/drop_missing_data.py b/feature_engine/imputation/drop_missing_data.py index 07c6f3e75..fa77bb315 100644 --- a/feature_engine/imputation/drop_missing_data.py +++ b/feature_engine/imputation/drop_missing_data.py @@ -9,11 +9,17 @@ from feature_engine._check_init_parameters.check_variables import ( _check_variables_input_value, ) +from feature_engine._check_init_parameters.check_init_input_params import ( + _check_return_empty_is_bool +) from feature_engine._docstrings.fit_attributes import ( _feature_names_in_docstring, _n_features_in_docstring, ) from feature_engine._docstrings.methods import _fit_transform_docstring +from feature_engine._docstrings.init_parameters.all_transformers import ( + _return_empty_docstring +) from feature_engine._docstrings.substitute import Substitution from feature_engine.dataframe_checks import check_X from feature_engine.imputation.base_imputer import BaseImputer @@ -22,6 +28,7 @@ @Substitution( + return_empty=_return_empty_docstring, feature_names_in_=_feature_names_in_docstring, n_features_in_=_n_features_in_docstring, fit_transform=_fit_transform_docstring, @@ -49,6 +56,8 @@ class DropMissingData(BaseImputer, TransformXyMixin): that had missing data in the train set. These might be a subset of the variables indicated in the list. + {return_empty} + missing_only: bool, default=True If `True`, rows will be dropped when they show missing data in variables that had missing data during `fit()`. If `False`, rows will be dropped if there is @@ -112,6 +121,7 @@ def __init__( missing_only: bool = True, threshold: Union[None, int, float] = None, variables: Union[None, int, str, List[Union[str, int]]] = None, + return_empty: bool = False, ) -> None: if not isinstance(missing_only, bool): @@ -131,6 +141,9 @@ def __init__( self.missing_only = missing_only self.threshold = threshold + _check_return_empty_is_bool(return_empty) + self.return_empty = return_empty + def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): """ Find the variables for which missing data should be evaluated to decide if a @@ -150,7 +163,7 @@ def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): # find variables for which indicator should be added if self.variables is None: - self.variables_ = find_all_variables(X) + self.variables_ = find_all_variables(X, self.return_empty) else: self.variables_ = check_all_variables(X, self.variables) diff --git a/feature_engine/imputation/end_tail.py b/feature_engine/imputation/end_tail.py index 59e59f32a..f612e2dab 100644 --- a/feature_engine/imputation/end_tail.py +++ b/feature_engine/imputation/end_tail.py @@ -8,14 +8,17 @@ from feature_engine._check_init_parameters.check_variables import ( _check_variables_input_value, ) +from feature_engine._check_init_parameters.check_init_input_params import ( + _check_return_empty_is_bool +) from feature_engine._docstrings.fit_attributes import ( _feature_names_in_docstring, _imputer_dict_docstring, _n_features_in_docstring, _variables_attribute_docstring, ) -from feature_engine._docstrings.init_parameters.all_trasnformers import ( - _variables_numerical_docstring, +from feature_engine._docstrings.init_parameters.all_transformers import ( + _variables_numerical_docstring, _return_empty_docstring ) from feature_engine._docstrings.methods import ( _fit_transform_docstring, @@ -32,6 +35,7 @@ @Substitution( variables=_variables_numerical_docstring, + return_empty=_return_empty_docstring, imputer_dict_=_imputer_dict_docstring, variables_=_variables_attribute_docstring, feature_names_in_=_feature_names_in_docstring, @@ -99,6 +103,8 @@ class EndTailImputer(BaseImputer): {variables} + {return_empty} + Attributes ---------- {imputer_dict_} @@ -142,6 +148,7 @@ def __init__( tail: str = "right", fold: int = 3, variables: Union[None, int, str, List[Union[str, int]]] = None, + return_empty: bool = False, ) -> None: if imputation_method not in ["gaussian", "iqr", "max"]: @@ -160,6 +167,9 @@ def __init__( self.fold = fold self.variables = _check_variables_input_value(variables) + _check_return_empty_is_bool(return_empty) + self.return_empty = return_empty + def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): """ Learn the values at the end of the variable distribution. @@ -177,7 +187,7 @@ def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): # find or check for numerical variables if self.variables is None: - self.variables_ = find_numerical_variables(X) + self.variables_ = find_numerical_variables(X, self.return_empty) else: self.variables_ = check_numerical_variables(X, self.variables) diff --git a/feature_engine/imputation/mean_median.py b/feature_engine/imputation/mean_median.py index da845e063..997ec2813 100644 --- a/feature_engine/imputation/mean_median.py +++ b/feature_engine/imputation/mean_median.py @@ -8,14 +8,17 @@ from feature_engine._check_init_parameters.check_variables import ( _check_variables_input_value, ) +from feature_engine._check_init_parameters.check_init_input_params import ( + _check_return_empty_is_bool +) from feature_engine._docstrings.fit_attributes import ( _feature_names_in_docstring, _imputer_dict_docstring, _n_features_in_docstring, _variables_attribute_docstring, ) -from feature_engine._docstrings.init_parameters.all_trasnformers import ( - _variables_numerical_docstring, +from feature_engine._docstrings.init_parameters.all_transformers import ( + _variables_numerical_docstring, _return_empty_docstring ) from feature_engine._docstrings.methods import ( _fit_transform_docstring, @@ -32,6 +35,7 @@ @Substitution( variables=_variables_numerical_docstring, + return_empty=_return_empty_docstring, imputer_dict_=_imputer_dict_docstring, variables_=_variables_attribute_docstring, feature_names_in_=_feature_names_in_docstring, @@ -57,6 +61,8 @@ class MeanMedianImputer(BaseImputer): {variables} + {return_empty} + Attributes ---------- {imputer_dict_} @@ -101,6 +107,7 @@ def __init__( self, imputation_method: str = "median", variables: Union[None, int, str, List[Union[str, int]]] = None, + return_empty: bool = False, ) -> None: if imputation_method not in ["median", "mean"]: @@ -109,6 +116,9 @@ def __init__( self.imputation_method = imputation_method self.variables = _check_variables_input_value(variables) + _check_return_empty_is_bool(return_empty) + self.return_empty = return_empty + def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): """ Learn the mean or median values. @@ -127,7 +137,7 @@ def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): # find or check for numerical variables if self.variables is None: - self.variables_ = find_numerical_variables(X) + self.variables_ = find_numerical_variables(X, self.return_empty) else: self.variables_ = check_numerical_variables(X, self.variables) diff --git a/feature_engine/imputation/missing_indicator.py b/feature_engine/imputation/missing_indicator.py index 01660a654..ad75267ac 100644 --- a/feature_engine/imputation/missing_indicator.py +++ b/feature_engine/imputation/missing_indicator.py @@ -8,11 +8,17 @@ from feature_engine._check_init_parameters.check_variables import ( _check_variables_input_value, ) +from feature_engine._check_init_parameters.check_init_input_params import ( + _check_return_empty_is_bool +) from feature_engine._docstrings.fit_attributes import ( _feature_names_in_docstring, _n_features_in_docstring, ) from feature_engine._docstrings.methods import _fit_transform_docstring +from feature_engine._docstrings.init_parameters.all_transformers import ( + _return_empty_docstring +) from feature_engine._docstrings.substitute import Substitution from feature_engine.dataframe_checks import check_X from feature_engine.imputation.base_imputer import BaseImputer @@ -21,6 +27,7 @@ @Substitution( + return_empty=_return_empty_docstring, feature_names_in_=_feature_names_in_docstring, n_features_in_=_n_features_in_docstring, fit_transform=_fit_transform_docstring, @@ -58,6 +65,7 @@ class AddMissingIndicator(BaseImputer): The list of variables to impute. If None, the imputer will find and select all variables. + {return_empty} Attributes ---------- @@ -103,6 +111,7 @@ def __init__( self, missing_only: bool = True, variables: Union[None, int, str, List[Union[str, int]]] = None, + return_empty: bool = False, ) -> None: if not isinstance(missing_only, bool): @@ -111,6 +120,9 @@ def __init__( self.variables = _check_variables_input_value(variables) self.missing_only = missing_only + _check_return_empty_is_bool(return_empty) + self.return_empty = return_empty + def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): """ Learn the variables for which the missing indicators will be created. @@ -129,7 +141,7 @@ def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): # find variables for which indicator should be added if self.variables is None: - self.variables_ = find_all_variables(X) + self.variables_ = find_all_variables(X, self.return_empty) else: self.variables_ = check_all_variables(X, self.variables) diff --git a/feature_engine/imputation/random_sample.py b/feature_engine/imputation/random_sample.py index d05aeaac8..cdc41b27f 100644 --- a/feature_engine/imputation/random_sample.py +++ b/feature_engine/imputation/random_sample.py @@ -9,6 +9,9 @@ from feature_engine._check_init_parameters.check_variables import ( _check_variables_input_value, ) +from feature_engine._check_init_parameters.check_init_input_params import ( + _check_return_empty_is_bool +) from feature_engine._docstrings.fit_attributes import ( _feature_names_in_docstring, _n_features_in_docstring, @@ -18,6 +21,9 @@ _fit_transform_docstring, _transform_imputers_docstring, ) +from feature_engine._docstrings.init_parameters.all_transformers import ( + _return_empty_docstring +) from feature_engine._docstrings.substitute import Substitution from feature_engine.dataframe_checks import check_X from feature_engine.imputation.base_imputer import BaseImputer @@ -43,6 +49,7 @@ def _define_seed( @Substitution( variables_=_variables_attribute_docstring, + return_empty=_return_empty_docstring, feature_names_in_=_feature_names_in_docstring, n_features_in_=_n_features_in_docstring, transform=_transform_imputers_docstring, @@ -69,6 +76,8 @@ class RandomSampleImputer(BaseImputer): The list of variables to be imputed. If None, the imputer will select all variables in the train set. + {return_empty} + random_state: int, str or list, default=None The random_state can take an integer to set the seed when extracting the random samples. Alternatively, it can take a variable name or a list of @@ -135,6 +144,7 @@ class RandomSampleImputer(BaseImputer): def __init__( self, variables: Union[None, int, str, List[Union[str, int]]] = None, + return_empty: bool = False, random_state: Union[None, int, str, List[Union[str, int]]] = None, seed: str = "general", seeding_method: str = "add", @@ -159,6 +169,10 @@ def __init__( ) self.variables = _check_variables_input_value(variables) + + _check_return_empty_is_bool(return_empty) + self.return_empty = return_empty + self.random_state = random_state self.seed = seed self.seeding_method = seeding_method @@ -184,7 +198,7 @@ def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): # find variables to impute if self.variables is None: - self.variables_ = find_all_variables(X) + self.variables_ = find_all_variables(X, self.return_empty) else: self.variables_ = check_all_variables(X, self.variables) diff --git a/feature_engine/outliers/artbitrary.py b/feature_engine/outliers/artbitrary.py index 87ec4a709..75a723e23 100644 --- a/feature_engine/outliers/artbitrary.py +++ b/feature_engine/outliers/artbitrary.py @@ -16,7 +16,7 @@ _right_tail_caps_docstring, _variables_attribute_docstring, ) -from feature_engine._docstrings.init_parameters.all_trasnformers import ( +from feature_engine._docstrings.init_parameters.all_transformers import ( _missing_values_docstring, ) from feature_engine._docstrings.methods import ( diff --git a/feature_engine/outliers/base_outlier.py b/feature_engine/outliers/base_outlier.py index 8f296bcff..83abfc0bf 100644 --- a/feature_engine/outliers/base_outlier.py +++ b/feature_engine/outliers/base_outlier.py @@ -5,6 +5,9 @@ from sklearn.utils.validation import check_is_fitted from feature_engine._base_transformers.mixins import GetFeatureNamesOutMixin +from feature_engine._check_init_parameters.check_init_input_params import ( + _check_return_empty_is_bool, +) from feature_engine._check_init_parameters.check_variables import ( _check_variables_input_value, ) @@ -155,6 +158,7 @@ def __init__( tail: str = "right", fold: Union[int, float, Literal["auto"]] = "auto", variables: Union[None, int, str, List[Union[str, int]]] = None, + return_empty: bool = False, missing_values: str = "raise", ) -> None: @@ -192,10 +196,13 @@ def __init__( f" Got {missing_values} instead." ) + _check_return_empty_is_bool(return_empty) + self.capping_method = capping_method self.tail = tail self.fold = fold self.variables = _check_variables_input_value(variables) + self.return_empty = return_empty self.missing_values = missing_values def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): @@ -216,7 +223,9 @@ def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): # find or check for numerical variables if self.variables is None: - self.variables_ = find_numerical_variables(X) + self.variables_ = find_numerical_variables( + X, return_empty=self.return_empty + ) else: self.variables_ = check_numerical_variables(X, self.variables) diff --git a/feature_engine/outliers/trimmer.py b/feature_engine/outliers/trimmer.py index 9356182d6..1e9fc381e 100644 --- a/feature_engine/outliers/trimmer.py +++ b/feature_engine/outliers/trimmer.py @@ -11,8 +11,9 @@ _right_tail_caps_docstring, _variables_attribute_docstring, ) -from feature_engine._docstrings.init_parameters.all_trasnformers import ( +from feature_engine._docstrings.init_parameters.all_transformers import ( _missing_values_docstring, + _return_empty_docstring, _variables_numerical_docstring, ) from feature_engine._docstrings.init_parameters.outliers import ( @@ -31,6 +32,7 @@ tail=_tail_docstring, fold=_fold_docstring, variables=_variables_numerical_docstring, + return_empty=_return_empty_docstring, missing_values=_missing_values_docstring, right_tail_caps_=_right_tail_caps_docstring, left_tail_caps_=_left_tail_caps_docstring, @@ -66,6 +68,8 @@ class OutlierTrimmer(WinsorizerBase, TransformXyMixin): {variables} + {return_empty} + {missing_values} Attributes diff --git a/feature_engine/outliers/winsorizer.py b/feature_engine/outliers/winsorizer.py index d9c2183d4..d07aa7fa0 100644 --- a/feature_engine/outliers/winsorizer.py +++ b/feature_engine/outliers/winsorizer.py @@ -13,8 +13,9 @@ _right_tail_caps_docstring, _variables_attribute_docstring, ) -from feature_engine._docstrings.init_parameters.all_trasnformers import ( +from feature_engine._docstrings.init_parameters.all_transformers import ( _missing_values_docstring, + _return_empty_docstring, _variables_numerical_docstring, ) from feature_engine._docstrings.init_parameters.outliers import ( @@ -34,6 +35,7 @@ tail=_tail_docstring, fold=_fold_docstring, variables=_variables_numerical_docstring, + return_empty=_return_empty_docstring, missing_values=_missing_values_docstring, right_tail_caps_=_right_tail_caps_docstring, left_tail_caps_=_left_tail_caps_docstring, @@ -73,6 +75,8 @@ class Winsorizer(WinsorizerBase): {variables} + {return_empty} + {missing_values} Attributes @@ -168,6 +172,7 @@ def __init__( fold: Union[int, float, Literal["auto"]] = "auto", add_indicators: bool = False, variables: Union[None, int, str, List[Union[str, int]]] = None, + return_empty: bool = False, missing_values: str = "raise", ) -> None: if not isinstance(add_indicators, bool): @@ -175,7 +180,9 @@ def __init__( "add_indicators takes only booleans True and False" f"Got {add_indicators} instead." ) - super().__init__(capping_method, tail, fold, variables, missing_values) + super().__init__( + capping_method, tail, fold, variables, return_empty, missing_values + ) self.add_indicators = add_indicators def transform(self, X: pd.DataFrame) -> pd.DataFrame: diff --git a/feature_engine/preprocessing/match_categories.py b/feature_engine/preprocessing/match_categories.py index 5fe9685c7..9241df261 100644 --- a/feature_engine/preprocessing/match_categories.py +++ b/feature_engine/preprocessing/match_categories.py @@ -4,13 +4,17 @@ import pandas as pd from feature_engine._base_transformers.mixins import GetFeatureNamesOutMixin +from feature_engine._check_init_parameters.check_init_input_params import ( + _check_return_empty_is_bool, +) from feature_engine._docstrings.fit_attributes import ( _feature_names_in_docstring, _n_features_in_docstring, _variables_attribute_docstring, ) -from feature_engine._docstrings.init_parameters.all_trasnformers import ( +from feature_engine._docstrings.init_parameters.all_transformers import ( _missing_values_docstring, + _return_empty_docstring, _variables_categorical_docstring, ) from feature_engine._docstrings.init_parameters.encoders import _ignore_format_docstring @@ -26,6 +30,7 @@ ignore_format=_ignore_format_docstring, missing_values=_missing_values_docstring, variables=_variables_categorical_docstring, + return_empty=_return_empty_docstring, variables_=_variables_attribute_docstring, feature_names_in_=_feature_names_in_docstring, n_features_in_=_n_features_in_docstring, @@ -54,6 +59,8 @@ class MatchCategories( ---------- {variables} + {return_empty} + {ignore_format} {missing_values} @@ -114,11 +121,15 @@ class MatchCategories( def __init__( self, variables: Union[None, int, str, List[Union[str, int]]] = None, + return_empty: bool = False, ignore_format: bool = False, missing_values: str = "raise", ) -> None: + _check_return_empty_is_bool(return_empty) + super().__init__(variables, missing_values, ignore_format) + self.return_empty = return_empty def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): """ diff --git a/feature_engine/scaling/mean_normalization.py b/feature_engine/scaling/mean_normalization.py index 78f4a958c..f8e393605 100644 --- a/feature_engine/scaling/mean_normalization.py +++ b/feature_engine/scaling/mean_normalization.py @@ -6,6 +6,9 @@ import pandas as pd from feature_engine._base_transformers.base_numerical import BaseNumericalTransformer +from feature_engine._check_init_parameters.check_init_input_params import ( + _check_return_empty_is_bool, +) from feature_engine._check_init_parameters.check_variables import ( _check_variables_input_value, ) @@ -14,7 +17,8 @@ _n_features_in_docstring, _variables_attribute_docstring, ) -from feature_engine._docstrings.init_parameters.all_trasnformers import ( +from feature_engine._docstrings.init_parameters.all_transformers import ( + _return_empty_docstring, _variables_numerical_docstring, ) from feature_engine._docstrings.methods import ( @@ -26,6 +30,7 @@ @Substitution( variables=_variables_numerical_docstring, + return_empty=_return_empty_docstring, variables_=_variables_attribute_docstring, feature_names_in_=_feature_names_in_docstring, n_features_in_=_n_features_in_docstring, @@ -51,6 +56,7 @@ class MeanNormalizationScaler(BaseNumericalTransformer): ---------- {variables} + {return_empty} Attributes ---------- @@ -101,9 +107,13 @@ class MeanNormalizationScaler(BaseNumericalTransformer): def __init__( self, variables: Union[None, int, str, List[Union[str, int]]] = None, + return_empty: bool = False, ) -> None: + _check_return_empty_is_bool(return_empty) + self.variables = _check_variables_input_value(variables) + self.return_empty = return_empty def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): """ diff --git a/feature_engine/selection/drop_psi_features.py b/feature_engine/selection/drop_psi_features.py index 9d050bf8f..36cb0b503 100644 --- a/feature_engine/selection/drop_psi_features.py +++ b/feature_engine/selection/drop_psi_features.py @@ -4,7 +4,7 @@ import numpy as np import pandas as pd import scipy.stats as stats -from pandas.api.types import is_numeric_dtype +from pandas.api.types import is_datetime64_any_dtype, is_numeric_dtype from feature_engine._check_init_parameters.check_variables import ( _check_variables_input_value, @@ -681,7 +681,12 @@ def _split_dataframe(self, X: pd.DataFrame): # Split the original dataframe if isinstance(self.cut_off_, list): - is_within_cut_off = np.array(reference.isin(self.cut_off_)) + cut_off = self.cut_off_ + # isin with values castable to datetime (strings, dates) is + # deprecated in pandas; cast them to the reference dtype first. + if is_datetime64_any_dtype(reference): + cut_off = pd.to_datetime(cut_off) + is_within_cut_off = np.array(reference.isin(cut_off)) else: is_within_cut_off = np.array(reference <= self.cut_off_) diff --git a/feature_engine/selection/mrmr.py b/feature_engine/selection/mrmr.py index 7ed189212..1669e5376 100644 --- a/feature_engine/selection/mrmr.py +++ b/feature_engine/selection/mrmr.py @@ -462,7 +462,10 @@ def _calculate_mrmr(self, relevance, redundance): if self.method in ["MID", "FCD"]: mrmr = relevance - redundance else: - mrmr = relevance / redundance + # redundance can be 0; the resulting inf makes the feature rank + # first in argmax, which is the intended MRMR behavior. + with np.errstate(divide="ignore", invalid="ignore"): + mrmr = relevance / redundance return mrmr def _more_tags(self): diff --git a/feature_engine/timeseries/forecasting/base_forecast_transformers.py b/feature_engine/timeseries/forecasting/base_forecast_transformers.py index f6edc95c0..f7ec468bf 100644 --- a/feature_engine/timeseries/forecasting/base_forecast_transformers.py +++ b/feature_engine/timeseries/forecasting/base_forecast_transformers.py @@ -8,6 +8,9 @@ GetFeatureNamesOutMixin, TransformXyMixin, ) +from feature_engine._check_init_parameters.check_init_input_params import ( + _check_return_empty_is_bool, +) from feature_engine._check_init_parameters.check_variables import ( _check_variables_input_value, ) @@ -15,7 +18,7 @@ _feature_names_in_docstring, _n_features_in_docstring, ) -from feature_engine._docstrings.init_parameters.all_trasnformers import ( +from feature_engine._docstrings.init_parameters.all_transformers import ( _drop_original_docstring, _missing_values_docstring, ) @@ -70,6 +73,7 @@ class BaseForecastTransformer( def __init__( self, variables: Union[None, int, str, List[Union[str, int]]] = None, + return_empty: bool = False, missing_values: str = "raise", drop_original: bool = False, drop_na: bool = False, @@ -93,7 +97,10 @@ def __init__( f"Got {drop_na} instead." ) + _check_return_empty_is_bool(return_empty) + self.variables = _check_variables_input_value(variables) + self.return_empty = return_empty self.missing_values = missing_values self.drop_original = drop_original self.drop_na = drop_na @@ -173,7 +180,9 @@ def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): # find or check for numerical variables if self.variables is None: - self.variables_ = find_numerical_variables(X) + self.variables_ = find_numerical_variables( + X, return_empty=self.return_empty + ) else: self.variables_ = check_numerical_variables(X, self.variables) diff --git a/feature_engine/timeseries/forecasting/expanding_window_features.py b/feature_engine/timeseries/forecasting/expanding_window_features.py index 72abf89a7..4f9358134 100644 --- a/feature_engine/timeseries/forecasting/expanding_window_features.py +++ b/feature_engine/timeseries/forecasting/expanding_window_features.py @@ -11,9 +11,10 @@ _feature_names_in_docstring, _n_features_in_docstring, ) -from feature_engine._docstrings.init_parameters.all_trasnformers import ( +from feature_engine._docstrings.init_parameters.all_transformers import ( _drop_original_docstring, _missing_values_docstring, + _return_empty_docstring, _variables_numerical_docstring, ) from feature_engine._docstrings.methods import ( @@ -28,6 +29,7 @@ @Substitution( variables=_variables_numerical_docstring, + return_empty=_return_empty_docstring, missing_values=_missing_values_docstring, drop_original=_drop_original_docstring, feature_names_in_=_feature_names_in_docstring, @@ -68,6 +70,8 @@ class ExpandingWindowFeatures(BaseForecastTransformer): ---------- {variables} + {return_empty} + min_periods: int, default None. Minimum number of observations in window required to have a value; otherwise, result is np.nan. See parameter `min_periods` in the pandas @@ -151,6 +155,7 @@ class ExpandingWindowFeatures(BaseForecastTransformer): def __init__( self, variables: None | int | str | list[str | int] = None, + return_empty: bool = False, min_periods: int | None = None, functions: str | list[str] = "mean", periods: int = 1, @@ -176,7 +181,9 @@ def __init__( f"periods must be a non-negative integer. Got {periods} instead." ) - super().__init__(variables, missing_values, drop_original, drop_na) + super().__init__( + variables, return_empty, missing_values, drop_original, drop_na + ) self.min_periods = min_periods self.functions = functions diff --git a/feature_engine/timeseries/forecasting/lag_features.py b/feature_engine/timeseries/forecasting/lag_features.py index ee9c1c151..ca682861e 100644 --- a/feature_engine/timeseries/forecasting/lag_features.py +++ b/feature_engine/timeseries/forecasting/lag_features.py @@ -10,9 +10,10 @@ _feature_names_in_docstring, _n_features_in_docstring, ) -from feature_engine._docstrings.init_parameters.all_trasnformers import ( +from feature_engine._docstrings.init_parameters.all_transformers import ( _drop_original_docstring, _missing_values_docstring, + _return_empty_docstring, _variables_numerical_docstring, ) from feature_engine._docstrings.methods import ( @@ -27,6 +28,7 @@ @Substitution( variables=_variables_numerical_docstring, + return_empty=_return_empty_docstring, missing_values=_missing_values_docstring, drop_original=_drop_original_docstring, feature_names_in_=_feature_names_in_docstring, @@ -57,6 +59,8 @@ class LagFeatures(BaseForecastTransformer): ---------- {variables} + {return_empty} + periods: int, list of ints, default=1 Number of periods to shift. Can be a positive integer or list of positive integers. If list, features will be created for each one of the periods in the @@ -135,6 +139,7 @@ class LagFeatures(BaseForecastTransformer): def __init__( self, variables: Union[None, int, str, List[Union[str, int]]] = None, + return_empty: bool = False, periods: Union[int, List[int]] = 1, freq: Union[str, List[str], None] = None, fill_value: Hashable = None, @@ -166,7 +171,9 @@ def __init__( "sort_index takes values True and False." f"Got {sort_index} instead." ) - super().__init__(variables, missing_values, drop_original, drop_na) + super().__init__( + variables, return_empty, missing_values, drop_original, drop_na + ) self.periods = periods self.freq = freq diff --git a/feature_engine/timeseries/forecasting/window_features.py b/feature_engine/timeseries/forecasting/window_features.py index a1e526c3e..be81c79a3 100644 --- a/feature_engine/timeseries/forecasting/window_features.py +++ b/feature_engine/timeseries/forecasting/window_features.py @@ -6,9 +6,10 @@ _feature_names_in_docstring, _n_features_in_docstring, ) -from feature_engine._docstrings.init_parameters.all_trasnformers import ( +from feature_engine._docstrings.init_parameters.all_transformers import ( _drop_original_docstring, _missing_values_docstring, + _return_empty_docstring, _variables_numerical_docstring, ) from feature_engine._docstrings.methods import ( @@ -23,6 +24,7 @@ @Substitution( variables=_variables_numerical_docstring, + return_empty=_return_empty_docstring, missing_values=_missing_values_docstring, drop_original=_drop_original_docstring, feature_names_in_=_feature_names_in_docstring, @@ -64,6 +66,8 @@ class WindowFeatures(BaseForecastTransformer): ---------- {variables} + {return_empty} + window: int, offset, BaseIndexer subclass, or list, default=3 Size of the moving window. If an integer, the fixed number of observations used for each window. If an offset (recommended), the time period of each window. It @@ -154,6 +158,7 @@ class WindowFeatures(BaseForecastTransformer): def __init__( self, variables: Union[None, int, str, List[Union[str, int]]] = None, + return_empty: bool = False, window: Union[str, int, Callable, List[int], List[str]] = 3, min_periods: Union[int, None] = None, functions: Union[str, List[str]] = "mean", @@ -183,7 +188,9 @@ def __init__( f"periods must be a positive integer. Got {periods} instead." ) - super().__init__(variables, missing_values, drop_original, drop_na) + super().__init__( + variables, return_empty, missing_values, drop_original, drop_na + ) self.window = window self.min_periods = min_periods diff --git a/feature_engine/transformation/arcsin.py b/feature_engine/transformation/arcsin.py index 059df813e..d31311f82 100644 --- a/feature_engine/transformation/arcsin.py +++ b/feature_engine/transformation/arcsin.py @@ -7,6 +7,9 @@ import pandas as pd from feature_engine._base_transformers.base_numerical import BaseNumericalTransformer +from feature_engine._check_init_parameters.check_init_input_params import ( + _check_return_empty_is_bool, +) from feature_engine._check_init_parameters.check_variables import ( _check_variables_input_value, ) @@ -15,7 +18,8 @@ _n_features_in_docstring, _variables_attribute_docstring, ) -from feature_engine._docstrings.init_parameters.all_trasnformers import ( +from feature_engine._docstrings.init_parameters.all_transformers import ( + _return_empty_docstring, _variables_numerical_docstring, ) from feature_engine._docstrings.methods import ( @@ -29,6 +33,7 @@ @Substitution( variables=_variables_numerical_docstring, + return_empty=_return_empty_docstring, variables_=_variables_attribute_docstring, feature_names_in_=_feature_names_in_docstring, n_features_in_=_n_features_in_docstring, @@ -61,6 +66,8 @@ class ArcsinTransformer(BaseNumericalTransformer): ---------- {variables} + {return_empty} + Attributes ---------- {variables_} @@ -101,10 +108,15 @@ class ArcsinTransformer(BaseNumericalTransformer): """ def __init__( - self, variables: Union[None, int, str, List[Union[str, int]]] = None + self, + variables: Union[None, int, str, List[Union[str, int]]] = None, + return_empty: bool = False, ) -> None: + _check_return_empty_is_bool(return_empty) + self.variables = _check_variables_input_value(variables) + self.return_empty = return_empty def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): """ diff --git a/feature_engine/transformation/arcsinh.py b/feature_engine/transformation/arcsinh.py index e0020ff86..659b2c4c2 100644 --- a/feature_engine/transformation/arcsinh.py +++ b/feature_engine/transformation/arcsinh.py @@ -7,6 +7,9 @@ import pandas as pd from feature_engine._base_transformers.base_numerical import BaseNumericalTransformer +from feature_engine._check_init_parameters.check_init_input_params import ( + _check_return_empty_is_bool, +) from feature_engine._check_init_parameters.check_variables import ( _check_variables_input_value, ) @@ -15,7 +18,8 @@ _n_features_in_docstring, _variables_attribute_docstring, ) -from feature_engine._docstrings.init_parameters.all_trasnformers import ( +from feature_engine._docstrings.init_parameters.all_transformers import ( + _return_empty_docstring, _variables_numerical_docstring, ) from feature_engine._docstrings.methods import ( @@ -29,6 +33,7 @@ @Substitution( variables=_variables_numerical_docstring, + return_empty=_return_empty_docstring, variables_=_variables_attribute_docstring, feature_names_in_=_feature_names_in_docstring, n_features_in_=_n_features_in_docstring, @@ -59,6 +64,8 @@ class ArcSinhTransformer(BaseNumericalTransformer): ---------- {variables} + {return_empty} + loc: float, default=0.0 Location parameter for shifting the data before transformation. The transformation becomes: arcsinh((x - loc) / scale) @@ -122,6 +129,7 @@ class ArcSinhTransformer(BaseNumericalTransformer): def __init__( self, variables: Union[None, int, str, List[Union[str, int]]] = None, + return_empty: bool = False, loc: float = 0.0, scale: float = 1.0, ) -> None: @@ -137,7 +145,10 @@ def __init__( f"scale must be a positive number (> 0). Got {scale} instead." ) + _check_return_empty_is_bool(return_empty) + self.variables = _check_variables_input_value(variables) + self.return_empty = return_empty self.loc = float(loc) self.scale = float(scale) diff --git a/feature_engine/transformation/boxcox.py b/feature_engine/transformation/boxcox.py index 1541ff8b5..78a60ad2b 100644 --- a/feature_engine/transformation/boxcox.py +++ b/feature_engine/transformation/boxcox.py @@ -8,6 +8,9 @@ from scipy.special import inv_boxcox from feature_engine._base_transformers.base_numerical import BaseNumericalTransformer +from feature_engine._check_init_parameters.check_init_input_params import ( + _check_return_empty_is_bool, +) from feature_engine._check_init_parameters.check_variables import ( _check_variables_input_value, ) @@ -16,7 +19,8 @@ _n_features_in_docstring, _variables_attribute_docstring, ) -from feature_engine._docstrings.init_parameters.all_trasnformers import ( +from feature_engine._docstrings.init_parameters.all_transformers import ( + _return_empty_docstring, _variables_numerical_docstring, ) from feature_engine._docstrings.methods import ( @@ -29,6 +33,7 @@ @Substitution( variables=_variables_numerical_docstring, + return_empty=_return_empty_docstring, variables_=_variables_attribute_docstring, feature_names_in_=_feature_names_in_docstring, n_features_in_=_n_features_in_docstring, @@ -65,6 +70,8 @@ class BoxCoxTransformer(BaseNumericalTransformer): ---------- {variables} + {return_empty} + Attributes ---------- lambda_dict_: @@ -115,10 +122,15 @@ class BoxCoxTransformer(BaseNumericalTransformer): """ def __init__( - self, variables: Union[None, int, str, List[Union[str, int]]] = None + self, + variables: Union[None, int, str, List[Union[str, int]]] = None, + return_empty: bool = False, ) -> None: + _check_return_empty_is_bool(return_empty) + self.variables = _check_variables_input_value(variables) + self.return_empty = return_empty def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): """ diff --git a/feature_engine/transformation/log.py b/feature_engine/transformation/log.py index 695243291..baca6dbbd 100644 --- a/feature_engine/transformation/log.py +++ b/feature_engine/transformation/log.py @@ -8,6 +8,9 @@ from feature_engine._base_transformers.base_numerical import BaseNumericalTransformer from feature_engine._base_transformers.mixins import FitFromDictMixin +from feature_engine._check_init_parameters.check_init_input_params import ( + _check_return_empty_is_bool, +) from feature_engine._check_init_parameters.check_variables import ( _check_variables_input_value, ) @@ -16,7 +19,8 @@ _n_features_in_docstring, _variables_attribute_docstring, ) -from feature_engine._docstrings.init_parameters.all_trasnformers import ( +from feature_engine._docstrings.init_parameters.all_transformers import ( + _return_empty_docstring, _variables_numerical_docstring, ) from feature_engine._docstrings.methods import ( @@ -30,6 +34,7 @@ @Substitution( variables=_variables_numerical_docstring, + return_empty=_return_empty_docstring, variables_=_variables_attribute_docstring, feature_names_in_=_feature_names_in_docstring, n_features_in_=_n_features_in_docstring, @@ -54,6 +59,8 @@ class LogTransformer(BaseNumericalTransformer): ---------- {variables} + {return_empty} + base: string, default='e' Indicates if the natural or base 10 logarithm should be applied. Can take values 'e' or '10'. @@ -100,13 +107,17 @@ class LogTransformer(BaseNumericalTransformer): def __init__( self, variables: Union[None, int, str, List[Union[str, int]]] = None, + return_empty: bool = False, base: str = "e", ) -> None: if base not in ["e", "10"]: raise ValueError("base can take only '10' or 'e' as values") + _check_return_empty_is_bool(return_empty) + self.variables = _check_variables_input_value(variables) + self.return_empty = return_empty self.base = base def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): @@ -221,6 +232,7 @@ def __sklearn_tags__(self): @Substitution( + return_empty=_return_empty_docstring, variables_=_variables_attribute_docstring, feature_names_in_=_feature_names_in_docstring, n_features_in_=_n_features_in_docstring, @@ -255,6 +267,8 @@ class LogCpTransformer(BaseNumericalTransformer, FitFromDictMixin): parameter is ignored and the variables to transform are selected from the dictionary keys. + {return_empty} + base: string, default='e' Indicates if the natural or base 10 logarithm should be applied. Can take values 'e' or '10'. @@ -317,6 +331,7 @@ class LogCpTransformer(BaseNumericalTransformer, FitFromDictMixin): def __init__( self, variables: Union[None, int, str, List[Union[str, int]]] = None, + return_empty: bool = False, base: str = "e", C: Union[int, float, str, Dict[Union[str, int], Union[float, int]]] = "auto", ) -> None: @@ -331,7 +346,10 @@ def __init__( f"C can take only 'auto', integers or floats. Got {C} instead." ) + _check_return_empty_is_bool(return_empty) + self.variables = _check_variables_input_value(variables) + self.return_empty = return_empty self.base = base self.C = C diff --git a/feature_engine/transformation/power.py b/feature_engine/transformation/power.py index ae10a16bf..f4081a864 100644 --- a/feature_engine/transformation/power.py +++ b/feature_engine/transformation/power.py @@ -7,6 +7,9 @@ import pandas as pd from feature_engine._base_transformers.base_numerical import BaseNumericalTransformer +from feature_engine._check_init_parameters.check_init_input_params import ( + _check_return_empty_is_bool, +) from feature_engine._check_init_parameters.check_variables import ( _check_variables_input_value, ) @@ -15,7 +18,8 @@ _n_features_in_docstring, _variables_attribute_docstring, ) -from feature_engine._docstrings.init_parameters.all_trasnformers import ( +from feature_engine._docstrings.init_parameters.all_transformers import ( + _return_empty_docstring, _variables_numerical_docstring, ) from feature_engine._docstrings.methods import ( @@ -28,6 +32,7 @@ @Substitution( variables=_variables_numerical_docstring, + return_empty=_return_empty_docstring, variables_=_variables_attribute_docstring, feature_names_in_=_feature_names_in_docstring, n_features_in_=_n_features_in_docstring, @@ -52,6 +57,8 @@ class PowerTransformer(BaseNumericalTransformer): ---------- {variables} + {return_empty} + exp: float or int, default=0.5 The power (or exponent). @@ -97,14 +104,18 @@ class PowerTransformer(BaseNumericalTransformer): def __init__( self, variables: Union[None, int, str, List[Union[str, int]]] = None, + return_empty: bool = False, exp: Union[float, int] = 0.5, ): if not isinstance(exp, (float, int)): raise ValueError("exp must be a float or an int") - self.exp = exp + _check_return_empty_is_bool(return_empty) + self.variables = _check_variables_input_value(variables) + self.return_empty = return_empty + self.exp = exp def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): """ diff --git a/feature_engine/transformation/reciprocal.py b/feature_engine/transformation/reciprocal.py index d51557331..42ac0ef87 100644 --- a/feature_engine/transformation/reciprocal.py +++ b/feature_engine/transformation/reciprocal.py @@ -6,6 +6,9 @@ import pandas as pd from feature_engine._base_transformers.base_numerical import BaseNumericalTransformer +from feature_engine._check_init_parameters.check_init_input_params import ( + _check_return_empty_is_bool, +) from feature_engine._check_init_parameters.check_variables import ( _check_variables_input_value, ) @@ -14,7 +17,8 @@ _n_features_in_docstring, _variables_attribute_docstring, ) -from feature_engine._docstrings.init_parameters.all_trasnformers import ( +from feature_engine._docstrings.init_parameters.all_transformers import ( + _return_empty_docstring, _variables_numerical_docstring, ) from feature_engine._docstrings.methods import ( @@ -28,6 +32,7 @@ @Substitution( variables=_variables_numerical_docstring, + return_empty=_return_empty_docstring, variables_=_variables_attribute_docstring, feature_names_in_=_feature_names_in_docstring, n_features_in_=_n_features_in_docstring, @@ -53,6 +58,8 @@ class ReciprocalTransformer(BaseNumericalTransformer): ---------- {variables} + {return_empty} + Attributes ---------- {variables_} @@ -93,9 +100,14 @@ class ReciprocalTransformer(BaseNumericalTransformer): """ def __init__( - self, variables: Union[None, int, str, List[Union[str, int]]] = None + self, + variables: Union[None, int, str, List[Union[str, int]]] = None, + return_empty: bool = False, ) -> None: + _check_return_empty_is_bool(return_empty) + self.variables = _check_variables_input_value(variables) + self.return_empty = return_empty def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): """ diff --git a/feature_engine/transformation/yeojohnson.py b/feature_engine/transformation/yeojohnson.py index f8d938e4a..b096433db 100644 --- a/feature_engine/transformation/yeojohnson.py +++ b/feature_engine/transformation/yeojohnson.py @@ -8,6 +8,9 @@ import scipy.stats as stats from feature_engine._base_transformers.base_numerical import BaseNumericalTransformer +from feature_engine._check_init_parameters.check_init_input_params import ( + _check_return_empty_is_bool, +) from feature_engine._check_init_parameters.check_variables import ( _check_variables_input_value, ) @@ -16,7 +19,8 @@ _n_features_in_docstring, _variables_attribute_docstring, ) -from feature_engine._docstrings.init_parameters.all_trasnformers import ( +from feature_engine._docstrings.init_parameters.all_transformers import ( + _return_empty_docstring, _variables_numerical_docstring, ) from feature_engine._docstrings.methods import ( @@ -29,6 +33,7 @@ @Substitution( variables=_variables_numerical_docstring, + return_empty=_return_empty_docstring, variables_=_variables_attribute_docstring, feature_names_in_=_feature_names_in_docstring, n_features_in_=_n_features_in_docstring, @@ -56,6 +61,8 @@ class YeoJohnsonTransformer(BaseNumericalTransformer): ---------- {variables} + {return_empty} + Attributes ---------- lambda_dict_ @@ -109,9 +116,14 @@ class YeoJohnsonTransformer(BaseNumericalTransformer): """ def __init__( - self, variables: Union[None, int, str, List[Union[str, int]]] = None + self, + variables: Union[None, int, str, List[Union[str, int]]] = None, + return_empty: bool = False, ) -> None: + _check_return_empty_is_bool(return_empty) + self.variables = _check_variables_input_value(variables) + self.return_empty = return_empty def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): """ diff --git a/feature_engine/variable_handling/find_variables.py b/feature_engine/variable_handling/find_variables.py index 075f79ab3..5d072eb56 100644 --- a/feature_engine/variable_handling/find_variables.py +++ b/feature_engine/variable_handling/find_variables.py @@ -32,6 +32,12 @@ def find_numerical_variables( Whether to return an empty list when no numerical variables are found. If False, the function raises an error. + .. versionadded:: 2.0 + `return_empty` currently defaults to False. The default will change to + True in version 2.1. To keep the current behaviour and silence the + warning, explicitly set `return_empty=False` instead of relying on the + default. + Returns ------- variables: List @@ -86,6 +92,12 @@ def find_categorical_variables( Whether to return an empty list when no categorical variables are found. If False, the function raises an error. + .. versionadded:: 2.0 + `return_empty` currently defaults to False. The default will change to + True in version 2.1. To keep the current behaviour and silence the + warning, explicitly set `return_empty=False` instead of relying on the + default. + Returns ------- variables: List @@ -144,9 +156,15 @@ def find_datetime_variables( The dataset. return_empty : bool, default=False - Whether to return an empty list when no datetimemvariables are found. + Whether to return an empty list when no datetime variables are found. If False, the function raises an error. + .. versionadded:: 2.0 + `return_empty` currently defaults to False. The default will change to + True in version 2.1. To keep the current behaviour and silence the + warning, explicitly set `return_empty=False` instead of relying on the + default. + Returns ------- variables: List @@ -193,7 +211,7 @@ def find_all_variables( ) -> List[Union[str, int]]: """ Returns a list with the names of all the variables in the dataframe. - Optionally, it exlcudes variables that can be parsed as datetime or datetimetz. + Optionally, it excludes variables that can be parsed as datetime or datetimetz. More details in the :ref:`User Guide `. @@ -209,6 +227,12 @@ def find_all_variables( Whether to return an empty list when no variables are found. If False, the function raises an error. + .. versionadded:: 2.0 + `return_empty` currently defaults to False. The default will change to + True in version 2.1. To keep the current behaviour and silence the + warning, explicitly set `return_empty=False` instead of relying on the + default. + Returns ------- variables: List @@ -278,6 +302,12 @@ def find_categorical_and_numerical_variables( Whether to return empty lists when no variables are found. If False, the function raises an error. + .. versionadded:: 2.0 + `return_empty` currently defaults to False. The default will change to + True in version 2.1. To keep the current behaviour and silence the + warning, explicitly set `return_empty=False` instead of relying on the + default. + Returns ------- variables: tuple diff --git a/feature_engine/wrappers/wrappers.py b/feature_engine/wrappers/wrappers.py index 6787ede9e..1f5ba8120 100644 --- a/feature_engine/wrappers/wrappers.py +++ b/feature_engine/wrappers/wrappers.py @@ -4,6 +4,9 @@ from sklearn.base import BaseEstimator, TransformerMixin, clone from sklearn.utils.validation import check_is_fitted +from feature_engine._check_init_parameters.check_init_input_params import ( + _check_return_empty_is_bool, +) from feature_engine._check_init_parameters.check_variables import ( _check_variables_input_value, ) @@ -96,6 +99,11 @@ class SklearnTransformerWrapper(TransformerMixin, BaseEstimator): OrdinalEncoder and OneHotEncoder, in which case, it will select all variables in the dataset. + return_empty: bool, default=False + Whether to return an empty list when no variables of the required type are + found. If False, the transformer raises an error. This parameter is only + used when `variables` is `None`. + Attributes ---------- transformer_: @@ -192,6 +200,7 @@ def __init__( self, transformer, variables: Union[None, int, str, List[Union[str, int]]] = None, + return_empty: bool = False, ) -> None: if not issubclass(transformer.__class__, TransformerMixin): @@ -235,8 +244,11 @@ def __init__( ): raise NotImplementedError(msg) + _check_return_empty_is_bool(return_empty) + self.transformer = transformer self.variables = _check_variables_input_value(variables) + self.return_empty = return_empty def fit(self, X: pd.DataFrame, y: Optional[str] = None): """ @@ -263,16 +275,26 @@ def fit(self, X: pd.DataFrame, y: Optional[str] = None): "FunctionTransformer", ]: if self.variables is None: - self.variables_ = find_all_variables(X) + self.variables_ = find_all_variables( + X, return_empty=self.return_empty + ) else: self.variables_ = check_all_variables(X, self.variables) else: if self.variables is None: - self.variables_ = find_numerical_variables(X) + self.variables_ = find_numerical_variables( + X, return_empty=self.return_empty + ) else: self.variables_ = check_numerical_variables(X, self.variables) + if len(self.variables_) == 0: + # save input features + self.feature_names_in_ = X.columns.tolist() + self.n_features_in_ = X.shape[1] + return self + self.transformer_.fit(X[self.variables_], y) if self.transformer_.__class__.__name__ in _SELECTORS: @@ -324,6 +346,10 @@ def transform(self, X: pd.DataFrame) -> pd.DataFrame: # reorder df to match train set X = X[self.feature_names_in_] + # nothing to transform, e.g. when return_empty selected no variables + if len(self.variables_) == 0: + return X + # Transformers that add features: creators if self.transformer_.__class__.__name__ in [ "OneHotEncoder", diff --git a/tests/estimator_checks/estimator_checks.py b/tests/estimator_checks/estimator_checks.py index 06aec8a6c..1e4b71ccd 100644 --- a/tests/estimator_checks/estimator_checks.py +++ b/tests/estimator_checks/estimator_checks.py @@ -6,6 +6,7 @@ from tests.estimator_checks.fit_functionality_checks import ( check_error_if_y_not_passed, check_feature_names_in, + check_return_empty, ) from tests.estimator_checks.get_feature_names_out_checks import ( check_get_feature_names_out, @@ -55,6 +56,8 @@ def check_feature_engine_estimator(estimator, needs_group: bool = False): - check that users enters permitted values to init parameters `missing_values`. + - checks correct functionality of parameter `return_empty`. + **Checks based on transformer tags.** - checks that transformer raises error if y is not passed. @@ -97,6 +100,9 @@ def check_feature_engine_estimator(estimator, needs_group: bool = False): if hasattr(estimator, "drop_original"): check_drop_original_variables(estimator) + if hasattr(estimator, "return_empty"): + check_return_empty(estimator) + return None diff --git a/tests/estimator_checks/fit_functionality_checks.py b/tests/estimator_checks/fit_functionality_checks.py index 4737da3ad..9a34537b7 100644 --- a/tests/estimator_checks/fit_functionality_checks.py +++ b/tests/estimator_checks/fit_functionality_checks.py @@ -1,5 +1,6 @@ """Checks functionality in the fit method shared by all transformers.""" +import pandas as pd import pytest from sklearn import clone @@ -32,3 +33,56 @@ def check_error_if_y_not_passed(estimator): estimator = clone(estimator) with pytest.raises(TypeError): estimator.fit(X) + + +def check_return_empty(estimator): + """ + Only for transformers with the init parameter `return_empty`. + + When `variables` is None and the train set contains no variables of the type + required by the transformer (numerical, categorical or datetime), `fit()` + raises a `TypeError` by default. When `return_empty` is set to `True`, `fit()` + instead assigns an empty list to `variables_`, and raises a `UserWarning` + instead of an error. Transformer should return the same dataframe in this case. + """ + # dataframe with no variables of the given type + variable_tag = estimator._more_tags().get("variables") + if variable_tag in ["numerical", "datetime"]: + df = pd.DataFrame({"var_cat": ["A", "B", "A", "B", "A", "B"]}) + elif variable_tag in ["all", "skip"]: + return + else: + df = pd.DataFrame({"var_num": [0.1, 0.2, 0.3, 0.4, 0.5, 0.6]}) + variable_tag = "categorical" + + y = pd.Series([0, 1, 0, 1, 0, 1]) + raise_match = f"No {variable_tag} variables found in this dataframe" + warn_match = ( + f"No {variable_tag} variables found in this dataframe. " + "Returning an empty list." + ) + + # ignore_format=True makes categorical transformers select all variables + # regardless of type, which defeats the purpose of this check. + base_params = {"variables": None, "return_empty": False} + if "ignore_format" in estimator.get_params(): + base_params["ignore_format"] = False + + # default: raises an error + transformer = clone(estimator) + transformer.set_params(**base_params) + with pytest.raises(TypeError, match=raise_match): + transformer.fit(df, y) + + # return_empty=True: warns and returns an empty list instead of raising + transformer = clone(estimator) + transformer.set_params(**{**base_params, "return_empty": True}) + with pytest.warns(UserWarning, match=warn_match): + transformer.fit(df, y) + assert transformer.variables_ == [] + + # if return_empty=True, transformer should return same df + # after transformation + dft = transformer.transform(df) + pd.testing.assert_frame_equal(dft, df) + assert transformer.get_feature_names_out() == list(df.columns) diff --git a/tests/test_base_transformers/test_base_numerical_transformer.py b/tests/test_base_transformers/test_base_numerical_transformer.py index 1629ab67e..88006114c 100644 --- a/tests/test_base_transformers/test_base_numerical_transformer.py +++ b/tests/test_base_transformers/test_base_numerical_transformer.py @@ -9,11 +9,22 @@ class MockClass(BaseNumericalTransformer): def __init__(self): self.variables = None + self.return_empty = False def transform(self, X): return self._check_transform_input_and_state(X) +def test_empty_find_numerical_variables(df_vartypes): + transformer = MockClass() + with pytest.raises(TypeError): + transformer.fit(df_vartypes.drop(columns=["Age", "Marks"])) + transformer = MockClass() + transformer.return_empty = True + transformer.fit(df_vartypes.drop(columns=["Age", "Marks"])) + assert transformer.variables_ == [] + + def test_fit_method(df_vartypes, df_na): transformer = MockClass() res = transformer.fit(df_vartypes) diff --git a/tests/test_creation/test_decision_tree_features.py b/tests/test_creation/test_decision_tree_features.py index a5e1cf0fd..4e8a93e8c 100644 --- a/tests/test_creation/test_decision_tree_features.py +++ b/tests/test_creation/test_decision_tree_features.py @@ -6,6 +6,7 @@ from sklearn.tree import DecisionTreeClassifier, DecisionTreeRegressor from feature_engine.creation import DecisionTreeFeatures +from tests.estimator_checks.fit_functionality_checks import check_return_empty @pytest.fixture(scope="module") @@ -582,3 +583,11 @@ def test_user_enter_param_grid(df_creation, classification_target): X_exp[varn] = preds[:, 1] pd.testing.assert_frame_equal(Xt, X_exp) + + +def test_check_return_empty(): + # DecisionTreeFeatures is not part of the check_feature_engine_estimator + # pipeline (test_check_estimator_creation.py only feeds MathFeatures, + # RelativeFeatures and CyclicalFeatures into it), so return_empty is + # tested directly here instead. + check_return_empty(DecisionTreeFeatures(regression=False)) diff --git a/tests/test_datetime/test_datetime_ordinal.py b/tests/test_datetime/test_datetime_ordinal.py index 84cd7dc79..aabeee395 100644 --- a/tests/test_datetime/test_datetime_ordinal.py +++ b/tests/test_datetime/test_datetime_ordinal.py @@ -7,28 +7,32 @@ @pytest.fixture(scope="module") def df_datetime_ordinal(): - df = pd.DataFrame({ - "date_col_1": pd.to_datetime( - ["2023-01-01", "2023-01-02", "2023-01-03", "2023-01-04", "2023-01-05"] - ), - "date_col_2": pd.to_datetime( - ["2024-02-10", "2024-02-11", "2024-02-12", "2024-02-13", "2024-02-14"] - ), - "non_date_col": [1, 2, 3, 4, 5], - }) + df = pd.DataFrame( + { + "date_col_1": pd.to_datetime( + ["2023-01-01", "2023-01-02", "2023-01-03", "2023-01-04", "2023-01-05"] + ), + "date_col_2": pd.to_datetime( + ["2024-02-10", "2024-02-11", "2024-02-12", "2024-02-13", "2024-02-14"] + ), + "non_date_col": [1, 2, 3, 4, 5], + } + ) return df @pytest.fixture(scope="module") def df_datetime_ordinal_na(): - df = pd.DataFrame({ - "date_col_1": pd.to_datetime( - ["2023-01-01", "2023-01-02", None, "2023-01-04", "2023-01-05"] - ), - "date_col_2": pd.to_datetime( - ["2024-02-10", "2024-02-11", "2024-02-12", None, "2024-02-14"] - ), - }) + df = pd.DataFrame( + { + "date_col_1": pd.to_datetime( + ["2023-01-01", "2023-01-02", None, "2023-01-04", "2023-01-05"] + ), + "date_col_2": pd.to_datetime( + ["2024-02-10", "2024-02-11", "2024-02-12", None, "2024-02-14"] + ), + } + ) return df @@ -36,11 +40,11 @@ def df_datetime_ordinal_na(): "variables_param", [ ["date_col_1", "date_col_2"], # Case 1: 'variables' are specified - None, # Case 2: 'variables' not specified + None, # Case 2: 'variables' not specified ], ids=[ "variables_specified", - "variables_auto_find" + "variables_auto_find", ], # Optional but recommended for test readability ) def test_datetime_ordinal_feature_creation(df_datetime_ordinal, variables_param): @@ -111,8 +115,7 @@ def test_datetime_ordinal_with_start_date_datetime_object(df_datetime_ordinal): def test_datetime_ordinal_missing_values_raise(df_datetime_ordinal_na): transformer = DatetimeOrdinal(missing_values="raise") with pytest.raises( - ValueError, - match="Some of the variables in the dataset contain NaN" + ValueError, match="Some of the variables in the dataset contain NaN" ): transformer.fit(df_datetime_ordinal_na) @@ -149,8 +152,7 @@ def test_datetime_ordinal_missing_values_ignore(df_datetime_ordinal_na): def test_datetime_ordinal_invalid_start_date(): with pytest.raises( - ValueError, - match="start_date could not be converted to datetime" + ValueError, match="start_date could not be converted to datetime" ): DatetimeOrdinal(start_date="not-a-date") @@ -267,3 +269,33 @@ def test_more_tags_returns_expected_tags(): transformer = DatetimeOrdinal() expected_tags = {"variables": "datetime"} assert transformer._more_tags() == expected_tags + + +def test_return_empty(): + # DatetimeOrdinal.__init__ does not store `self.start_date = start_date` + # (only the derived `self.start_date_`), which breaks sklearn's + # get_params()/clone() for this transformer. Because of that, it cannot go + # through the shared, clone-based check_return_empty check, nor through + # check_feature_engine_estimator at all. This test instantiates the + # transformer directly instead. + X = pd.DataFrame({"var_num": [1.0, 2.0, 3.0]}) + + transformer = DatetimeOrdinal(variables=None, return_empty=False) + with pytest.raises( + TypeError, match="No datetime variables found in this dataframe" + ): + transformer.fit(X) + + transformer = DatetimeOrdinal(variables=None, return_empty=True) + with pytest.warns( + UserWarning, + match="No datetime variables found in this dataframe. Returning an empty list.", + ): + transformer.fit(X) + assert transformer.variables_ == [] + + # if return_empty=True, transformer should return same df + # after transformation + dft = transformer.transform(X) + pd.testing.assert_frame_equal(dft, X) + assert transformer.get_feature_names_out() == list(X.columns) diff --git a/tests/test_datetime/test_datetime_subtraction.py b/tests/test_datetime/test_datetime_subtraction.py index 25a62ca9b..4e854d04e 100644 --- a/tests/test_datetime/test_datetime_subtraction.py +++ b/tests/test_datetime/test_datetime_subtraction.py @@ -6,7 +6,10 @@ from tests.estimator_checks.estimator_checks import ( check_raises_error_when_input_not_a_df, ) -from tests.estimator_checks.fit_functionality_checks import check_feature_names_in +from tests.estimator_checks.fit_functionality_checks import ( + check_feature_names_in, + check_return_empty, +) from tests.estimator_checks.init_params_triggered_functionality_checks import ( check_drop_original_variables, ) @@ -373,3 +376,4 @@ def test_common_tests(estimator): check_raises_error_when_input_not_a_df(estimator) check_feature_names_in(estimator) check_drop_original_variables(estimator) + check_return_empty(estimator) diff --git a/tests/test_encoding/test_base_encoders/test_categorical_method_mixin.py b/tests/test_encoding/test_base_encoders/test_categorical_method_mixin.py index 6116518e2..80dc30813 100644 --- a/tests/test_encoding/test_base_encoders/test_categorical_method_mixin.py +++ b/tests/test_encoding/test_base_encoders/test_categorical_method_mixin.py @@ -6,9 +6,10 @@ class MockClassFit(CategoricalMethodsMixin): - def __init__(self, missing_values="raise", ignore_format=False): + def __init__(self, missing_values="raise", ignore_format=False, return_empty=False): self.missing_values = missing_values self.variables = None + self.return_empty = return_empty self.ignore_format = ignore_format diff --git a/tests/test_pipeline/test_pipeline_sklearn.py b/tests/test_pipeline/test_pipeline_sklearn.py index 699b038c8..d545793a4 100644 --- a/tests/test_pipeline/test_pipeline_sklearn.py +++ b/tests/test_pipeline/test_pipeline_sklearn.py @@ -288,8 +288,8 @@ def test_pipeline_methods_pca_svm(): iris = load_iris() X = iris.data y = iris.target - # Test with PCA + SVC - clf = SVC(gamma="scale", probability=True, random_state=0) + # Test with PCA + classifier with predict_proba + clf = LogisticRegression(solver="lbfgs", random_state=0) pca = PCA(svd_solver="full", n_components="mle", whiten=True) pipe = Pipeline([("pca", pca), ("svc", clf)]) pipe.fit(X, y) @@ -308,12 +308,7 @@ def test_pipeline_methods_preprocessing_svm(): n_classes = len(np.unique(y)) scaler = StandardScaler() pca = PCA(n_components=2, svd_solver="randomized", whiten=True) - clf = SVC( - gamma="scale", - probability=True, - random_state=0, - decision_function_shape="ovr", - ) + clf = LogisticRegression(solver="lbfgs", random_state=0) for preprocessing in [scaler, pca]: pipe = Pipeline([("preprocess", preprocessing), ("svc", clf)]) @@ -612,8 +607,8 @@ def test_pipeline_memory_transformer(): cachedir = mkdtemp() try: memory = Memory(cachedir, verbose=10) - # Test with Transformer + SVC - clf = SVC(gamma="scale", probability=True, random_state=0) + # Test with Transformer + classifier with predict_proba + clf = LogisticRegression(solver="lbfgs", random_state=0) transf = DummyTransf() pipe = Pipeline([("transf", clone(transf)), ("svc", clf)]) cached_pipe = Pipeline([("transf", transf), ("svc", clf)], memory=memory) @@ -648,7 +643,7 @@ def test_pipeline_memory_transformer(): assert cached_pipe.named_steps["transf"].timestamp_ == expected_ts # Create a new pipeline with cloned estimators # Check that even changing the name step does not affect the cache hit - clf_2 = SVC(gamma="scale", probability=True, random_state=0) + clf_2 = LogisticRegression(solver="lbfgs", random_state=0) transf_2 = DummyTransf() cached_pipe_2 = Pipeline( [("transf_2", transf_2), ("svc", clf_2)], memory=memory @@ -765,7 +760,7 @@ def test_pipeline_param_error(): clf.fit([[0], [0]], [0, 1], sample_weight=[1, 1]) -parameter_grid_test_verbose = ( +parameter_grid_test_verbose = [ (est, pattern, method) for (est, pattern), method in itertools.product( [ @@ -825,7 +820,7 @@ def test_pipeline_param_error(): and hasattr(est, "steps") and isinstance(est.steps[-1][1], FitParamT) ) -) +] @pytest.mark.parametrize("est, pattern, method", parameter_grid_test_verbose) diff --git a/tests/test_scaling/test_mean_normalization.py b/tests/test_scaling/test_mean_normalization.py index 240cb7d3f..600b89275 100644 --- a/tests/test_scaling/test_mean_normalization.py +++ b/tests/test_scaling/test_mean_normalization.py @@ -5,6 +5,7 @@ from sklearn.exceptions import NotFittedError from feature_engine.scaling import MeanNormalizationScaler +from tests.estimator_checks.fit_functionality_checks import check_return_empty def test_transforming_int_vars(): @@ -126,3 +127,7 @@ def test_constant_columns_error(): transformer = MeanNormalizationScaler() with pytest.raises(ValueError, match=re.escape("Division by zero is not allowed")): transformer.fit(df) + + +def test_check_return_empty(): + check_return_empty(MeanNormalizationScaler()) diff --git a/tests/test_selection/test_drop_high_psi_features.py b/tests/test_selection/test_drop_high_psi_features.py index e936160b9..8cc296837 100644 --- a/tests/test_selection/test_drop_high_psi_features.py +++ b/tests/test_selection/test_drop_high_psi_features.py @@ -645,6 +645,9 @@ def test_split_by_list(df_mixed_types, col, cut_off_list): test = DropHighPSIFeatures(split_col=col, cut_off=cut_off_list, bins=3) a, b = test._split_dataframe(df_mixed_types) + if pd.api.types.is_datetime64_any_dtype(df_mixed_types[col]): + cut_off_list = pd.to_datetime(cut_off_list) + pd.testing.assert_frame_equal( a, df_mixed_types[df_mixed_types[col].isin(cut_off_list)] ) diff --git a/tests/test_wrappers/test_check_estimator_wrappers.py b/tests/test_wrappers/test_check_estimator_wrappers.py index f6506342b..d12a8741c 100644 --- a/tests/test_wrappers/test_check_estimator_wrappers.py +++ b/tests/test_wrappers/test_check_estimator_wrappers.py @@ -1,3 +1,4 @@ +import pandas as pd import pytest import sklearn from sklearn.impute import SimpleImputer @@ -56,3 +57,37 @@ def test_raises_error_when_no_transformer_passed(): # this transformer needs an estimator as an input param. with pytest.raises(TypeError): SklearnTransformerWrapper() + + +def test_return_empty(): + X = pd.DataFrame({"var_cat": ["A", "B", "A"]}) + + transformer = SklearnTransformerWrapper( + transformer=StandardScaler(), variables=None, return_empty=False + ) + with pytest.raises( + TypeError, match="No numerical variables found in this dataframe" + ): + transformer.fit(X) + + transformer = SklearnTransformerWrapper( + transformer=StandardScaler(), variables=None, return_empty=True + ) + with pytest.warns( + UserWarning, + match="No numerical variables found in this dataframe. " + "Returning an empty list.", + ): + transformer.fit(X) + assert transformer.variables_ == [] + + # if return_empty=True, transformer should return same df + # after transformation + dft = transformer.transform(X) + pd.testing.assert_frame_equal(dft, X) + assert transformer.get_feature_names_out() == list(X.columns) + + # when wrapping a transformer that selects all variable types (e.g. + # OrdinalEncoder), find_all_variables always finds at least the 1 column + # present in a non-empty dataframe, so return_empty can't be exercised + # this way; there is no dataframe that reaches the "no variables" branch.