Skip to content

Migrate CategoricalMethodsMixin (encoding base) to narwhals, add pola… #999

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
solegalli wants to merge 1 commit into
base: narwhals-migration
Choose a base branch
from
Open

Migrate CategoricalMethodsMixin (encoding base) to narwhals, add pola… #999

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
115 changes: 69 additions & 46 deletions feature_engine/encoding/base_encoder.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,9 @@
import warnings
from typing import List, Union

import pandas as pd
import narwhals as nw
import narwhals.dependencies as nwd
from narwhals.typing import IntoDataFrame
from sklearn.base import BaseEstimator, TransformerMixin
from sklearn.utils.validation import check_is_fitted

Expand Down Expand Up @@ -121,19 +123,19 @@ class CategoricalMethodsMixin(TransformerMixin, BaseEstimator, GetFeatureNamesOu
- GetFeatureNamesOutMixin brings method get_feature_names_out().
"""

def _check_na(self, X: pd.DataFrame, variables):
def _check_na(self, X: IntoDataFrame, variables):
if self.missing_values == "raise":
_check_contains_na(X, variables, error_msg="optional")

def _check_or_select_variables(self, X: pd.DataFrame):
def _check_or_select_variables(self, X: IntoDataFrame):
"""
Finds categorical variables, or alternatively checks that the variables
entered by the user are of type object (categorical).
Checks absence of NA.

Parameters
----------
X: Pandas DataFrame
X: dataframe

Raises
------
Expand All @@ -159,37 +161,41 @@ def _check_or_select_variables(self, X: pd.DataFrame):

return variables_

def _get_feature_names_in(self, X: pd.DataFrame):
def _get_feature_names_in(self, X: IntoDataFrame):
"""
Returns attributes `featrure_names_in_` and `n_feature_names_in_`, which are
standard for all transformers in the library.
"""
# save input features
self.feature_names_in_ = X.columns.tolist()
is_pandas = nwd.is_pandas_dataframe(X)
if is_pandas is True:
self.feature_names_in_ = list(X.columns)
else:
self.feature_names_in_ = nw.from_native(X, eager_only=True).columns

# save train set shape
self.n_features_in_ = X.shape[1]

def _check_transform_input_and_state(self, X: pd.DataFrame) -> pd.DataFrame:
def _check_transform_input_and_state(self, X: IntoDataFrame) -> IntoDataFrame:
"""
Checks that the input is a dataframe and of the same size than the one used
in the fit method. Checks absence of NA.

Parameters
----------
X: Pandas DataFrame
X: dataframe

Raises
------
TypeError
If the input is not a Pandas DataFrame
If the input is not a dataframe
ValueError
- If the variable(s) contain null values.
- If the df has different number of features than the df used in fit()

Returns
-------
X: Pandas DataFrame
X: dataframe
The same dataframe entered by the user.
"""

Expand All @@ -203,21 +209,29 @@ def _check_transform_input_and_state(self, X: pd.DataFrame) -> pd.DataFrame:
_check_X_matches_training_df(X, self.n_features_in_)

# reorder df to match train set
X = X[self.feature_names_in_]
is_pandas = nwd.is_pandas_dataframe(X)
if is_pandas is True:
X = X[self.feature_names_in_]
else:
X = (
nw.from_native(X, eager_only=True)
.select(self.feature_names_in_)
.to_native()
)

return X

def transform(self, X: pd.DataFrame) -> pd.DataFrame:
def transform(self, X: IntoDataFrame) -> IntoDataFrame:
"""Replace categories with the learned parameters.

Parameters
----------
X: pandas dataframe of shape = [n_samples, n_features].
X: dataframe of shape = [n_samples, n_features].
The dataset to transform.

Returns
-------
X_new: pandas dataframe of shape = [n_samples, n_features].
X_new: dataframe of shape = [n_samples, n_features].
The dataframe containing the categories replaced by numbers.
"""

Expand All @@ -231,22 +245,25 @@ def transform(self, X: pd.DataFrame) -> pd.DataFrame:

return X

def _encode(self, X: pd.DataFrame) -> pd.DataFrame:
# replace categories by the learned parameters
for feature in self.encoder_dict_.keys():
X[feature] = X[feature].map(self.encoder_dict_[feature])

# if original variables are cast as categorical, they will remain
# categorical after the encoding, and this is probably not desired
if X[feature].dtype.name == "category":
if all(isinstance(x, int) for x in X[feature]):
X[feature] = X[feature].astype("int")
else:
X[feature] = X[feature].astype("float")

if self.unseen == "encode":
X[self.variables_] = X[self.variables_].fillna(self._unseen)
else:
def _encode(self, X: IntoDataFrame) -> IntoDataFrame:
# replace categories by the learned parameters.
# narwhals' replace_strict() lets one expression both map known
# categories and fill unseen/missing ones via `default`, so the
# pandas-only category-dtype fixup this used to need (map() leaves
# category dtype behind) is no longer necessary: replace_strict
# already resolves to a plain numeric dtype on both backends.
# get_column()/Series.replace_strict() (rather than nw.col(), which
# only accepts string names) is what lets this handle pandas
# integer column names too, same as DecisionTreeFeatures.
default = self._unseen if self.unseen == "encode" else None
nw_X = nw.from_native(X, eager_only=True)
new_series = [
nw_X.get_column(feature).replace_strict(mapping, default=default)
for feature, mapping in self.encoder_dict_.items()
]
X = nw_X.with_columns(*new_series).to_native()

if self.unseen != "encode":
# check if nan values were introduced by the transformation
self._check_nan_values_after_transformation(X)

Expand All @@ -255,19 +272,19 @@ def _encode(self, X: pd.DataFrame) -> pd.DataFrame:
def _check_nan_values_after_transformation(self, X):

# check if NaN values were introduced by the encoding
if X[self.variables_].isnull().sum().sum() > 0:
nw_X = nw.from_native(X, eager_only=True)
nan_columns = [
feature
for feature in self.encoder_dict_.keys()
if nw_X.get_column(feature).null_count() > 0
]

# obtain the name(s) of the columns have null values
nan_columns = (
X[self.encoder_dict_.keys()]
.columns[X[self.encoder_dict_.keys()].isnull().any()]
.tolist()
)
if len(nan_columns) > 0:

if len(nan_columns) > 1:
nan_columns_str = ", ".join(nan_columns)
nan_columns_str = ", ".join(str(col) for col in nan_columns)
else:
nan_columns_str = nan_columns[0]
nan_columns_str = str(nan_columns[0])

if self.unseen == "ignore":
warnings.warn(
Expand All @@ -280,27 +297,33 @@ def _check_nan_values_after_transformation(self, X):
f"{nan_columns_str}."
)

def inverse_transform(self, X: pd.DataFrame) -> pd.DataFrame:
def inverse_transform(self, X: IntoDataFrame) -> IntoDataFrame:
"""Convert the encoded variable back to the original values.

Parameters
----------
X: pandas dataframe of shape = [n_samples, n_features].
X: dataframe of shape = [n_samples, n_features].
The transformed dataframe.

Returns
-------
X_tr: pandas dataframe of shape = [n_samples, n_features].
X_tr: dataframe of shape = [n_samples, n_features].
The un-transformed dataframe, with the categorical variables containing the
original values.
"""

X = self._check_transform_input_and_state(X)

# replace encoded categories by the original values
for feature in self.encoder_dict_.keys():
inv_map = {v: k for k, v in self.encoder_dict_[feature].items()}
X[feature] = X[feature].map(inv_map)
# replace encoded categories by the original values. get_column()
# rather than nw.col() again, to support pandas integer column names.
nw_X = nw.from_native(X, eager_only=True)
new_series = [
nw_X.get_column(feature).replace_strict(
{v: k for k, v in mapping.items()}, default=None
)
for feature, mapping in self.encoder_dict_.items()
]
X = nw_X.with_columns(*new_series).to_native()

return X

Expand Down