Skip to content

[MNT] migrate scaling module to narwhals #1004

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
ojassharma7 wants to merge 1 commit into
base: narwhals-migration
Choose a base branch
from
Open

[MNT] migrate scaling module to narwhals #1004

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
55 changes: 50 additions & 5 deletions docs/user_guide/scaling/MeanNormalisationScaler.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -137,11 +137,56 @@ In the following data, we see the scaled variables returned to their original re

.. code:: python

Name City Age Height Marks dob
0 tom London 20 1.80 0.9 2020-02-24 00:00:00
1 nick Manchester 21 1.77 0.8 2020-02-24 00:01:00
2 krish Liverpool 19 1.90 0.7 2020-02-24 00:02:00
3 jack Bristol 18 2.00 0.6 2020-02-24 00:03:00
Name City Age Height Marks dob
0 tom London 20.0 1.80 0.9 2020-02-24 00:00:00
1 nick Manchester 21.0 1.77 0.8 2020-02-24 00:01:00
2 krish Liverpool 19.0 1.90 0.7 2020-02-24 00:02:00
3 jack Bristol 18.0 2.00 0.6 2020-02-24 00:03:00

Note that **Age** comes back as a float, not the original integer: multiplying and
adding floats (the range and mean) always produces a float in both pandas and
polars, so the inverse transformation cannot restore the original integer dtype.

With polars
-----------

:class:`MeanNormalisationScaler()` works in the same way with a polars dataframe:

.. code:: python

import polars as pl
from feature_engine.scaling import MeanNormalisationScaler

df = pl.DataFrame(
{
"Name": ["tom", "nick", "krish", "jack"],
"City": ["London", "Manchester", "Liverpool", "Bristol"],
"Age": [20, 21, 19, 18],
"Height": [1.80, 1.77, 1.90, 2.00],
"Marks": [0.9, 0.8, 0.7, 0.6],
}
)

scaler = MeanNormalisationScaler(variables=["Age", "Marks", "Height"])
scaler.fit(df)

print(scaler.transform(df))

The resulting values match those found with pandas:

.. code:: text

shape: (4, 5)
┌───────┬────────────┬───────────┬───────────┬───────────┐
│ Name ┆ City ┆ Age ┆ Height ┆ Marks │
│ --- ┆ --- ┆ --- ┆ --- ┆ --- │
│ str ┆ str ┆ f64 ┆ f64 ┆ f64 │
╞═══════╪════════════╪═══════════╪═══════════╪═══════════╡
│ tom ┆ London ┆ 0.166667 ┆ -0.293478 ┆ 0.5 │
│ nick ┆ Manchester ┆ 0.5 ┆ -0.423913 ┆ 0.166667 │
│ krish ┆ Liverpool ┆ -0.166667 ┆ 0.141304 ┆ -0.166667 │
│ jack ┆ Bristol ┆ -0.5 ┆ 0.576087 ┆ -0.5 │
└───────┴────────────┴───────────┴───────────┴───────────┘


Additional resources
Expand Down
98 changes: 76 additions & 22 deletions feature_engine/scaling/mean_normalization.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,8 @@
import warnings
from typing import List, Optional, Union

import pandas as pd
import narwhals as nw
from narwhals.typing import IntoDataFrame, IntoSeries

from feature_engine._base_transformers.base_numerical import BaseNumericalTransformer
from feature_engine._check_init_parameters.check_init_input_params import (
Expand Down Expand Up @@ -96,12 +97,36 @@ class MeanNormalisationScaler(BaseNumericalTransformer):
>>> mns.fit(X)
>>> X = mns.transform(X)
>>> X.head()
x
0 0.496714
1 -0.138264
2 0.647689
3 1.523030
4 -0.234153
x
0 0.051125
1 -0.071456
2 0.093623
3 0.518122
4 -0.084093

With polars:

>>> import numpy as np
>>> import polars as pl
>>> from feature_engine.scaling import MeanNormalisationScaler
>>> np.random.seed(42)
>>> X = pl.DataFrame(dict(x = np.random.lognormal(size = 100)))
>>> mns = MeanNormalisationScaler()
>>> mns.fit(X)
>>> X = mns.transform(X)
>>> X.head()
shape: (5, 1)
┌───────────┐
│ x │
│ --- │
│ f64 │
╞═══════════╡
│ 0.051125 │
│ -0.071456 │
│ 0.093623 │
│ 0.518122 │
│ -0.084093 │
└───────────┘
"""

def __init__(
Expand All @@ -115,25 +140,36 @@ def __init__(
self.variables = _check_variables_input_value(variables)
self.return_empty = return_empty

def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None):
def fit(self, X: IntoDataFrame, y: Optional[IntoSeries] = None):
"""
Finds the mean and value range of each variable.

Parameters
----------
X: pandas dataframe of shape = [n_samples, n_features].
X: dataframe of shape = [n_samples, n_features].
The training input samples. Can be the entire dataframe, not just the
variables to transform.

y: pandas Series, default=None
y: Series, default=None
It is not needed in this transformer. You can pass y or None.
"""

# check input dataframe
X, variables_ = self._fit_setup(X)

mean_ = X[variables_].mean().to_dict()
range_ = (X[variables_].max() - X[variables_].min()).to_dict()
if len(variables_) == 0:
# return_empty=True can leave variables_ empty; narwhals' select([])
# collapses row count too, so .to_numpy() would reduce over 0 rows.
mean_: dict = {}
range_: dict = {}
else:
values = nw.from_native(X, eager_only=True).select(variables_).to_numpy()
mean_arr = values.mean(axis=0)
range_arr = values.max(axis=0) - values.min(axis=0)
# .tolist() converts numpy scalars to plain Python int/float,
# matching the dtype the old pandas .to_dict() used to return.
mean_ = dict(zip(variables_, mean_arr.tolist()))
range_ = dict(zip(variables_, range_arr.tolist()))

# check for constant columns
constant_columns = [col for col, value in range_.items() if value == 0]
Expand All @@ -150,51 +186,69 @@ def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None):

return self

def transform(self, X: pd.DataFrame) -> pd.DataFrame:
def transform(self, X: IntoDataFrame) -> IntoDataFrame:
"""
Transform the variables using mean normalisation.

Parameters
----------
X: pandas dataframe of shape = [n_samples, n_features]
X: dataframe of shape = [n_samples, n_features]
The data to be transformed.

Returns
-------
X_new: pandas dataframe
X_new: dataframe
The dataframe with the transformed variables.
"""

# check input dataframe and if class was fitted
X = self._check_transform_input_and_state(X)

# transformation
X[self.variables_] = (X[self.variables_] - self.mean_) / self.range_
nw_X = nw.from_native(X, eager_only=True)
new_series = [
nw.new_series(
var,
(nw_X.get_column(var).to_numpy() - self.mean_[var]) / self.range_[var],
backend=nw_X.implementation,
)
for var in self.variables_
]
nw_X = nw_X.with_columns(*new_series)

return X
return nw_X.to_native()

def inverse_transform(self, X: pd.DataFrame) -> pd.DataFrame:
def inverse_transform(self, X: IntoDataFrame) -> IntoDataFrame:
"""
Convert the data back to the original representation.

Parameters
----------
X: pandas dataframe of shape = [n_samples, n_features]
X: dataframe of shape = [n_samples, n_features]
The data to be transformed.

Returns
-------
X_tr: pandas dataframe
X_tr: dataframe
The dataframe with the transformed variables.
"""

# check input dataframe and if class was fitted
X = self._check_transform_input_and_state(X)

# inverse transform
X[self.variables_] = X[self.variables_] * self.range_ + self.mean_
nw_X = nw.from_native(X, eager_only=True)
new_series = [
nw.new_series(
var,
nw_X.get_column(var).to_numpy() * self.range_[var] + self.mean_[var],
backend=nw_X.implementation,
)
for var in self.variables_
]
nw_X = nw_X.with_columns(*new_series)

return X
return nw_X.to_native()


# TODO: remove in version 2.1.0
Expand Down
Loading