diff --git a/docs/user_guide/transformation/LogCpTransformer.rst b/docs/user_guide/transformation/LogCpTransformer.rst index 0f90f85af..b4b224d86 100644 --- a/docs/user_guide/transformation/LogCpTransformer.rst +++ b/docs/user_guide/transformation/LogCpTransformer.rst @@ -93,7 +93,7 @@ before applying the logarithm transformation: .. code:: python - {'MedInc': 0, 'HouseAge': 0} + {'MedInc': 0.0, 'HouseAge': 0.0} .. note:: @@ -298,6 +298,47 @@ And the constant values will be those from the dictionary: You can now apply `transform()` to transform all these variables. +With polars +----------- + +:class:`LogCpTransformer()` works in the same way with a polars dataframe: + +.. code:: python + + import polars as pl + from feature_engine.transformation import LogCpTransformer + + df = pl.DataFrame({"var_1": [-2.0, -1.0, 0.0, 1.0, 2.0]}) + + tf = LogCpTransformer(variables=None) + tf.fit(df) + + print(tf.C_) + +.. code:: text + + {'var_1': 3.0} + +.. code:: python + + print(tf.transform(df)) + +.. code:: text + + shape: (5, 1) + ┌──────────┐ + │ var_1 │ + │ --- │ + │ f64 │ + ╞══════════╡ + │ 0.0 │ + │ 0.693147 │ + │ 1.098612 │ + │ 1.386294 │ + │ 1.609438 │ + └──────────┘ + + Additional resources -------------------- diff --git a/docs/user_guide/transformation/LogTransformer.rst b/docs/user_guide/transformation/LogTransformer.rst index b73d317d8..8bab8fd38 100644 --- a/docs/user_guide/transformation/LogTransformer.rst +++ b/docs/user_guide/transformation/LogTransformer.rst @@ -217,6 +217,48 @@ mapping each variable to its own constant (``C={"bmi": 2, "s3": 3}``), the same way you would with the deprecated :class:`LogCpTransformer()`. +With polars +----------- + +:class:`LogTransformer()` works in the same way with a polars dataframe, including +the ``C="auto"`` shift for variables that contain zero or negative values: + +.. code:: python + + import polars as pl + from feature_engine.transformation import LogTransformer + + df = pl.DataFrame({"var_1": [-2.0, -1.0, 0.0, 1.0, 2.0]}) + + logt = LogTransformer(variables=None, C="auto") + logt.fit(df) + + print(logt.C_) + +.. code:: text + + {'var_1': 3.0} + +.. code:: python + + print(logt.transform(df)) + +.. code:: text + + shape: (5, 1) + ┌──────────┐ + │ var_1 │ + │ --- │ + │ f64 │ + ╞══════════╡ + │ 0.0 │ + │ 0.693147 │ + │ 1.098612 │ + │ 1.386294 │ + │ 1.609438 │ + └──────────┘ + + Additional resources -------------------- diff --git a/feature_engine/transformation/log.py b/feature_engine/transformation/log.py index 9cd12c307..0ee13ee35 100644 --- a/feature_engine/transformation/log.py +++ b/feature_engine/transformation/log.py @@ -4,8 +4,9 @@ import warnings from typing import Dict, List, Optional, Union +import narwhals as nw import numpy as np -import pandas as pd +from narwhals.typing import IntoDataFrame, IntoSeries from feature_engine._base_transformers.base_numerical import BaseNumericalTransformer from feature_engine._base_transformers.mixins import FitFromDictMixin @@ -126,6 +127,30 @@ class LogTransformer(BaseNumericalTransformer, FitFromDictMixin): 2 0.647689 3 1.523030 4 -0.234153 + + With polars: + + >>> import numpy as np + >>> import polars as pl + >>> from feature_engine.transformation import LogTransformer + >>> np.random.seed(42) + >>> X = pl.DataFrame({"x": list(np.random.lognormal(size=6))}) + >>> lt = LogTransformer() + >>> lt.fit(X) + >>> lt.transform(X) + shape: (6, 1) + ┌───────────┐ + │ x │ + │ --- │ + │ f64 │ + ╞═══════════╡ + │ 0.496714 │ + │ -0.138264 │ + │ 0.647689 │ + │ 1.52303 │ + │ -0.234153 │ + │ -0.234137 │ + └───────────┘ """ def __init__( @@ -154,7 +179,7 @@ def __init__( self.base = base self.C = C - def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): + def fit(self, X: IntoDataFrame, y: Optional[IntoSeries] = None): """ Learn the constant C to add to the variable before the logarithm transformation, if C="auto". Otherwise, this transformer does not learn @@ -162,11 +187,11 @@ def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): Parameters ---------- - X: pandas DataFrame of shape = [n_samples, n_features]. + X: dataframe of shape = [n_samples, n_features]. The training input samples. Can be the entire dataframe, not just the variables to transform. - y: pandas Series, default=None + y: Series, default=None It is not needed in this transformer. You can pass y or None. """ @@ -176,21 +201,21 @@ def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): else: X, variables_ = self._fit_setup(X) + values = nw.from_native(X, eager_only=True).select(variables_).to_numpy() + values = values.astype(float) + C_ = self.C - # calculate C to add to each variable + # 0 for strictly positive variables, abs(min) + 1 (shift to positive) + # otherwise. if self.C == "auto": - # we add 0 to positive variables - c_dict = {var: 0 for var in variables_ if X[var].min() > 0} - - # we add the minimum plus 1 to non-positive variables - non_positive_vars = [var for var in variables_ if var not in c_dict.keys()] - c_dict.update(dict(X[non_positive_vars].min(axis=0).abs() + 1)) - C_ = c_dict # type:ignore + mins = values.min(axis=0) + c_values = np.where(mins > 0, 0, np.abs(mins) + 1) + C_ = dict(zip(variables_, c_values.tolist())) # C=0 is the original LogTransformer contract: no constant is added, # so fail fast at fit time exactly as before this class supported C. - if C_ == 0 and (X[variables_] <= 0).any().any(): + if C_ == 0 and np.any(values <= 0): raise ValueError( "Some variables contain zero or negative values, can't apply log" ) @@ -201,18 +226,25 @@ def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): return self - def transform(self, X: pd.DataFrame) -> pd.DataFrame: + def _c_as_array(self) -> Union[int, float, np.ndarray]: + """Broadcastable form of C_: a plain scalar, or a numpy array ordered + to line up column-wise with self.variables_ when C_ is a dict.""" + if isinstance(self.C_, dict): + return np.array([self.C_[var] for var in self.variables_], dtype=float) + return self.C_ + + def transform(self, X: IntoDataFrame) -> IntoDataFrame: """ Transform the variables with the logarithm of x plus the constant C. Parameters ---------- - X: pandas DataFrame of shape = [n_samples, n_features] + X: dataframe of shape = [n_samples, n_features] The data to be transformed. Returns ------- - X_new: pandas dataframe + X_new: dataframe The dataframe with the transformed variables. """ @@ -229,42 +261,60 @@ def transform(self, X: pd.DataFrame) -> pd.DataFrame: + " constant C, can't apply log." ) - if (X[self.variables_] + self.C_ <= 0).any().any(): - raise ValueError(error_msg) + nw_X = nw.from_native(X, eager_only=True) + values = nw_X.select(self.variables_).to_numpy().astype(float) + shifted = values + self._c_as_array() - X[self.variables_] = X[self.variables_].astype(float) + if np.any(shifted <= 0): + raise ValueError(error_msg) # transform if self.base == "e": - X.loc[:, self.variables_] = np.log(X.loc[:, self.variables_] + self.C_) - elif self.base == "10": - X.loc[:, self.variables_] = np.log10(X.loc[:, self.variables_] + self.C_) + result = np.log(shifted) + else: + result = np.log10(shifted) + + new_series = [ + nw.new_series(var, result[:, i], backend=nw_X.implementation) + for i, var in enumerate(self.variables_) + ] + X = nw_X.with_columns(*new_series).to_native() return X - def inverse_transform(self, X: pd.DataFrame) -> pd.DataFrame: + def inverse_transform(self, X: IntoDataFrame) -> IntoDataFrame: """ Convert the data back to the original representation. Parameters ---------- - X: pandas DataFrame of shape = [n_samples, n_features] + X: dataframe of shape = [n_samples, n_features] The data to be transformed. Returns ------- - X_tr: pandas dataframe + X_tr: dataframe The dataframe with the transformed variables. """ # check input dataframe and if class was fitted X = self._check_transform_input_and_state(X) + nw_X = nw.from_native(X, eager_only=True) + values = nw_X.select(self.variables_).to_numpy().astype(float) + c_arr = self._c_as_array() + # inverse_transform if self.base == "e": - X.loc[:, self.variables_] = np.exp(X.loc[:, self.variables_]) - self.C_ - elif self.base == "10": - X.loc[:, self.variables_] = 10 ** X.loc[:, self.variables_] - self.C_ + result = np.exp(values) - c_arr + else: + result = 10**values - c_arr + + new_series = [ + nw.new_series(var, result[:, i], backend=nw_X.implementation) + for i, var in enumerate(self.variables_) + ] + X = nw_X.with_columns(*new_series).to_native() return X diff --git a/tests/test_transformation/test_log_transformer.py b/tests/test_transformation/test_log_transformer.py index 23a74104c..757becf5d 100644 --- a/tests/test_transformation/test_log_transformer.py +++ b/tests/test_transformation/test_log_transformer.py @@ -1,175 +1,195 @@ +import re + +import narwhals as nw import numpy as np import pandas as pd +import polars as pl import pytest from sklearn.exceptions import NotFittedError from feature_engine.transformation import LogTransformer - -def test_transforming_int_vars(): - df = pd.DataFrame( - { - "var1": [1, 2, 3], - "var2": [4, 5, 3], - } - ) - dft = np.log(df) +DATA = { + "Name": ["tom", "nick", "krish", "jack"], + "City": ["London", "Manchester", "Liverpool", "Bristol"], + "Age": [20, 21, 19, 18], + "Marks": [0.9, 0.8, 0.7, 0.6], +} +DATA_NA = { + "Name": ["tom", "nick", "krish", "jack"], + "City": ["London", "Manchester", "Liverpool", "Bristol"], + "Age": [20.0, 21.0, 19.0, np.nan], + "Marks": [0.9, 0.8, 0.7, np.nan], +} +DATA_C = { + "vara": [0, 1, 2, 3], + "varb": [5, 5, 6, 7], + "varc": [-2, -1, 0, 4], + "vard": [-3, -2, -1, -5], + "vare": ["a", "b", "c", "d"], +} +DATA_C_VARS = ["vara", "varb", "varc", "vard"] +DATA_C_AUTO = {"vara": 1, "varb": 0, "varc": 3, "vard": 6} + + +def _to_dict(X): + return nw.from_native(X, eager_only=True).to_dict(as_series=False) + + +def _expected_log(c, base): + fn = np.log if base == "e" else np.log10 + out = {} + for var in DATA_C_VARS: + c_var = c[var] if isinstance(c, dict) else c + out[var] = [fn(x + c_var) for x in DATA_C[var]] + return out + + +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_transforming_int_vars(make_df): + X = make_df({"var1": [1, 2, 3], "var2": [4, 5, 3]}) transformer = LogTransformer(base="e", variables=None) - X = transformer.fit_transform(df) - pd.testing.assert_frame_equal(X, dft) + Xt = transformer.fit_transform(X) + result = _to_dict(Xt) + assert result["var1"] == pytest.approx(list(np.log([1, 2, 3]))) + assert result["var2"] == pytest.approx(list(np.log([4, 5, 3]))) -def test_log_base_e_plus_automatically_find_variables(df_vartypes): - # test case 1: log base e, automatically select variables +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_log_base_e_plus_automatically_find_variables(make_df): + X = make_df(DATA) transformer = LogTransformer(base="e", variables=None) - X = transformer.fit_transform(df_vartypes) - - # expected output - transf_df = df_vartypes.copy() - transf_df["Age"] = [2.99573, 3.04452, 2.94444, 2.89037] - transf_df["Marks"] = [-0.105361, -0.223144, -0.356675, -0.510826] + Xt = transformer.fit_transform(X) # test init params assert transformer.base == "e" assert transformer.variables is None # test fit attr assert transformer.variables_ == ["Age", "Marks"] - assert transformer.n_features_in_ == 5 + assert transformer.n_features_in_ == 4 + # test transform output - pd.testing.assert_frame_equal(X, transf_df) + result = _to_dict(Xt) + assert result["Age"] == pytest.approx( + [2.99573, 3.04452, 2.94444, 2.89037], abs=1e-5 + ) + assert result["Marks"] == pytest.approx( + [-0.105361, -0.223144, -0.356675, -0.510826], abs=1e-5 + ) # test inverse_transform - Xit = transformer.inverse_transform(X) - - # convert numbers to original format. - Xit["Age"] = Xit["Age"].round().astype("int64") - Xit["Marks"] = Xit["Marks"].round(1) + Xit = transformer.inverse_transform(Xt) + result_it = _to_dict(Xit) + assert [round(v) for v in result_it["Age"]] == DATA["Age"] + assert [round(v, 1) for v in result_it["Marks"]] == DATA["Marks"] - # test - pd.testing.assert_frame_equal(Xit, df_vartypes) - -def test_log_base_10_plus_user_passes_var_list(df_vartypes): - # test case 2: log base 10, user passes variables +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_log_base_10_plus_user_passes_var_list(make_df): + X = make_df(DATA) transformer = LogTransformer(base="10", variables="Age") - X = transformer.fit_transform(df_vartypes) - - # expected output - transf_df = df_vartypes.copy() - transf_df["Age"] = [1.30103, 1.32222, 1.27875, 1.25527] + Xt = transformer.fit_transform(X) # test init params assert transformer.base == "10" assert transformer.variables == "Age" # test fit attr assert transformer.variables_ == ["Age"] - assert transformer.n_features_in_ == 5 + assert transformer.n_features_in_ == 4 + # test transform output - pd.testing.assert_frame_equal(X, transf_df) + result = _to_dict(Xt) + assert result["Age"] == pytest.approx( + [1.30103, 1.32222, 1.27875, 1.25527], abs=1e-5 + ) # test inverse_transform - Xit = transformer.inverse_transform(X) - - # convert numbers to original format. - Xit["Age"] = Xit["Age"].round().astype("int64") - - # test - pd.testing.assert_frame_equal(Xit, df_vartypes) + Xit = transformer.inverse_transform(Xt) + result_it = _to_dict(Xit) + assert [round(v) for v in result_it["Age"]] == DATA["Age"] def test_error_if_base_value_not_allowed(): - with pytest.raises(ValueError) as record: + msg = "base can take only '10' or 'e' as values. Got other instead." + with pytest.raises(ValueError, match=re.escape(msg)): LogTransformer(base="other") - assert str(record.value) == ( - "base can take only '10' or 'e' as values. Got other instead." - ) -def test_fit_raises_error_if_na_in_df(df_na): - # test case 3: when dataset contains na, fit method +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_fit_raises_error_if_na_in_df(make_df): + X = make_df(DATA_NA) with pytest.raises(ValueError): transformer = LogTransformer() - transformer.fit(df_na) + transformer.fit(X) -def test_transform_raises_error_if_na_in_df(df_vartypes, df_na): - # test case 4: when dataset contains na, transform method +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_transform_raises_error_if_na_in_df(make_df): + X = make_df(DATA) + X_na = make_df(DATA_NA) + transformer = LogTransformer() + transformer.fit(X) with pytest.raises(ValueError): - transformer = LogTransformer() - transformer.fit(df_vartypes) - transformer.transform(df_na[["Name", "City", "Age", "Marks", "dob"]]) + transformer.transform(X_na) -def test_error_if_df_contains_negative_values(df_vartypes): - # test error when data contains negative values - df_neg = df_vartypes.copy() - df_neg.loc[1, "Age"] = -1 +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_error_if_df_contains_negative_values(make_df): + data_neg = dict(DATA) + data_neg["Age"] = [20, -1, 19, 18] + X = make_df(DATA) + X_neg = make_df(data_neg) - # test case 5: when variable contains negative value, fit + # when variable contains negative value, fit with pytest.raises(ValueError): transformer = LogTransformer() - transformer.fit(df_neg) + transformer.fit(X_neg) - # test case 6: when variable contains negative value, transform + # when variable contains negative value, transform with pytest.raises(ValueError): transformer = LogTransformer() - transformer.fit(df_vartypes) - transformer.transform(df_neg) + transformer.fit(X) + transformer.transform(X_neg) -def test_non_fitted_error(df_vartypes): +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_non_fitted_error(make_df): + X = make_df(DATA) with pytest.raises(NotFittedError): transformer = LogTransformer() - transformer.transform(df_vartypes) + transformer.transform(X) -def test_inverse_e_plus_user_passes_var_list(df_vartypes): - # test case 7: inverse log, user passes variables +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_inverse_e_plus_user_passes_var_list(make_df): + X = make_df(DATA) transformer = LogTransformer(variables="Age") - Xt = transformer.fit_transform(df_vartypes) - X = transformer.inverse_transform(Xt) - - # convert floats to int - X["Age"] = X["Age"].round().astype("int64") + Xt = transformer.fit_transform(X) + Xit = transformer.inverse_transform(Xt) # test init params assert transformer.base == "e" assert transformer.variables == "Age" # test fit attr assert transformer.variables_ == ["Age"] - assert transformer.n_features_in_ == 5 + assert transformer.n_features_in_ == 4 # test transform output - pd.testing.assert_frame_equal(X, df_vartypes) + result_it = _to_dict(Xit) + assert [round(v) for v in result_it["Age"]] == DATA["Age"] -def test_default_C_preserves_original_fail_fast_behavior(): +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_default_C_preserves_original_fail_fast_behavior(make_df): """LogTransformer()'s default C=0 must raise at fit() time, with the original exact message, matching pre-merge behavior. See #957.""" - df = pd.DataFrame({"x": [1, 2, 0, 4]}) + X = make_df({"x": [1, 2, 0, 4]}) tr = LogTransformer() assert tr.C == 0 - with pytest.raises(ValueError) as record: - tr.fit(df) - - assert str(record.value) == ( - "Some variables contain zero or negative values, can't apply log" - ) - - -@pytest.fixture(scope="module") -def df_c(): - df = pd.DataFrame( - { - "vara": [0, 1, 2, 3], - "varb": [5, 5, 6, 7], - "varc": [-2, -1, 0, 4], - "vard": [-3, -2, -1, -5], - "vare": ["a", "b", "c", "d"], - } - ) - return df + msg = "Some variables contain zero or negative values, can't apply log" + with pytest.raises(ValueError, match=re.escape(msg)): + tr.fit(X) @pytest.mark.parametrize("c", [1, 0.1, {"var1": 1, "var2": 2}, "auto"]) @@ -181,86 +201,86 @@ def test_c_parameter(c): @pytest.mark.parametrize("c", ["string", [1, 2]]) def test_c_raises_error(c): msg = f"C can take only 'auto', integers, floats or dictionaries. Got {c} instead." - with pytest.raises(ValueError) as record: + with pytest.raises(ValueError, match=re.escape(msg)): LogTransformer(C=c) - assert str(record.value) == msg -def test_C_when_auto(df_c): +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_C_when_auto(make_df): + X = make_df(DATA_C) tr = LogTransformer(C="auto") - tr.fit(df_c) - c = {"vara": 1, "varb": 0, "varc": 3, "vard": 6} - assert tr.C_ == c + tr.fit(X) + assert tr.C_ == DATA_C_AUTO -def test_C_when_dict(df_c): - c = {"vara": 1, "varb": 0, "varc": 3, "vard": 6} - tr = LogTransformer(C=c) - tr.fit(df_c) - assert tr.C_ == c +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_C_when_dict(make_df): + X = make_df(DATA_C) + tr = LogTransformer(C=DATA_C_AUTO) + tr.fit(X) + assert tr.C_ == DATA_C_AUTO -def test_C_when_int(df_c): +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_C_when_int(make_df): + X = make_df(DATA_C) tr = LogTransformer(C=10) - tr.fit(df_c) + tr.fit(X) assert tr.C_ == 10 -def test_raises_error_when_transformed_data_has_negative_values_with_C(df_c): +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_raises_error_when_transformed_data_has_negative_values_with_C(make_df): + X = make_df(DATA_C) tr = LogTransformer(C="auto") - tr.fit(df_c) - dft = df_c.copy() - dft["vara"] = dft["vara"] - 2 + tr.fit(X) + + data_shifted = dict(DATA_C) + data_shifted["vara"] = [v - 2 for v in DATA_C["vara"]] + Xt = make_df(data_shifted) + msg = ( "Some variables contain zero or negative values after adding constant C, " "can't apply log." ) - with pytest.raises(ValueError) as record: - tr.transform(dft) - assert str(record.value) == msg + with pytest.raises(ValueError, match=re.escape(msg)): + tr.transform(Xt) -def test_log_base_e_with_C(df_c): - dft = LogTransformer(C="auto").fit_transform(df_c) - exp = np.log( - df_c[["vara", "varb", "varc", "vard"]] - + {"vara": 1, "varb": 0, "varc": 3, "vard": 6} - ) - exp["vare"] = df_c["vare"] - pd.testing.assert_frame_equal(dft, exp) +@pytest.mark.parametrize("base", ["e", "10"]) +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_log_with_C(make_df, base): + X = make_df(DATA_C) - dft = LogTransformer(C=10).fit_transform(df_c) - exp = np.log(df_c[["vara", "varb", "varc", "vard"]] + 10) - exp["vare"] = df_c["vare"] - pd.testing.assert_frame_equal(dft, exp) + dft = LogTransformer(C="auto", base=base).fit_transform(X) + result = _to_dict(dft) + expected = _expected_log(DATA_C_AUTO, base) + for var in DATA_C_VARS: + assert result[var] == pytest.approx(expected[var], abs=1e-6) + assert result["vare"] == DATA_C["vare"] + dft = LogTransformer(C=10, base=base).fit_transform(X) + result = _to_dict(dft) + expected = _expected_log(10, base) + for var in DATA_C_VARS: + assert result[var] == pytest.approx(expected[var], abs=1e-6) + assert result["vare"] == DATA_C["vare"] -def test_log_base_10_with_C(df_c): - dft = LogTransformer(C="auto", base="10").fit_transform(df_c) - exp = np.log10( - df_c[["vara", "varb", "varc", "vard"]] - + {"vara": 1, "varb": 0, "varc": 3, "vard": 6} - ) - exp["vare"] = df_c["vare"] - pd.testing.assert_frame_equal(dft, exp) - dft = LogTransformer(C=10, base="10").fit_transform(df_c) - exp = np.log10(df_c[["vara", "varb", "varc", "vard"]] + 10) - exp["vare"] = df_c["vare"] - pd.testing.assert_frame_equal(dft, exp) +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_inverse_transform_with_C(make_df): + X = make_df(DATA_C) - -def test_inverse_transform_with_C(df_c): tr = LogTransformer(C="auto", base="10") - dft = tr.fit_transform(df_c) + dft = tr.fit_transform(X) orig = tr.inverse_transform(dft) - pd.testing.assert_frame_equal( - orig, df_c, check_dtype=False, check_exact=False, rtol=0.1 - ) + result = _to_dict(orig) + for var in DATA_C_VARS: + assert result[var] == pytest.approx(DATA_C[var], abs=0.1) tr = LogTransformer(C=10, base="e") - dft = tr.fit_transform(df_c) + dft = tr.fit_transform(X) orig = tr.inverse_transform(dft) - pd.testing.assert_frame_equal( - orig, df_c, check_dtype=False, check_exact=False, rtol=0.1 - ) + result = _to_dict(orig) + for var in DATA_C_VARS: + assert result[var] == pytest.approx(DATA_C[var], abs=0.1) diff --git a/tests/test_transformation/test_logcp_transformer.py b/tests/test_transformation/test_logcp_transformer.py index 753cc43bf..98e154735 100644 --- a/tests/test_transformation/test_logcp_transformer.py +++ b/tests/test_transformation/test_logcp_transformer.py @@ -1,10 +1,50 @@ +import re + +import narwhals as nw import numpy as np import pandas as pd +import polars as pl import pytest from sklearn.exceptions import NotFittedError from feature_engine.transformation import LogCpTransformer +DATA = { + "vara": [0, 1, 2, 3], + "varb": [5, 5, 6, 7], + "varc": [-2, -1, 0, 4], + "vard": [-3, -2, -1, -5], + "vare": ["a", "b", "c", "d"], +} +DATA_VARS = ["vara", "varb", "varc", "vard"] +DATA_AUTO_C = {"vara": 1, "varb": 0, "varc": 3, "vard": 6} + +DATA_VARTYPES = { + "Name": ["tom", "nick", "krish", "jack"], + "City": ["London", "Manchester", "Liverpool", "Bristol"], + "Age": [20, 21, 19, 18], + "Marks": [0.9, 0.8, 0.7, 0.6], +} +DATA_NA = { + "Name": ["tom", "nick", "krish", "jack"], + "City": ["London", "Manchester", "Liverpool", "Bristol"], + "Age": [20.0, 21.0, 19.0, np.nan], + "Marks": [0.9, 0.8, 0.7, np.nan], +} + + +def _to_dict(X): + return nw.from_native(X, eager_only=True).to_dict(as_series=False) + + +def _expected_log(c, base): + fn = np.log if base == "e" else np.log10 + out = {} + for var in DATA_VARS: + c_var = c[var] if isinstance(c, dict) else c + out[var] = [fn(x + c_var) for x in DATA[var]] + return out + @pytest.mark.parametrize("base", ["e", "10"]) def test_base_parameter(base): @@ -15,9 +55,8 @@ def test_base_parameter(base): @pytest.mark.parametrize("base", [False, 1, 10]) def test_base_raises_error(base): msg = f"base can take only '10' or 'e' as values. Got {base} instead." - with pytest.raises(ValueError) as record: + with pytest.raises(ValueError, match=re.escape(msg)): LogCpTransformer(base=base) - assert str(record.value) == msg @pytest.mark.parametrize("c", [1, 0.1, {"var1": 1, "var2": 2}, "auto"]) @@ -29,9 +68,8 @@ def test_c_parameter(c): @pytest.mark.parametrize("c", ["string", [1, 2]]) def test_c_raises_error(c): msg = f"C can take only 'auto', integers, floats or dictionaries. Got {c} instead." - with pytest.raises(ValueError) as record: + with pytest.raises(ValueError, match=re.escape(msg)): LogCpTransformer(C=c) - assert str(record.value) == msg def test_instantiation_raises_future_warning(): @@ -40,119 +78,111 @@ def test_instantiation_raises_future_warning(): "LogTransformer and will be removed in version 2.1.0. " 'Use LogTransformer(C="auto") instead.' ) - with pytest.warns(FutureWarning) as record: + with pytest.warns(FutureWarning, match=re.escape(msg)): LogCpTransformer() - assert str(record[0].message) == msg - - -@pytest.fixture(scope="module") -def df(): - df = pd.DataFrame( - { - "vara": [0, 1, 2, 3], - "varb": [5, 5, 6, 7], - "varc": [-2, -1, 0, 4], - "vard": [-3, -2, -1, -5], - "vare": ["a", "b", "c", "d"], - } - ) - return df -def test_C_when_auto(df): +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_C_when_auto(make_df): + X = make_df(DATA) tr = LogCpTransformer(C="auto") - tr.fit(df) - c = {"vara": 1, "varb": 0, "varc": 3, "vard": 6} - assert tr.C_ == c + tr.fit(X) + assert tr.C_ == DATA_AUTO_C -def test_C_when_dict(df): - c = {"vara": 1, "varb": 0, "varc": 3, "vard": 6} - tr = LogCpTransformer(C=c) - tr.fit(df) - assert tr.C_ == c +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_C_when_dict(make_df): + X = make_df(DATA) + tr = LogCpTransformer(C=DATA_AUTO_C) + tr.fit(X) + assert tr.C_ == DATA_AUTO_C -def test_C_when_int(df): +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_C_when_int(make_df): + X = make_df(DATA) tr = LogCpTransformer(C=10) - tr.fit(df) + tr.fit(X) assert tr.C_ == 10 -def test_raises_error_when_transformed_data_has_negative_values(df): +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_raises_error_when_transformed_data_has_negative_values(make_df): + X = make_df(DATA) tr = LogCpTransformer(C="auto") - tr.fit(df) - dft = df.copy() - dft["vara"] = dft["vara"] - 2 + tr.fit(X) + + data_shifted = dict(DATA) + data_shifted["vara"] = [v - 2 for v in DATA["vara"]] + Xt = make_df(data_shifted) + msg = ( "Some variables contain zero or negative values after adding constant C, " "can't apply log." ) - with pytest.raises(ValueError) as record: - tr.transform(dft) - assert str(record.value) == msg + with pytest.raises(ValueError, match=re.escape(msg)): + tr.transform(Xt) -def test_log_base_e(df): - dft = LogCpTransformer(C="auto").fit_transform(df) - exp = np.log( - df[["vara", "varb", "varc", "vard"]] - + {"vara": 1, "varb": 0, "varc": 3, "vard": 6} - ) - exp["vare"] = df["vare"] - pd.testing.assert_frame_equal(dft, exp) - - dft = LogCpTransformer(C=10).fit_transform(df) - exp = np.log(df[["vara", "varb", "varc", "vard"]] + 10) - exp["vare"] = df["vare"] - pd.testing.assert_frame_equal(dft, exp) +@pytest.mark.parametrize("base", ["e", "10"]) +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_log_with_C(make_df, base): + X = make_df(DATA) + dft = LogCpTransformer(C="auto", base=base).fit_transform(X) + result = _to_dict(dft) + expected = _expected_log(DATA_AUTO_C, base) + for var in DATA_VARS: + assert result[var] == pytest.approx(expected[var], abs=1e-6) + assert result["vare"] == DATA["vare"] -def test_log_base_10(df): - dft = LogCpTransformer(C="auto", base="10").fit_transform(df) - exp = np.log10( - df[["vara", "varb", "varc", "vard"]] - + {"vara": 1, "varb": 0, "varc": 3, "vard": 6} - ) - exp["vare"] = df["vare"] - pd.testing.assert_frame_equal(dft, exp) + dft = LogCpTransformer(C=10, base=base).fit_transform(X) + result = _to_dict(dft) + expected = _expected_log(10, base) + for var in DATA_VARS: + assert result[var] == pytest.approx(expected[var], abs=1e-6) + assert result["vare"] == DATA["vare"] - dft = LogCpTransformer(C=10, base="10").fit_transform(df) - exp = np.log10(df[["vara", "varb", "varc", "vard"]] + 10) - exp["vare"] = df["vare"] - pd.testing.assert_frame_equal(dft, exp) +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_inverse_transform(make_df): + X = make_df(DATA) -def test_inverse_transform(df): tr = LogCpTransformer(C="auto", base="10") - dft = tr.fit_transform(df) + dft = tr.fit_transform(X) orig = tr.inverse_transform(dft) - pd.testing.assert_frame_equal( - orig, df, check_dtype=False, check_exact=False, rtol=0.1 - ) + result = _to_dict(orig) + for var in DATA_VARS: + assert result[var] == pytest.approx(DATA[var], abs=0.1) tr = LogCpTransformer(C=10, base="e") - dft = tr.fit_transform(df) + dft = tr.fit_transform(X) orig = tr.inverse_transform(dft) - pd.testing.assert_frame_equal( - orig, df, check_dtype=False, check_exact=False, rtol=0.1 - ) + result = _to_dict(orig) + for var in DATA_VARS: + assert result[var] == pytest.approx(DATA[var], abs=0.1) + +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_raises_error_if_na_in_df(make_df): + X_na = make_df(DATA_NA) + X = make_df(DATA_VARTYPES) -def test_raises_error_if_na_in_df(df_na, df_vartypes): # when dataset contains na, fit method transformer = LogCpTransformer() with pytest.raises(ValueError): - transformer.fit(df_na) + transformer.fit(X_na) # when dataset contains na, transform method transformer = LogCpTransformer() - transformer.fit(df_vartypes) + transformer.fit(X) with pytest.raises(ValueError): - transformer.transform(df_na[["Name", "City", "Age", "Marks", "dob"]]) + transformer.transform(X_na) -def test_non_fitted_error(df_vartypes): +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_non_fitted_error(make_df): + X = make_df(DATA_VARTYPES) transformer = LogCpTransformer() with pytest.raises(NotFittedError): - transformer.transform(df_vartypes) + transformer.transform(X)