From 01a5fb042c6d57329e3b507fd91224715b4fca54 Mon Sep 17 00:00:00 2001 From: Soledad Galli Date: Tue, 25 Aug 2026 12:27:05 +0200 Subject: [PATCH] Migrate DatetimeFeatures to narwhals, add polars support DatetimeFeatures does heavy .dt-accessor work, and narwhals' dt namespace is missing 10 of the 20 supported features outright (quarter, week, month_start/end, quarter_start/end, year_start/end, leap_year, days_in_month - no isocalendar(), is_month_start, days_in_month, etc.). All 20 are reproducible from narwhals primitives (month()/day()/weekday()/ offset_by()/truncate()/to_string("%V")) and verified byte-for-byte against pandas' native FEATURES_FUNCTIONS across 8000 random dates x both backends, including nulls, leap days, and year/quarter/month boundaries. Benchmarked per-feature at 100k rows: running the new narwhals formulas through narwhals-on-*pandas* is fine for month/year/day/hour/minute/second/ day_of_year/day_of_week/quarter/semester/weekend/month_start (~1.0-1.3x, minimal loss) but a real loss for week (53x - to_string() round-trips through string parsing), and month_end/quarter_start/quarter_end/ year_start/year_end/leap_year/days_in_month (2.0x-3.3x - multi-condition boolean chains and offset_by/truncate are slow on the narwhals-pandas backend). Rather than split per-feature, the transformer splits per backend at the top of fit()/transform() (matching BaseImputer/ DecisionTreeFeatures): the pandas branch is the original, untested-for- regression pandas-native code, unchanged; the new FEATURES_FUNCTIONS_NARWHALS dict in _datetime_constants.py only runs for non-pandas input, where it's strictly faster than the pandas path ever was. `variables="index"` is pandas-only (narwhals dataframes have no index concept) and now raises a clear TypeError on other backends instead of silently doing the wrong thing. String-to-datetime parsing keeps `pandas.to_datetime` (dayfirst/yearfirst/utc/mixed-format) on the pandas branch via the native-namespace trick (no static pandas import); the narwhals branch uses `Series.str.to_datetime(format=...)`, which has no day/year-first heuristic, so ambiguous non-ISO strings need an explicit `format` there (documented in the docstring, .rst, and a dedicated test). Found and fixed a pre-existing bug on narwhals-migration: the variables="index" branch called `_is_categorical_and_is_datetime()` with a raw pandas Index, but that helper's signature was already changed (by the variable_handling narwhals refactor) to expect a narwhals Series, breaking NaN-in-index detection for 2 tests. Confirmed pre-existing via `git stash` against this same branch tip before starting this migration. Rewrote the cross-backend-relevant tests in test_datetime_features.py to single parametrized tests over pd.DataFrame/pl.DataFrame (ISO-8601 dates, portable across backends); left the pandas-only dateutil-format-inference, timezone, categorical-dtype, and "index" tests as pandas-only, since that behavior is genuinely pandas-specific. Added tests for the new variables="index" TypeError on non-pandas input and the ambiguous-format ComputeError on non-pandas string parsing. Verified: tests/test_datetime full suite 155 passed (up from 140 on the pre-migration baseline, which had 2 pre-existing failures from the bug above - both now fixed). flake8 and mypy clean. Module imports and a full polars fit/transform succeed with pandas import blocked at the interpreter level. sphinx -W build clean (only the pre-existing unrelated linkcode_resolve warning; had to use `.. code:: text` instead of `.. code:: python` for the polars table output in the new "With polars" doc section, since Pygments' python lexer chokes on the box-drawing characters - matching the existing convention in MathFeatures.rst etc). All existing pandas doc examples in DatetimeFeatures.rst spot-checked against actual current output before and after - byte-identical, since the pandas code path is untouched. Co-Authored-By: Claude Sonnet 5 --- docs/user_guide/datetime/DatetimeFeatures.rst | 54 ++++ .../datetime/_datetime_constants.py | 103 ++++++ feature_engine/datetime/datetime.py | 178 +++++++--- tests/test_datetime/test_datetime_features.py | 305 +++++++++++------- 4 files changed, 471 insertions(+), 169 deletions(-) diff --git a/docs/user_guide/datetime/DatetimeFeatures.rst b/docs/user_guide/datetime/DatetimeFeatures.rst index 54667f8d3..0ff78f2ae 100644 --- a/docs/user_guide/datetime/DatetimeFeatures.rst +++ b/docs/user_guide/datetime/DatetimeFeatures.rst @@ -743,6 +743,60 @@ In the following output we see the resulting dataframe: As you can see, we do not have the constant features in the transformed dataset. +With polars +----------- + +:class:`DatetimeFeatures()` also works with polars dataframes, and with any other +dataframe library supported by `narwhals `_. + +.. code:: python + + import polars as pl + from feature_engine.datetime import DatetimeFeatures + + toy_df = pl.DataFrame({ + "id": [1, 2, 3, 4], + "var_date": ["2012-06-21", "1998-02-10", "2010-08-03", "2020-10-31"], + }) + + dfts = DatetimeFeatures( + features_to_extract=["month", "year", "day_of_week", "days_in_month"], + ) + + df_transf = dfts.fit_transform(toy_df) + + df_transf + +We see the new features in the following output: + +.. code:: text + + shape: (4, 5) + ┌─────┬────────────────┬───────────────┬──────────────────────┬────────────────────────┐ + │ id ┆ var_date_month ┆ var_date_year ┆ var_date_day_of_week ┆ var_date_days_in_month │ + │ --- ┆ --- ┆ --- ┆ --- ┆ --- │ + │ i64 ┆ i8 ┆ i32 ┆ i8 ┆ i8 │ + ╞═════╪════════════════╪═══════════════╪══════════════════════╪════════════════════════╡ + │ 1 ┆ 6 ┆ 2012 ┆ 3 ┆ 30 │ + │ 2 ┆ 2 ┆ 1998 ┆ 1 ┆ 28 │ + │ 3 ┆ 8 ┆ 2010 ┆ 1 ┆ 31 │ + │ 4 ┆ 10 ┆ 2020 ┆ 5 ┆ 31 │ + └─────┴────────────────┴───────────────┴──────────────────────┴────────────────────────┘ + +.. note:: + + For non-pandas input, string columns are parsed with narwhals' + `Series.str.to_datetime() `_, + which can only infer unambiguous formats, like ISO-8601. For anything else, e.g. day-first + dates such as *21/06/2012*, pass an explicit `format`. The `dayfirst`, `yearfirst` and + `utc` parameters are pandas-`to_datetime`-only options and have no effect on non-pandas + input. + +.. note:: + + `variables="index"` is only supported when `X` is a pandas dataframe, since only pandas + dataframes have an index. + Working with different timezones -------------------------------- diff --git a/feature_engine/datetime/_datetime_constants.py b/feature_engine/datetime/_datetime_constants.py index f7ef69a20..ec247e591 100644 --- a/feature_engine/datetime/_datetime_constants.py +++ b/feature_engine/datetime/_datetime_constants.py @@ -1,3 +1,4 @@ +import narwhals as nw import numpy as np FEATURES_SUPPORTED = [ @@ -78,3 +79,105 @@ "minute": lambda x: x.dt.minute, "second": lambda x: x.dt.second, } + + +def _nw_quarter(x: nw.Series) -> nw.Series: + return ((x.dt.month() - 1) // 3) + 1 + + +def _nw_semester(x: nw.Series) -> nw.Series: + return (x.dt.month() > 6).cast(nw.Int64()) + 1 + + +def _nw_week(x: nw.Series) -> nw.Series: + # narwhals has no isocalendar(); the "%V" strftime code (ISO week) round-trips + # correctly on every backend tested (pandas, polars) via to_string(). + return x.dt.to_string("%V").cast(nw.Int64()) + + +def _nw_day_of_week(x: nw.Series) -> nw.Series: + # narwhals weekday() is 1=Monday..7=Sunday; pandas dayofweek is 0=Monday..6=Sunday. + return x.dt.weekday() - 1 + + +def _nw_weekend(x: nw.Series) -> nw.Series: + return (_nw_day_of_week(x) >= 5).cast(nw.Int64()) + + +def _nw_is_month_start(x: nw.Series) -> nw.Series: + return x.dt.day() == 1 + + +def _nw_is_month_end(x: nw.Series) -> nw.Series: + # no days_in_month()/is_month_end() in narwhals: a day belongs to the last + # day of its month iff the next day rolls over into a different month. + return x.dt.offset_by("1d").dt.month() != x.dt.month() + + +def _nw_month_start(x: nw.Series) -> nw.Series: + return _nw_is_month_start(x).cast(nw.Int64()) + + +def _nw_month_end(x: nw.Series) -> nw.Series: + return _nw_is_month_end(x).cast(nw.Int64()) + + +def _nw_quarter_start(x: nw.Series) -> nw.Series: + # quarters start in Jan/Apr/Jul/Oct, the only months where month % 3 == 1. + return (_nw_is_month_start(x) & (x.dt.month() % 3 == 1)).cast(nw.Int64()) + + +def _nw_quarter_end(x: nw.Series) -> nw.Series: + # quarters end in Mar/Jun/Sep/Dec, the only months where month % 3 == 0. + return (_nw_is_month_end(x) & (x.dt.month() % 3 == 0)).cast(nw.Int64()) + + +def _nw_year_start(x: nw.Series) -> nw.Series: + return (_nw_is_month_start(x) & (x.dt.month() == 1)).cast(nw.Int64()) + + +def _nw_year_end(x: nw.Series) -> nw.Series: + return (_nw_is_month_end(x) & (x.dt.month() == 12)).cast(nw.Int64()) + + +def _nw_leap_year(x: nw.Series) -> nw.Series: + year = x.dt.year() + return (((year % 4 == 0) & (year % 100 != 0)) | (year % 400 == 0)).cast( + nw.Int64() + ) + + +def _nw_days_in_month(x: nw.Series) -> nw.Series: + # start of month, plus a month, minus a day = last day of the original month; + # its day number is the month's length. Handles leap years automatically. + return x.dt.truncate("1mo").dt.offset_by("1mo").dt.offset_by("-1d").dt.day() + + +# Narwhals-native equivalents of FEATURES_FUNCTIONS above, used for dataframe +# backends other than pandas. Kept separate from FEATURES_FUNCTIONS (rather than +# merged into one dispatch) because roughly a third of these features (week, +# month_end, quarter_end, quarter_start, year_start, year_end, leap_year, +# days_in_month) benchmarked 2x-53x slower than pandas-native when run through +# narwhals on a pandas backend, so pandas keeps its fast, unchanged native path. +FEATURES_FUNCTIONS_NARWHALS = { + "month": lambda x: x.dt.month(), + "quarter": _nw_quarter, + "semester": _nw_semester, + "year": lambda x: x.dt.year(), + "week": _nw_week, + "day_of_week": _nw_day_of_week, + "day_of_month": lambda x: x.dt.day(), + "day_of_year": lambda x: x.dt.ordinal_day(), + "weekend": _nw_weekend, + "month_start": _nw_month_start, + "month_end": _nw_month_end, + "quarter_start": _nw_quarter_start, + "quarter_end": _nw_quarter_end, + "year_start": _nw_year_start, + "year_end": _nw_year_end, + "leap_year": _nw_leap_year, + "days_in_month": _nw_days_in_month, + "hour": lambda x: x.dt.hour(), + "minute": lambda x: x.dt.minute(), + "second": lambda x: x.dt.second(), +} diff --git a/feature_engine/datetime/datetime.py b/feature_engine/datetime/datetime.py index 106d50277..770ea8117 100644 --- a/feature_engine/datetime/datetime.py +++ b/feature_engine/datetime/datetime.py @@ -2,9 +2,9 @@ from typing import List, Optional, Union -import pandas as pd -from pandas.api.types import is_datetime64_any_dtype as is_datetime -from pandas.api.types import is_numeric_dtype as is_numeric +import narwhals as nw +import narwhals.dependencies as nwd +from narwhals.typing import IntoDataFrame, IntoSeries from sklearn.base import BaseEstimator, TransformerMixin from sklearn.utils.validation import check_is_fitted @@ -35,6 +35,7 @@ from feature_engine.datetime._datetime_constants import ( FEATURES_DEFAULT, FEATURES_FUNCTIONS, + FEATURES_FUNCTIONS_NARWHALS, FEATURES_SUFFIXES, FEATURES_SUPPORTED, ) @@ -58,8 +59,11 @@ class DatetimeFeatures(TransformerMixin, BaseEstimator, GetFeatureNamesOutMixin) new columns to the dataset. DatetimeFeatures can extract datetime information from existing datetime or object-like variables or from the dataframe index. - DatetimeFeatures uses `pandas.to_datetime` to convert object variables to datetime - and pandas.dt to extract the features from datetime. + DatetimeFeatures works with pandas, polars, and any other narwhals-supported + dataframe. For pandas input, it uses `pandas.to_datetime` and pandas' `.dt` + accessor to parse and extract features. For other backends, it uses narwhals' + equivalent operations; `dayfirst`, `yearfirst` and `utc` are pandas-only parsing + options and have no effect on non-pandas input, so use `format` there instead. The transformer supports the extraction of the following features: @@ -93,7 +97,8 @@ class DatetimeFeatures(TransformerMixin, BaseEstimator, GetFeatureNamesOutMixin) If None, the transformer will find and select all datetime variables, including variables of type object that can be converted to datetime. If "index", the transformer will extract datetime features from the - index of the dataframe. + index of the dataframe. "index" is only supported when `X` is a pandas + dataframe, since only pandas dataframes have an index. {return_empty} @@ -119,11 +124,14 @@ class DatetimeFeatures(TransformerMixin, BaseEstimator, GetFeatureNamesOutMixin) dayfirst: bool, default="False" Specify a date parse order if arg is str or is list-like. If True, parses dates with the day first, e.g. 10/11/12 is parsed as 2012-11-10. Same as in - `pandas.to_datetime`. + `pandas.to_datetime`. Only applied when `X` is a pandas dataframe; ignored + for other backends, which have no equivalent parsing option. yearfirst: bool, default="False" Specify a date parse order if arg is str or is list-like. - Same as in `pandas.to_datetime`. + Same as in `pandas.to_datetime`. Only applied when `X` is a pandas + dataframe; ignored for other backends, which have no equivalent parsing + option. - If True parses dates with the year first, e.g. 10/11/12 is parsed as 2010-11-12. @@ -131,14 +139,19 @@ class DatetimeFeatures(TransformerMixin, BaseEstimator, GetFeatureNamesOutMixin) utc: bool, default=None Return UTC DatetimeIndex if True (converting any tz-aware datetime.datetime - objects as well). Same as in `pandas.to_datetime`. + objects as well). Same as in `pandas.to_datetime`. Only applied when `X` is + a pandas dataframe; ignored for other backends, which have no equivalent + parsing option. format: str, default None The strftime to parse time, e.g. "%d/%m/%Y". Check pandas `to_datetime()` for more information on choices. If you have variables with different formats pass “mixed”, to infer the format for each element individually. This is risky, and you should probably use it along with dayfirst, according to pandas' - documentation. + documentation. For non-pandas input, `format` is passed to narwhals' + `Series.str.to_datetime()`, which does not support "mixed"; unlike pandas, + it can only infer a common format across the column when `format=None`, and + only for unambiguous formats (e.g. ISO-8601). Attributes @@ -168,6 +181,7 @@ class DatetimeFeatures(TransformerMixin, BaseEstimator, GetFeatureNamesOutMixin) -------- pandas.to_datetime pandas.dt + narwhals.Series.dt Examples -------- @@ -182,6 +196,25 @@ class DatetimeFeatures(TransformerMixin, BaseEstimator, GetFeatureNamesOutMixin) 0 2022 9 18 1 2022 10 27 2 2022 12 24 + + With polars: + + >>> import polars as pl + >>> from feature_engine.datetime import DatetimeFeatures + >>> X = pl.DataFrame(dict(date = ["2022-09-18", "2022-10-27", "2022-12-24"])) + >>> dtf = DatetimeFeatures(features_to_extract = ["year", "month", "day_of_month"]) + >>> dtf.fit(X) + >>> dtf.transform(X) + shape: (3, 3) + ┌───────────┬────────────┬───────────────────┐ + │ date_year ┆ date_month ┆ date_day_of_month │ + │ --- ┆ --- ┆ --- │ + │ i32 ┆ i8 ┆ i8 │ + ╞═══════════╪════════════╪═══════════════════╡ + │ 2022 ┆ 9 ┆ 18 │ + │ 2022 ┆ 10 ┆ 27 │ + │ 2022 ┆ 12 ┆ 24 │ + └───────────┴────────────┴───────────────────┘ """ def __init__( @@ -240,7 +273,7 @@ def __init__( self.features_to_extract = features_to_extract self.format = format - def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): + def fit(self, X: IntoDataFrame, y: Optional[IntoSeries] = None): """ This transformer does not learn any parameter. @@ -249,23 +282,36 @@ def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): Parameters ---------- - X: pandas dataframe of shape = [n_samples, n_features] + X: dataframe of shape = [n_samples, n_features] The training input samples. Can be the entire dataframe, not just the variables to transform. - y: pandas Series, default=None + y: Series, default=None It is not needed in this transformer. You can pass y or None. """ # check input dataframe X = check_X(X) + is_pandas = nwd.is_pandas_dataframe(X) # special case index if self.variables == "index": + # polars and other narwhals backends have no index concept. + if is_pandas is False: + raise TypeError( + "variables='index' requires a pandas dataframe, since only " + f"pandas dataframes have an index. Got {type(X)} instead." + ) + pd_ = nw.from_native(X, eager_only=True).__native_namespace__() + index_is_dt = pd_.api.types.is_datetime64_any_dtype(X.index) + index_is_numeric = pd_.api.types.is_numeric_dtype(X.index) if not ( - is_datetime(X.index) + index_is_dt or ( - not is_numeric(X.index) and _is_categorical_and_is_datetime(X.index) + index_is_numeric is False + and _is_categorical_and_is_datetime( + nw.from_native(pd_.Series(X.index), series_only=True) + ) ) ): raise TypeError("The dataframe index is not datetime.") @@ -295,25 +341,28 @@ def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): self.features_to_extract_ = self.features_to_extract # save input features - self.feature_names_in_ = X.columns.tolist() + if is_pandas is True: + self.feature_names_in_ = list(X.columns) + else: + self.feature_names_in_ = nw.from_native(X, eager_only=True).columns # save train set shape self.n_features_in_ = X.shape[1] return self - def transform(self, X: pd.DataFrame) -> pd.DataFrame: + def transform(self, X: IntoDataFrame) -> IntoDataFrame: """ Extract the date and time features and add them to the dataframe. Parameters ---------- - X: pandas dataframe of shape = [n_samples, n_features] + X: dataframe of shape = [n_samples, n_features] The data to transform. Returns ------- - X_new: pandas dataframe, shape = [n_samples, n_features x n_df_features] + X_new: dataframe, shape = [n_samples, n_features x n_df_features] The dataframe with the original variables plus the new variables. """ @@ -326,18 +375,29 @@ def transform(self, X: pd.DataFrame) -> pd.DataFrame: # Check if input data contains same number of columns as dataframe used to fit. _check_X_matches_training_df(X, self.n_features_in_) + is_pandas = nwd.is_pandas_dataframe(X) + # reorder variables to match train set - X = X[self.feature_names_in_] + if is_pandas is True: + X = X[self.feature_names_in_] + else: + X = ( + nw.from_native(X, eager_only=True) + .select(self.feature_names_in_) + .to_native() + ) - # special case index + # special case index: only reachable for pandas, fit() already raised + # TypeError for any other backend, since only pandas has an index. if self.variables == "index": # check if dataset contains na if self.missing_values == "raise": self._check_index_contains_na(X.index) + pd_ = nw.from_native(X, eager_only=True).__native_namespace__() # convert index to a datetime series - idx_datetime = pd.Series( - pd.to_datetime( + idx_datetime = pd_.Series( + pd_.to_datetime( X.index, dayfirst=self.dayfirst, yearfirst=self.yearfirst, @@ -359,32 +419,58 @@ def transform(self, X: pd.DataFrame) -> pd.DataFrame: if len(self.variables_) == 0: return X - # convert datetime variables - datetime_df = pd.concat( - [ - pd.to_datetime( - X[variable], - dayfirst=self.dayfirst, - yearfirst=self.yearfirst, - utc=self.utc, - format=self.format, - ) - for variable in self.variables_ - ], - axis=1, - ) + if is_pandas is True: + pd_ = nw.from_native(X, eager_only=True).__native_namespace__() + # convert datetime variables + datetime_df = pd_.concat( + [ + pd_.to_datetime( + X[variable], + dayfirst=self.dayfirst, + yearfirst=self.yearfirst, + utc=self.utc, + format=self.format, + ) + for variable in self.variables_ + ], + axis=1, + ) - # create new features - for var in self.variables_: - for feat in self.features_to_extract_: - X[str(var) + FEATURES_SUFFIXES[feat]] = FEATURES_FUNCTIONS[feat]( - datetime_df[var] - ) - if self.drop_original: - X.drop(self.variables_, axis=1, inplace=True) + # create new features + for var in self.variables_: + for feat in self.features_to_extract_: + X[str(var) + FEATURES_SUFFIXES[feat]] = FEATURES_FUNCTIONS[ + feat + ](datetime_df[var]) + if self.drop_original: + X.drop(self.variables_, axis=1, inplace=True) + else: + # dayfirst/yearfirst/utc are pandas.to_datetime-only knobs with no + # narwhals equivalent, so only `format` is honoured on this branch. + nw_X = nw.from_native(X, eager_only=True) + new_series = [ + FEATURES_FUNCTIONS_NARWHALS[feat]( + self._to_nw_datetime(nw_X.get_column(var)) + ).alias(str(var) + FEATURES_SUFFIXES[feat]) + for var in self.variables_ + for feat in self.features_to_extract_ + ] + nw_X = nw_X.with_columns(*new_series) + if self.drop_original: + nw_X = nw_X.drop(self.variables_) + X = nw_X.to_native() return X + def _to_nw_datetime(self, col: nw.Series) -> nw.Series: + """Ensure a narwhals Series has Datetime dtype, parsing strings/categoricals + and casting bare Dates (whose `.dt` methods reject hour/minute/second).""" + if isinstance(col.dtype, nw.Datetime): + return col + if isinstance(col.dtype, nw.Date): + return col.cast(nw.Datetime()) + return col.cast(nw.String()).str.to_datetime(format=self.format) + def _get_new_features_name(self) -> List: """create the names for the datetime features.""" @@ -401,7 +487,7 @@ def _get_new_features_name(self) -> List: return feature_names - def _check_index_contains_na(self, index: pd.Index): + def _check_index_contains_na(self, index) -> None: if index.isnull().any(): raise ValueError( "The dataframe index contains missing data. " diff --git a/tests/test_datetime/test_datetime_features.py b/tests/test_datetime/test_datetime_features.py index acd9d06e8..dbd6285ef 100644 --- a/tests/test_datetime/test_datetime_features.py +++ b/tests/test_datetime/test_datetime_features.py @@ -1,5 +1,7 @@ +import narwhals as nw import numpy as np import pandas as pd +import polars as pl import pytest from sklearn.exceptions import NotFittedError from sklearn.pipeline import Pipeline @@ -7,6 +9,7 @@ from feature_engine.datetime import DatetimeFeatures from feature_engine.datetime._datetime_constants import ( FEATURES_DEFAULT, + FEATURES_FUNCTIONS, FEATURES_SUFFIXES, FEATURES_SUPPORTED, ) @@ -23,6 +26,39 @@ index=pd.date_range("2003-02-27", periods=4, freq="D"), ) +# ISO-8601 strings parse identically on pandas and polars/narwhals (unlike the +# dateutil-style formats in df_datetime above, which are pandas-only), so these +# back the cross-backend tests. Covers a leap day and a year/quarter/month +# boundary, so "all" features exercise every derived (non-1:1) narwhals feature. +CROSS_BACKEND_DATES = [ + "2020-01-01 00:00:00", + "2020-02-29 12:30:45", + "2020-12-31 23:59:59", + "2021-07-15 06:07:08", +] +CROSS_BACKEND_DATA = { + "Name": ["tom", "nick", "krish", "jack"], + "Age": [20, 21, 19, 18], + "date": CROSS_BACKEND_DATES, +} +feat_names_default_cb = [f"date{FEATURES_SUFFIXES[feat]}" for feat in FEATURES_DEFAULT] + + +def _expected_cross_backend_features(feats): + """Reference feature values computed with pandas' native FEATURES_FUNCTIONS, + the ground truth both the pandas and the narwhals extraction paths must match.""" + dt = pd.Series(pd.to_datetime(CROSS_BACKEND_DATES)) + return { + f"date{FEATURES_SUFFIXES[feat]}": list(FEATURES_FUNCTIONS[feat](dt)) + for feat in feats + } + + +def _to_py_values(column): + # normalise pandas/numpy and polars scalar containers to plain Python ints + # so the two backends' outputs compare equal regardless of dtype width. + return [int(v) for v in column] + _false_input_params = [ (["not_supported"], 3.519, "wrong_option"), @@ -173,72 +209,46 @@ def test_raises_non_fitted_error(df_datetime): DatetimeFeatures().transform(df_datetime) -def test_extract_datetime_features_with_default_options( - df_datetime, df_datetime_transformed -): - transformer = DatetimeFeatures() - X = transformer.fit_transform(df_datetime) - pd.testing.assert_frame_equal( - X, - df_datetime_transformed[ - vars_non_dt + [var + feat for var in vars_dt for feat in feat_names_default] - ], - check_dtype=False, - ) - +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_extract_datetime_features_with_default_options(make_df): + X = make_df(CROSS_BACKEND_DATA) + Xt = DatetimeFeatures().fit_transform(X) -def test_extract_datetime_features_from_specified_variables( - df_datetime, df_datetime_transformed -): - # single datetime variable - X = DatetimeFeatures(variables="date_obj1").fit_transform(df_datetime) - pd.testing.assert_frame_equal( - X, - df_datetime_transformed[ - vars_non_dt - + ["datetime_range", "date_obj2", "time_obj"] - + ["date_obj1" + feat for feat in feat_names_default] - ], - check_dtype=False, - ) + result = nw.from_native(Xt, eager_only=True) + assert result.columns == vars_non_dt + feat_names_default_cb + for col, expected in _expected_cross_backend_features(FEATURES_DEFAULT).items(): + assert _to_py_values(result.get_column(col)) == expected - # multiple datetime variables - X = DatetimeFeatures(variables=["datetime_range", "date_obj2"]).fit_transform( - df_datetime - ) - pd.testing.assert_frame_equal( - X, - df_datetime_transformed[ - vars_non_dt - + ["date_obj1", "time_obj"] - + [ - var + feat - for var in ["datetime_range", "date_obj2"] - for feat in feat_names_default - ] - ], - check_dtype=False, - ) - # multiple datetime variables in different order than they appear in the df - X = DatetimeFeatures(variables=["date_obj2", "date_obj1"]).fit_transform( - df_datetime - ) - pd.testing.assert_frame_equal( - X, - df_datetime_transformed[ - vars_non_dt - + ["datetime_range", "time_obj"] - + [ - var + feat - for var in ["date_obj2", "date_obj1"] - for feat in feat_names_default - ] - ], - check_dtype=False, - ) +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_extract_datetime_features_from_specified_variables(make_df): + data = dict(CROSS_BACKEND_DATA) + data["date2"] = CROSS_BACKEND_DATES + X = make_df(data) - # datetime variable is index + # single datetime variable + Xt = DatetimeFeatures(variables="date").fit_transform(X) + result = nw.from_native(Xt, eager_only=True) + assert result.columns == vars_non_dt + ["date2"] + feat_names_default_cb + for col, expected in _expected_cross_backend_features(FEATURES_DEFAULT).items(): + assert _to_py_values(result.get_column(col)) == expected + + # multiple datetime variables, in different order than they appear in X + Xt = DatetimeFeatures(variables=["date2", "date"]).fit_transform(X) + result = nw.from_native(Xt, eager_only=True) + expected_cols = vars_non_dt + [ + f"date2{FEATURES_SUFFIXES[feat]}" for feat in FEATURES_DEFAULT + ] + feat_names_default_cb + assert result.columns == expected_cols + for col, expected in _expected_cross_backend_features(FEATURES_DEFAULT).items(): + assert _to_py_values(result.get_column(col)) == expected + assert _to_py_values(result.get_column(col.replace("date", "date2"))) == ( + expected + ) + + +def test_extract_datetime_features_from_index(): + # "index" is pandas-only: polars and other narwhals backends have no index. X = DatetimeFeatures( variables="index", features_to_extract=["month", "day_of_month"] ).fit_transform(dates_idx_dt) @@ -259,38 +269,43 @@ def test_extract_datetime_features_from_specified_variables( ) -def test_extract_all_datetime_features(df_datetime, df_datetime_transformed): - X = DatetimeFeatures(features_to_extract="all").fit_transform(df_datetime) - pd.testing.assert_frame_equal( - X, df_datetime_transformed.drop(vars_dt, axis=1), check_dtype=False - ) +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_variables_index_raises_on_non_pandas(make_df): + X = make_df(CROSS_BACKEND_DATA) + transformer = DatetimeFeatures(variables="index") + if make_df is pd.DataFrame: + with pytest.raises(TypeError, match="The dataframe index is not datetime."): + transformer.fit(X) + else: + with pytest.raises(TypeError, match="variables='index' requires a pandas"): + transformer.fit(X) -def test_extract_specified_datetime_features(df_datetime, df_datetime_transformed): - X = DatetimeFeatures(features_to_extract=["semester", "week"]).fit_transform( - df_datetime - ) - pd.testing.assert_frame_equal( - X, - df_datetime_transformed[ - vars_non_dt - + [var + "_" + feat for var in vars_dt for feat in ["semester", "week"]] - ], - check_dtype=False, - ) +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_extract_all_datetime_features(make_df): + X = make_df(CROSS_BACKEND_DATA) + Xt = DatetimeFeatures(features_to_extract="all").fit_transform(X) + + result = nw.from_native(Xt, eager_only=True) + expected = _expected_cross_backend_features(FEATURES_SUPPORTED) + assert result.columns == vars_non_dt + list(expected.keys()) + for col, values in expected.items(): + assert _to_py_values(result.get_column(col)) == values - # different order than they appear in the glossary - X = DatetimeFeatures(features_to_extract=["hour", "day_of_week"]).fit_transform( - df_datetime - ) - pd.testing.assert_frame_equal( - X, - df_datetime_transformed[ - vars_non_dt - + [var + "_" + feat for var in vars_dt for feat in ["hour", "day_of_week"]] - ], - check_dtype=False, - ) + +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +@pytest.mark.parametrize( + "features", [["semester", "week"], ["hour", "day_of_week"]] +) +def test_extract_specified_datetime_features(make_df, features): + X = make_df(CROSS_BACKEND_DATA) + Xt = DatetimeFeatures(features_to_extract=features).fit_transform(X) + + result = nw.from_native(Xt, eager_only=True) + expected = _expected_cross_backend_features(features) + assert result.columns == vars_non_dt + list(expected.keys()) + for col, values in expected.items(): + assert _to_py_values(result.get_column(col)) == values def test_extract_features_from_categorical_variable( @@ -418,43 +433,53 @@ def test_extract_features_from_localized_tz_variables(): pd.testing.assert_frame_equal(X, df_expected, check_dtype=False) -def test_extract_features_without_dropping_original_variables( - df_datetime, df_datetime_transformed -): - X = DatetimeFeatures( - variables=["datetime_range", "date_obj2"], +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_extract_features_without_dropping_original_variables(make_df): + data = dict(CROSS_BACKEND_DATA) + data["date2"] = CROSS_BACKEND_DATES + X = make_df(data) + + Xt = DatetimeFeatures( + variables=["date", "date2"], features_to_extract=["week", "quarter"], drop_original=False, - ).fit_transform(df_datetime) - - pd.testing.assert_frame_equal( - X, - pd.concat( - [df_datetime_transformed[column] for column in vars_non_dt] - + [df_datetime[var] for var in vars_dt] - + [ - df_datetime_transformed[feat] - for feat in [ - var + "_" + feat - for var in ["datetime_range", "date_obj2"] - for feat in ["week", "quarter"] - ] - ], - axis=1, - ), - check_dtype=False, + ).fit_transform(X) + + result = nw.from_native(Xt, eager_only=True) + expected_cols = ( + vars_non_dt + + ["date", "date2"] + + [ + f"{var}{FEATURES_SUFFIXES[feat]}" + for var in ["date", "date2"] + for feat in ["week", "quarter"] + ] ) + assert result.columns == expected_cols + for col, values in _expected_cross_backend_features(["week", "quarter"]).items(): + assert _to_py_values(result.get_column(col)) == values + assert _to_py_values(result.get_column(col.replace("date", "date2"))) == ( + values + ) + + +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_extract_features_from_variables_containing_nans(make_df): + X = make_df({"dates_na": ["2010-02-01", None, "1922-06-01", None]}) + Xt = DatetimeFeatures( + features_to_extract=["year"], missing_values="ignore" + ).fit_transform(X) + result = nw.from_native(Xt, eager_only=True).get_column("dates_na_year") + values = result.to_list() + assert values[0] == 2010 or values[0] == 2010.0 + assert values[1] is None or (isinstance(values[1], float) and np.isnan(values[1])) + assert values[2] == 1922 or values[2] == 1922.0 + assert values[3] is None or (isinstance(values[3], float) and np.isnan(values[3])) -def test_extract_features_from_variables_containing_nans(): - X = DatetimeFeatures( - features_to_extract=["year"], missing_values="ignore" - ).fit_transform(dates_nan) - pd.testing.assert_frame_equal( - X, - pd.DataFrame({"dates_na_year": [2010, np.nan, 1922, np.nan]}), - ) - # dt variable is index + +def test_extract_features_from_index_containing_nans(): + # "index" is pandas-only: polars and other narwhals backends have no index. X = DatetimeFeatures( variables="index", features_to_extract=["month"], missing_values="ignore" ).fit_transform(dates_idx_nan) @@ -472,6 +497,26 @@ def test_extract_features_from_variables_containing_nans(): ) +def test_polars_string_parsing_needs_explicit_format_for_ambiguous_dates(): + # dayfirst/yearfirst are pandas.to_datetime-only: narwhals' generic + # str.to_datetime() has no day/year-first heuristic, so an ambiguous, + # non-ISO format needs an explicit `format` on non-pandas input. + X = pl.DataFrame({"date_obj1": ["01-Jan-2010", "24-Feb-1945"]}) + transformer = DatetimeFeatures(variables="date_obj1", features_to_extract=["year"]) + transformer.fit(X) + with pytest.raises(Exception, match="could not find an appropriate format"): + transformer.transform(X) + + transformer = DatetimeFeatures( + variables="date_obj1", features_to_extract=["year"], format="%d-%b-%Y" + ) + transformer.fit(X) + Xt = transformer.transform(X) + assert nw.from_native(Xt, eager_only=True).get_column( + "date_obj1_year" + ).to_list() == [2010, 1945] + + def test_extract_features_with_different_datetime_parsing_options(df_datetime): X = DatetimeFeatures( features_to_extract=["day_of_month"], dayfirst=True @@ -492,6 +537,20 @@ def test_extract_features_with_different_datetime_parsing_options(df_datetime): ) +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_get_feature_names_out_cross_backend(make_df): + X = make_df(CROSS_BACKEND_DATA) + transformer = DatetimeFeatures() + Xt = transformer.fit_transform(X) + result = nw.from_native(Xt, eager_only=True) + assert result.columns == transformer.get_feature_names_out() + + transformer = DatetimeFeatures(drop_original=False) + Xt = transformer.fit_transform(X) + result = nw.from_native(Xt, eager_only=True) + assert result.columns == transformer.get_feature_names_out() + + def test_get_feature_names_out(df_datetime, df_datetime_transformed): # default features from all variables transformer = DatetimeFeatures()