Skip to content

Single Month Feature Set

Module for the Time Related Features and Config classes.

Author: Bryce Senekal

TimeFeatureSet

Bases: FeatureSet[TimeFeaturesConfig]

A feature set for generating features based on the relationship between outlets and relative dates.

Source code in amee_utils/feature_generator/feature_set/time_related.py 
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
class TimeFeatureSet(FeatureSet[TimeFeaturesConfig]):
    """A feature set for generating features based on the relationship between outlets and relative dates."""

    def calculate(
        self,
        df: DataFrame,
        dataset_config: DatasetConfig,
        feature_config: TimeFeaturesConfig,
        calculation_date: datetime,
    ) -> DataFrame:
        """
        Calculate the features for the given DataFrame, DatasetConfig, and FeatureConfig.

        Parameters
        ----------
        df : DataFrame
            The DataFrame to calculate features for.
        dataset_config : DatasetConfig
            The configuration for the dataset.
        feature_config : FeatureConfig
            The configuration for the features to generate.
        calculation_date : datetime
            Feature calculation date.

        Returns
        -------
        DataFrame
            A DataFrame with the calculated features.
        """
        keys = dataset_config.key_cols
        base = df.select(keys).distinct()

        filtered_df = self.filter_order_df(
            df=df,
            date_col=dataset_config.date_col,
            key_cols=keys,
            calc_date=calculation_date,
            months=feature_config.months_for_avg,
        )

        avg_order_time = self.time_btwn_orders(df=filtered_df, date_col=dataset_config.date_col, keys=keys)
        tenure = self.tenure(
            df=df,
            date_col=dataset_config.date_col,
            keys=keys,
            calc_date=calculation_date,
        )
        recency = self.time_since_last_order(
            df=df,
            date_col=dataset_config.date_col,
            keys=keys,
            calc_date=calculation_date,
        )

        return join_multiple_to_base(
            base_df=base,
            df_list=[
                avg_order_time,
                tenure,
                recency,
            ],
            key_cols=keys,
            join_type="left",
        )

    @staticmethod
    def filter_order_df(
        df: DataFrame,
        date_col: str,
        key_cols: list[str],
        calc_date: datetime,
        months: int,
    ) -> DataFrame:
        """Partition the DataFrame by the config key column(s) and then add Previous_Order by date.

        Previous_Order is used to filter to the last x months.

        Parameters
        ----------
        df : DataFrame
            The DataFrame to filter.
        date_col : str
            The date column in the DataFrame.
        key_cols : list[str]
            The key columns in the DataFrame.
        calc_date : datetime
            The calculation date.
        months : int
            The number of months to filter the data by.

        Returns
        -------
        DataFrame
            A DataFrame with the filtered data.
        """
        window = W.Window.partitionBy(key_cols).orderBy(date_col)
        return df.withColumn("Previous_Order", F.lag(F.col(date_col), 1).over(window)).filter(
            F.col(date_col) >= calc_date - relativedelta(months=months),
        )

    @staticmethod
    def time_btwn_orders(df: DataFrame, date_col: str, keys: list[str]) -> DataFrame:
        """Get the average time between each order per outlet over the last X months.

        Parameters
        ----------
        df : DataFrame
            The DataFrame to calculate the features for.
        date_col : str
            The date column in the DataFrame.
        keys : list[str]
            The key columns in the DataFrame.

        Returns
        -------
        DataFrame
            A DataFrame with the calculated features.
        """
        return (
            df.withColumn(
                "_Time_Difference",
                F.datediff(F.col(date_col), F.col("Previous_Order")),
            )
            .groupBy(keys)
            .agg(F.round(F.avg(F.col("_Time_Difference")), 2).alias("AVG_DAYS_BETWEEN_ORDERS"))
        )

    @staticmethod
    def tenure(df: DataFrame, date_col: str, keys: list[str], calc_date: datetime) -> DataFrame:
        """Get the time difference of calculation date to first invoice in the data per outlet.

        Parameters
        ----------
        df : DataFrame
            The DataFrame to calculate the features for.
        date_col : str
            The date column in the DataFrame.
        keys : list[str]
            The key columns in the DataFrame.
        calc_date : datetime
            The calculation date.

        Returns
        -------
        DataFrame
            A DataFrame with the calculated features.
        """
        return df.groupBy(keys).agg(
            F.datediff(
                F.lit(calc_date),
                F.min(F.col(date_col)),
            ).alias("TIME_SINCE_FIRST_ORDER")
        )

    @staticmethod
    def time_since_last_order(df, date_col: str, keys: list[str], calc_date: datetime) -> DataFrame:
        """Get the time difference of calculation date to last invoice in the data per outlet.

        Parameters
        ----------
        df : DataFrame
            The DataFrame to calculate the features for.
        date_col : str
            The date column in the DataFrame.
        keys : list[str]
            The key columns in the DataFrame.
        calc_date : datetime
            The calculation date.

        Returns
        -------
        DataFrame
            A DataFrame with the calculated features.
        """
        return df.groupBy(keys).agg(F.datediff(F.lit(calc_date), F.max(F.col(date_col))).alias("TIME_SINCE_LAST_ORDER"))

calculate(df, dataset_config, feature_config, calculation_date)

Calculate the features for the given DataFrame, DatasetConfig, and FeatureConfig.

Parameters:

Name Type Description Default
df DataFrame

The DataFrame to calculate features for.

 required
dataset_config DatasetConfig

The configuration for the dataset.

 required
feature_config FeatureConfig

The configuration for the features to generate.

 required
calculation_date datetime

Feature calculation date.

 required

Returns:

Type Description
DataFrame

A DataFrame with the calculated features.
Source code in amee_utils/feature_generator/feature_set/time_related.py 
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
def calculate(
    self,
    df: DataFrame,
    dataset_config: DatasetConfig,
    feature_config: TimeFeaturesConfig,
    calculation_date: datetime,
) -> DataFrame:
    """
    Calculate the features for the given DataFrame, DatasetConfig, and FeatureConfig.

    Parameters
    ----------
    df : DataFrame
        The DataFrame to calculate features for.
    dataset_config : DatasetConfig
        The configuration for the dataset.
    feature_config : FeatureConfig
        The configuration for the features to generate.
    calculation_date : datetime
        Feature calculation date.

    Returns
    -------
    DataFrame
        A DataFrame with the calculated features.
    """
    keys = dataset_config.key_cols
    base = df.select(keys).distinct()

    filtered_df = self.filter_order_df(
        df=df,
        date_col=dataset_config.date_col,
        key_cols=keys,
        calc_date=calculation_date,
        months=feature_config.months_for_avg,
    )

    avg_order_time = self.time_btwn_orders(df=filtered_df, date_col=dataset_config.date_col, keys=keys)
    tenure = self.tenure(
        df=df,
        date_col=dataset_config.date_col,
        keys=keys,
        calc_date=calculation_date,
    )
    recency = self.time_since_last_order(
        df=df,
        date_col=dataset_config.date_col,
        keys=keys,
        calc_date=calculation_date,
    )

    return join_multiple_to_base(
        base_df=base,
        df_list=[
            avg_order_time,
            tenure,
            recency,
        ],
        key_cols=keys,
        join_type="left",
    )

filter_order_df(df, date_col, key_cols, calc_date, months) staticmethod

Partition the DataFrame by the config key column(s) and then add Previous_Order by date.

Previous_Order is used to filter to the last x months.

Parameters:

Name Type Description Default
df DataFrame

The DataFrame to filter.

 required
date_col str

The date column in the DataFrame.

 required
key_cols list[str]

The key columns in the DataFrame.

 required
calc_date datetime

The calculation date.

 required
months int

The number of months to filter the data by.

 required

Returns:

Type Description
DataFrame

A DataFrame with the filtered data.
Source code in amee_utils/feature_generator/feature_set/time_related.py 
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
@staticmethod
def filter_order_df(
    df: DataFrame,
    date_col: str,
    key_cols: list[str],
    calc_date: datetime,
    months: int,
) -> DataFrame:
    """Partition the DataFrame by the config key column(s) and then add Previous_Order by date.

    Previous_Order is used to filter to the last x months.

    Parameters
    ----------
    df : DataFrame
        The DataFrame to filter.
    date_col : str
        The date column in the DataFrame.
    key_cols : list[str]
        The key columns in the DataFrame.
    calc_date : datetime
        The calculation date.
    months : int
        The number of months to filter the data by.

    Returns
    -------
    DataFrame
        A DataFrame with the filtered data.
    """
    window = W.Window.partitionBy(key_cols).orderBy(date_col)
    return df.withColumn("Previous_Order", F.lag(F.col(date_col), 1).over(window)).filter(
        F.col(date_col) >= calc_date - relativedelta(months=months),
    )

tenure(df, date_col, keys, calc_date) staticmethod

Get the time difference of calculation date to first invoice in the data per outlet.

Parameters:

Name Type Description Default
df DataFrame

The DataFrame to calculate the features for.

 required
date_col str

The date column in the DataFrame.

 required
keys list[str]

The key columns in the DataFrame.

 required
calc_date datetime

The calculation date.

 required

Returns:

Type Description
DataFrame

A DataFrame with the calculated features.
Source code in amee_utils/feature_generator/feature_set/time_related.py 
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
@staticmethod
def tenure(df: DataFrame, date_col: str, keys: list[str], calc_date: datetime) -> DataFrame:
    """Get the time difference of calculation date to first invoice in the data per outlet.

    Parameters
    ----------
    df : DataFrame
        The DataFrame to calculate the features for.
    date_col : str
        The date column in the DataFrame.
    keys : list[str]
        The key columns in the DataFrame.
    calc_date : datetime
        The calculation date.

    Returns
    -------
    DataFrame
        A DataFrame with the calculated features.
    """
    return df.groupBy(keys).agg(
        F.datediff(
            F.lit(calc_date),
            F.min(F.col(date_col)),
        ).alias("TIME_SINCE_FIRST_ORDER")
    )

time_btwn_orders(df, date_col, keys) staticmethod

Get the average time between each order per outlet over the last X months.

Parameters:

Name Type Description Default
df DataFrame

The DataFrame to calculate the features for.

 required
date_col str

The date column in the DataFrame.

 required
keys list[str]

The key columns in the DataFrame.

 required

Returns:

Type Description
DataFrame

A DataFrame with the calculated features.
Source code in amee_utils/feature_generator/feature_set/time_related.py 
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
@staticmethod
def time_btwn_orders(df: DataFrame, date_col: str, keys: list[str]) -> DataFrame:
    """Get the average time between each order per outlet over the last X months.

    Parameters
    ----------
    df : DataFrame
        The DataFrame to calculate the features for.
    date_col : str
        The date column in the DataFrame.
    keys : list[str]
        The key columns in the DataFrame.

    Returns
    -------
    DataFrame
        A DataFrame with the calculated features.
    """
    return (
        df.withColumn(
            "_Time_Difference",
            F.datediff(F.col(date_col), F.col("Previous_Order")),
        )
        .groupBy(keys)
        .agg(F.round(F.avg(F.col("_Time_Difference")), 2).alias("AVG_DAYS_BETWEEN_ORDERS"))
    )

time_since_last_order(df, date_col, keys, calc_date) staticmethod

Get the time difference of calculation date to last invoice in the data per outlet.

Parameters:

Name Type Description Default
df DataFrame

The DataFrame to calculate the features for.

 required
date_col str

The date column in the DataFrame.

 required
keys list[str]

The key columns in the DataFrame.

 required
calc_date datetime

The calculation date.

 required

Returns:

Type Description
DataFrame

A DataFrame with the calculated features.
Source code in amee_utils/feature_generator/feature_set/time_related.py 
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
@staticmethod
def time_since_last_order(df, date_col: str, keys: list[str], calc_date: datetime) -> DataFrame:
    """Get the time difference of calculation date to last invoice in the data per outlet.

    Parameters
    ----------
    df : DataFrame
        The DataFrame to calculate the features for.
    date_col : str
        The date column in the DataFrame.
    keys : list[str]
        The key columns in the DataFrame.
    calc_date : datetime
        The calculation date.

    Returns
    -------
    DataFrame
        A DataFrame with the calculated features.
    """
    return df.groupBy(keys).agg(F.datediff(F.lit(calc_date), F.max(F.col(date_col))).alias("TIME_SINCE_LAST_ORDER"))

TimeFeaturesConfig

Bases: FeatureConfig

Configuration class for the Time Related features based on the past month of data.

Attributes:

Name Type Description
months_for_avg int

Number of months of data from the calculation_date to include for calculating averages.
Source code in amee_utils/feature_generator/feature_set/time_related.py 
20
21
22
23
24
25
26
27
28
29
30
31
@attrs.define
class TimeFeaturesConfig(FeatureConfig):
    """
    Configuration class for the Time Related features based on the past month of data.

    Attributes
    ----------
    months_for_avg : int
        Number of months of data from the calculation_date to include for calculating averages.
    """

    months_for_avg: int = 3