yujunjun/autogen
1
0
Fork 0
mirror of https://github.com/microsoft/autogen.git synced 2025-11-16 01:55:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

datetime feature engineering (#285)

Browse Source 
resolve #284
When transforming test data, keep a derived column as long as it is kept in the training data.
This commit is contained in:
Chi Wang 2021-11-18 11:19:53 -08:00 committed by GitHub
parent 72caa2172d
commit db1fb9b47b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 47 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
flaml/data.py
View File

@ -269,12 +269,10 @@ class DataTransformer:
else: else:
X[column] = X[column].fillna("__NAN__") X[column] = X[column].fillna("__NAN__")
cat_columns.append(column) cat_columns.append(column)
else: elif X[column].nunique(dropna=True) < 2:
# print(X[column].dtype.name)
if X[column].nunique(dropna=True) < 2:
X.drop(columns=column, inplace=True) X.drop(columns=column, inplace=True)
drop = True drop = True
else: else: # datetime or numeric
if X[column].dtype.name == "datetime64[ns]": if X[column].dtype.name == "datetime64[ns]":
tmp_dt = X[column].dt tmp_dt = X[column].dt
new_columns_dict = { new_columns_dict = {
@ -288,16 +286,13 @@ class DataTransformer:
f"dayofyear_{column}": tmp_dt.dayofyear, f"dayofyear_{column}": tmp_dt.dayofyear,
f"quarter_{column}": tmp_dt.quarter, f"quarter_{column}": tmp_dt.quarter,
} }
for new_col_name in new_columns_dict.keys(): for key, value in new_columns_dict.items():
if ( if (
new_col_name not in X.columns key not in X.columns
and new_columns_dict.get(new_col_name).nunique( and value.nunique(dropna=False) >= 2
dropna=False
)
>= 2
): ):
X[new_col_name] = new_columns_dict.get(new_col_name) X[key] = value
num_columns.append(new_col_name) num_columns.append(key)
X[column] = X[column].map(datetime.toordinal) X[column] = X[column].map(datetime.toordinal)
datetime_columns.append(column) datetime_columns.append(column)
del tmp_dt del tmp_dt
@ -380,7 +375,6 @@ class DataTransformer:
if self._task == TS_FORECAST: if self._task == TS_FORECAST:
X = X.rename(columns={X.columns[0]: TS_TIMESTAMP_COL}) X = X.rename(columns={X.columns[0]: TS_TIMESTAMP_COL})
ds_col = X.pop(TS_TIMESTAMP_COL) ds_col = X.pop(TS_TIMESTAMP_COL)
if datetime_columns:
for column in datetime_columns: for column in datetime_columns:
tmp_dt = X[column].dt tmp_dt = X[column].dt
new_columns_dict = { new_columns_dict = {
@ -394,13 +388,9 @@ class DataTransformer:
f"dayofyear_{column}": tmp_dt.dayofyear, f"dayofyear_{column}": tmp_dt.dayofyear,
f"quarter_{column}": tmp_dt.quarter, f"quarter_{column}": tmp_dt.quarter,
} }
for new_col_name in new_columns_dict.keys(): for new_col_name, new_col_value in new_columns_dict.items():
if ( if new_col_name not in X.columns and new_col_name in num_columns:
new_col_name not in X.columns X[new_col_name] = new_col_value
and new_columns_dict.get(new_col_name).nunique(dropna=False)
>= 2
):
X[new_col_name] = new_columns_dict.get(new_col_name)
X[column] = X[column].map(datetime.toordinal) X[column] = X[column].map(datetime.toordinal)
del tmp_dt del tmp_dt
X = X[cat_columns + num_columns].copy() X = X[cat_columns + num_columns].copy()