yujunjun/autogen
1
0
Fork 0
mirror of https://github.com/microsoft/autogen.git synced 2025-11-15 17:44:33 +00:00
Code Issues Packages Projects Releases Wiki Activity

datetime feature engineering (#285)

Browse Source 
resolve #284
When transforming test data, keep a derived column as long as it is kept in the training data.
This commit is contained in:
Chi Wang 2021-11-18 11:19:53 -08:00 committed by GitHub
parent 72caa2172d
commit db1fb9b47b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 47 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
flaml/data.py
View File

@ -269,12 +269,10 @@ class DataTransformer:
else:
X[column] = X[column].fillna("__NAN__")
cat_columns.append(column)
else:
# print(X[column].dtype.name)
if X[column].nunique(dropna=True) < 2:
elif X[column].nunique(dropna=True) < 2:
X.drop(columns=column, inplace=True)
drop = True
else:
else: # datetime or numeric
if X[column].dtype.name == "datetime64[ns]":
tmp_dt = X[column].dt
new_columns_dict = {
@ -288,16 +286,13 @@ class DataTransformer:
f"dayofyear_{column}": tmp_dt.dayofyear,
f"quarter_{column}": tmp_dt.quarter,
}
for new_col_name in new_columns_dict.keys():
for key, value in new_columns_dict.items():
if (
new_col_name not in X.columns
and new_columns_dict.get(new_col_name).nunique(
dropna=False
)
>= 2
key not in X.columns
and value.nunique(dropna=False) >= 2
):
X[new_col_name] = new_columns_dict.get(new_col_name)
num_columns.append(new_col_name)
X[key] = value
num_columns.append(key)
X[column] = X[column].map(datetime.toordinal)
datetime_columns.append(column)
del tmp_dt
@ -380,7 +375,6 @@ class DataTransformer:
if self._task == TS_FORECAST:
X = X.rename(columns={X.columns[0]: TS_TIMESTAMP_COL})
ds_col = X.pop(TS_TIMESTAMP_COL)
if datetime_columns:
for column in datetime_columns:
tmp_dt = X[column].dt
new_columns_dict = {
@ -394,13 +388,9 @@ class DataTransformer:
f"dayofyear_{column}": tmp_dt.dayofyear,
f"quarter_{column}": tmp_dt.quarter,
}
for new_col_name in new_columns_dict.keys():
if (
new_col_name not in X.columns
and new_columns_dict.get(new_col_name).nunique(dropna=False)
>= 2
):
X[new_col_name] = new_columns_dict.get(new_col_name)
for new_col_name, new_col_value in new_columns_dict.items():
if new_col_name not in X.columns and new_col_name in num_columns:
X[new_col_name] = new_col_value
X[column] = X[column].map(datetime.toordinal)
del tmp_dt
X = X[cat_columns + num_columns].copy()