From 225d1fc6ec199380bf1075d17d9d5e77ce577998 Mon Sep 17 00:00:00 2001 From: "Yi (Alan) Wang" Date: Fri, 12 May 2017 18:16:53 -0700 Subject: [PATCH] Modify HIVE ETL to commit often (#505) --- wherehows-etl/src/main/resources/jython/HiveTransform.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wherehows-etl/src/main/resources/jython/HiveTransform.py b/wherehows-etl/src/main/resources/jython/HiveTransform.py index ba83e8a72b..9d95e2d372 100644 --- a/wherehows-etl/src/main/resources/jython/HiveTransform.py +++ b/wherehows-etl/src/main/resources/jython/HiveTransform.py @@ -119,9 +119,9 @@ class HiveTransform: db_name = names[0].lower() table_name = names[1].lower() if db_name and table_name: - rows = [] self.curs.execute(depends_sql.format(db_name=db_name, table_name=table_name, version='{version}')) rows = self.curs.fetchall() + self.conn_hms.commit() if rows and len(rows) > 0: for row_index, row_value in enumerate(rows): dependent_record = HiveDependencyInstanceRecord(