yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-12-12 10:35:51 +00:00
Code Issues Packages Projects Releases Wiki Activity

Modify HIVE ETL to commit often (#505)

Browse Source
This commit is contained in:
Yi (Alan) Wang 2017-05-12 18:16:53 -07:00 committed by Mars Lan
parent 35bc64f8d2
commit 225d1fc6ec
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
wherehows-etl/src/main/resources/jython/HiveTransform.py
View File

@ -119,9 +119,9 @@ class HiveTransform:
db_name = names[0].lower() db_name = names[0].lower()
table_name = names[1].lower() table_name = names[1].lower()
if db_name and table_name: if db_name and table_name:
rows = []
self.curs.execute(depends_sql.format(db_name=db_name, table_name=table_name, version='{version}')) self.curs.execute(depends_sql.format(db_name=db_name, table_name=table_name, version='{version}'))
rows = self.curs.fetchall() rows = self.curs.fetchall()
self.conn_hms.commit()
if rows and len(rows) > 0: if rows and len(rows) > 0:
for row_index, row_value in enumerate(rows): for row_index, row_value in enumerate(rows):
dependent_record = HiveDependencyInstanceRecord( dependent_record = HiveDependencyInstanceRecord(