yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-11-03 04:10:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

update the column names from schema to schema_text and view_expanded_text to ddl_text

Browse Source
This commit is contained in:
jbai 2016-07-20 18:01:25 -07:00
parent 33b05cde4b
commit f3c299480f
2 changed files with 8 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
data-model/DDL/ETL_DDL/dataset_metadata.sql
View File

@ -285,8 +285,8 @@ CREATE TABLE dict_dataset_instance (
logical_name varchar(250) NOT NULL, logical_name varchar(250) NOT NULL,
version varchar(30) COMMENT '1.2.3 or 0.3.131' NULL, version varchar(30) COMMENT '1.2.3 or 0.3.131' NULL,
version_sort_id bigint(20) COMMENT '4-digit for each version number: 000100020003, 000000030131' NOT NULL DEFAULT '0', version_sort_id bigint(20) COMMENT '4-digit for each version number: 000100020003, 000000030131' NOT NULL DEFAULT '0',
`schema` MEDIUMTEXT CHARACTER SET utf8 NULL, schema_text MEDIUMTEXT CHARACTER SET utf8 NULL,
view_expanded_text MEDIUMTEXT CHARACTER SET utf8 NULL, ddl_text MEDIUMTEXT CHARACTER SET utf8 NULL,
instance_created_time int(10) UNSIGNED COMMENT 'source instance created time' NULL, instance_created_time int(10) UNSIGNED COMMENT 'source instance created time' NULL,
created_time int(10) UNSIGNED COMMENT 'wherehows created time' NULL, created_time int(10) UNSIGNED COMMENT 'wherehows created time' NULL,
modified_time int(10) UNSIGNED COMMENT 'latest wherehows modified' NULL, modified_time int(10) UNSIGNED COMMENT 'latest wherehows modified' NULL,

12
metadata-etl/src/main/resources/jython/HiveLoad.py
View File

@ -98,7 +98,7 @@ class HiveLoad:
""".format(source_file=self.input_schema_file, db_id=self.db_id, wh_etl_exec_id=self.wh_etl_exec_id) """.format(source_file=self.input_schema_file, db_id=self.db_id, wh_etl_exec_id=self.wh_etl_exec_id)
for state in load_cmd.split(";"): for state in load_cmd.split(";"):
self.logger.debug(state) self.logger.info(state)
cursor.execute(state) cursor.execute(state)
self.conn_mysql.commit() self.conn_mysql.commit()
cursor.close() cursor.close()
@ -288,7 +288,7 @@ class HiveLoad:
FIELDS TERMINATED BY '\x1a' ESCAPED BY '\0' FIELDS TERMINATED BY '\x1a' ESCAPED BY '\0'
(dataset_urn, deployment_tier, data_center, server_cluster, slice, (dataset_urn, deployment_tier, data_center, server_cluster, slice,
status_id, native_name, logical_name, version, instance_created_time, status_id, native_name, logical_name, version, instance_created_time,
`schema`, view_expanded_text, abstract_dataset_urn) schema_text, ddl_text, abstract_dataset_urn)
SET db_id = {db_id}, SET db_id = {db_id},
created_time=unix_timestamp(now()), created_time=unix_timestamp(now()),
wh_etl_exec_id = {wh_etl_exec_id}; wh_etl_exec_id = {wh_etl_exec_id};
@ -311,8 +311,8 @@ class HiveLoad:
version, version,
version_sort_id, version_sort_id,
instance_created_time, instance_created_time,
`schema`, schema_text,
view_expanded_text, ddl_text,
created_time, created_time,
wh_etl_exec_id wh_etl_exec_id
) )
@ -323,14 +323,14 @@ class HiveLoad:
cast(substring_index(substring_index(s.version, '.', 2), '.', -1) as unsigned) * 10000 + cast(substring_index(substring_index(s.version, '.', 2), '.', -1) as unsigned) * 10000 +
cast(substring_index(s.version, '.', -1) as unsigned) cast(substring_index(s.version, '.', -1) as unsigned)
else 0 else 0
end version_sort_id, s.schema, s.view_expanded_text, end version_sort_id, s.schema_text, s.ddl_text,
s.instance_created_time, s.created_time, s.wh_etl_exec_id s.instance_created_time, s.created_time, s.wh_etl_exec_id
from stg_dict_dataset_instance s join dict_dataset d on s.dataset_id = d.id from stg_dict_dataset_instance s join dict_dataset d on s.dataset_id = d.id
where s.db_id = {db_id} where s.db_id = {db_id}
on duplicate key update on duplicate key update
deployment_tier=s.deployment_tier, data_center=s.data_center, server_cluster=s.server_cluster, slice=s.slice, deployment_tier=s.deployment_tier, data_center=s.data_center, server_cluster=s.server_cluster, slice=s.slice,
status_id=s.status_id, native_name=s.native_name, logical_name=s.logical_name, version=s.version, status_id=s.status_id, native_name=s.native_name, logical_name=s.logical_name, version=s.version,
`schema`=s.schema, view_expanded_text=s.view_expanded_text, schema_text=s.schema_text, ddl_text=s.ddl_text,
instance_created_time=s.instance_created_time, created_time=s.created_time, wh_etl_exec_id=s.wh_etl_exec_id instance_created_time=s.instance_created_time, created_time=s.created_time, wh_etl_exec_id=s.wh_etl_exec_id
; ;
""".format(source_file=self.input_instance_file, db_id=self.db_id, wh_etl_exec_id=self.wh_etl_exec_id) """.format(source_file=self.input_instance_file, db_id=self.db_id, wh_etl_exec_id=self.wh_etl_exec_id)