diff --git a/wherehows-backend/jobs/templates/UMP_METRIC_ETL.job b/wherehows-backend/jobs/templates/UMP_METRIC_ETL.job new file mode 100644 index 0000000000..f03efea1f7 --- /dev/null +++ b/wherehows-backend/jobs/templates/UMP_METRIC_ETL.job @@ -0,0 +1,17 @@ +# Fetch UMP metric metadata + +# Common ETL configs +job.class=metadata.etl.JythonEtlJob +job.cron.expr=0 0 0/6 1/1 * ? * +job.timeout=600 +#job.cmd.params= +#job.disabled=1 +job.ref.id=99 + +# Jython scripts +job.jython.extract=jython/UmpMetricExtract.py +job.jython.transform=jython/UmpMetricTransform.py +job.jython.load=jython/UmpMetricLoad.py + +# Place to store metric metadata csv file +ump.metric_csv_file=/var/tmp/metadata/ump_v2_metric_definition.csv diff --git a/wherehows-common/src/main/java/wherehows/common/Constant.java b/wherehows-common/src/main/java/wherehows/common/Constant.java index 201d005beb..0d7b982374 100644 --- a/wherehows-common/src/main/java/wherehows/common/Constant.java +++ b/wherehows-common/src/main/java/wherehows/common/Constant.java @@ -264,4 +264,7 @@ public class Constant { // kerberos public static final String KRB5_REALM = "krb5.realm"; public static final String KRB5_KDC = "krb5.kdc"; + + // ump metrics + public static final String UMP_METRIC_CSV_FILE_KEY = "ump.metric_csv_file"; } diff --git a/wherehows-data-model/DDL/ETL_DDL/metric_metadata.sql b/wherehows-data-model/DDL/ETL_DDL/metric_metadata.sql index 798fa274f6..e0f33dca93 100644 --- a/wherehows-data-model/DDL/ETL_DDL/metric_metadata.sql +++ b/wherehows-data-model/DDL/ETL_DDL/metric_metadata.sql @@ -21,7 +21,7 @@ CREATE TABLE dict_business_metric ( `metric_group` VARCHAR(100) COMMENT 'Hierarchy Level 2' NULL, `metric_category` VARCHAR(100) COMMENT 'Hierarchy Level 3' NULL, `metric_sub_category` VARCHAR(100) COMMENT 'Additional Classification, such as Product, Line of Business' NULL, - `metric_level` VARCHAR(50) COMMENT 'CORE, DEPARTMENT, TEAM, OPERATION, STRATEGIC, TIER1, TIER2' NULL, + `metric_level` VARCHAR(50) COMMENT 'CORE, DEPARTMENT, TEAM, OPERATION, STRATEGIC, TIER1, TIER2' NULL, `metric_source_type` VARCHAR(50) COMMENT 'Table, Cube, File, Web Service' NULL, `metric_source` VARCHAR(300) CHAR SET latin1 COMMENT 'Table Name, Cube Name, URL' NULL, `metric_source_dataset_id` INT(11) COMMENT 'If metric_source can be matched in dict_dataset' NULL, @@ -34,18 +34,51 @@ CREATE TABLE dict_business_metric ( `metric_display_factor_sym` VARCHAR(20) COMMENT '%, (K), (M), (B), (GB), (TB), (PB)' NULL, `metric_good_direction` VARCHAR(20) COMMENT 'UP, DOWN, ZERO, FLAT' NULL, `metric_formula` TEXT COMMENT 'Expression, Code Snippet or Calculation Logic' NULL, - `dimensions` VARCHAR(500) CHAR SET latin1 NULL, + `dimensions` VARCHAR(800) CHAR SET latin1 NULL, `owners` VARCHAR(300) NULL, `tags` VARCHAR(300) NULL, `urn` VARCHAR(300) CHAR SET latin1 NOT NULL, `metric_url` VARCHAR(300) CHAR SET latin1 NULL, `wiki_url` VARCHAR(300) CHAR SET latin1 NULL, `scm_url` VARCHAR(300) CHAR SET latin1 NULL, + `wh_etl_exec_id` BIGINT COMMENT 'wherehows etl execution id that modified this record', PRIMARY KEY(metric_id), - KEY `idx_dict_business_metric__urn` (`urn`) USING BTREE, + UNIQUE KEY `uq_dataset_urn` (`urn`), KEY `idx_dict_business_metric__ref_id` (`metric_ref_id`) USING BTREE, FULLTEXT KEY `fti_dict_business_metric_all` (`metric_name`, `metric_description`, `metric_category`, `metric_group`, `dashboard_name`) ) ENGINE = InnoDB AUTO_INCREMENT = 0 ; + +CREATE TABLE `stg_dict_business_metric` ( + `metric_name` varchar(200) NOT NULL, + `metric_description` varchar(500) DEFAULT NULL, + `dashboard_name` varchar(100) DEFAULT NULL COMMENT 'Hierarchy Level 1', + `metric_group` varchar(100) DEFAULT NULL COMMENT 'Hierarchy Level 2', + `metric_category` varchar(100) DEFAULT NULL COMMENT 'Hierarchy Level 3', + `metric_sub_category` varchar(100) DEFAULT NULL COMMENT 'Additional Classification, such as Product, Line of Business', + `metric_level` varchar(50) DEFAULT NULL COMMENT 'CORE, DEPARTMENT, TEAM, OPERATION, STRATEGIC, TIER1, TIER2', + `metric_source_type` varchar(50) DEFAULT NULL COMMENT 'Table, Cube, File, Web Service', + `metric_source` varchar(300) CHARACTER SET latin1 DEFAULT NULL COMMENT 'Table Name, Cube Name, URL', + `metric_source_dataset_id` int(11) DEFAULT NULL COMMENT 'If metric_source can be matched in dict_dataset', + `metric_ref_id_type` varchar(50) CHARACTER SET latin1 DEFAULT NULL COMMENT 'DWH, ABTEST, FINANCE, SEGMENT, SALESAPP', + `metric_ref_id` varchar(100) CHARACTER SET latin1 DEFAULT NULL COMMENT 'ID in the reference system', + `metric_type` varchar(100) DEFAULT NULL COMMENT 'NUMBER, BOOLEAN, LIST', + `metric_additive_type` varchar(100) DEFAULT NULL COMMENT 'FULL, SEMI, NONE', + `metric_grain` varchar(100) DEFAULT NULL COMMENT 'DAILY, WEEKLY, UNIQUE, ROLLING 7D, ROLLING 30D', + `metric_display_factor` decimal(15,4) DEFAULT NULL, + `metric_display_factor_sym` varchar(20) DEFAULT NULL COMMENT '%, (K), (M), (B), (GB), (TB), (PB)', + `metric_good_direction` varchar(20) DEFAULT NULL COMMENT 'UP, DOWN, ZERO, FLAT', + `metric_formula` text COMMENT 'Expression, Code Snippet or Calculation Logic', + `dimensions` varchar(800) CHARACTER SET latin1 DEFAULT NULL, + `owners` varchar(300) DEFAULT NULL, + `tags` varchar(500) DEFAULT NULL, + `urn` varchar(300) CHARACTER SET latin1 NOT NULL, + `metric_url` varchar(300) CHARACTER SET latin1 DEFAULT NULL, + `wiki_url` varchar(300) CHARACTER SET latin1 DEFAULT NULL, + `scm_url` varchar(300) CHARACTER SET latin1 DEFAULT NULL, + `wh_etl_exec_id` BIGINT COMMENT 'wherehows etl execution id that modified this record', + PRIMARY KEY(`urn`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 +;