yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-11-03 12:16:10 +00:00
Code Issues Packages Projects Releases Wiki Activity

ump metric ETL

Browse Source
This commit is contained in:
Na Zhang 2017-08-16 11:05:02 -07:00
parent 93ebe430b5
commit ca34cd920d
3 changed files with 56 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
wherehows-backend/jobs/templates/UMP_METRIC_ETL.job Normal file
View File

@ -0,0 +1,17 @@
# Fetch UMP metric metadata
# Common ETL configs
job.class=metadata.etl.JythonEtlJob
job.cron.expr=0 0 0/6 1/1 * ? *
job.timeout=600
#job.cmd.params=
#job.disabled=1
job.ref.id=99
# Jython scripts
job.jython.extract=jython/UmpMetricExtract.py
job.jython.transform=jython/UmpMetricTransform.py
job.jython.load=jython/UmpMetricLoad.py
# Place to store metric metadata csv file
ump.metric_csv_file=/var/tmp/metadata/ump_v2_metric_definition.csv

3
wherehows-common/src/main/java/wherehows/common/Constant.java
View File

@ -264,4 +264,7 @@ public class Constant {
// kerberos
public static final String KRB5_REALM = "krb5.realm";
public static final String KRB5_KDC = "krb5.kdc";
// ump metrics
public static final String UMP_METRIC_CSV_FILE_KEY = "ump.metric_csv_file";
}

39
wherehows-data-model/DDL/ETL_DDL/metric_metadata.sql
View File

@ -21,7 +21,7 @@ CREATE TABLE dict_business_metric (
`metric_group` VARCHAR(100) COMMENT 'Hierarchy Level 2' NULL,
`metric_category` VARCHAR(100) COMMENT 'Hierarchy Level 3' NULL,
`metric_sub_category` VARCHAR(100) COMMENT 'Additional Classification, such as Product, Line of Business' NULL,
`metric_level` VARCHAR(50) COMMENT 'CORE, DEPARTMENT, TEAM, OPERATION, STRATEGIC, TIER1, TIER2' NULL,
`metric_level` VARCHAR(50) COMMENT 'CORE, DEPARTMENT, TEAM, OPERATION, STRATEGIC, TIER1, TIER2' NULL,
`metric_source_type` VARCHAR(50) COMMENT 'Table, Cube, File, Web Service' NULL,
`metric_source` VARCHAR(300) CHAR SET latin1 COMMENT 'Table Name, Cube Name, URL' NULL,
`metric_source_dataset_id` INT(11) COMMENT 'If metric_source can be matched in dict_dataset' NULL,
@ -34,18 +34,51 @@ CREATE TABLE dict_business_metric (
`metric_display_factor_sym` VARCHAR(20) COMMENT '%, (K), (M), (B), (GB), (TB), (PB)' NULL,
`metric_good_direction` VARCHAR(20) COMMENT 'UP, DOWN, ZERO, FLAT' NULL,
`metric_formula` TEXT COMMENT 'Expression, Code Snippet or Calculation Logic' NULL,
`dimensions` VARCHAR(500) CHAR SET latin1 NULL,
`dimensions` VARCHAR(800) CHAR SET latin1 NULL,
`owners` VARCHAR(300) NULL,
`tags` VARCHAR(300) NULL,
`urn` VARCHAR(300) CHAR SET latin1 NOT NULL,
`metric_url` VARCHAR(300) CHAR SET latin1 NULL,
`wiki_url` VARCHAR(300) CHAR SET latin1 NULL,
`scm_url` VARCHAR(300) CHAR SET latin1 NULL,
`wh_etl_exec_id` BIGINT COMMENT 'wherehows etl execution id that modified this record',
PRIMARY KEY(metric_id),
KEY `idx_dict_business_metric__urn` (`urn`) USING BTREE,
UNIQUE KEY `uq_dataset_urn` (`urn`),
KEY `idx_dict_business_metric__ref_id` (`metric_ref_id`) USING BTREE,
FULLTEXT KEY `fti_dict_business_metric_all` (`metric_name`, `metric_description`, `metric_category`, `metric_group`, `dashboard_name`)
)
ENGINE = InnoDB
AUTO_INCREMENT = 0
;
CREATE TABLE `stg_dict_business_metric` (
`metric_name` varchar(200) NOT NULL,
`metric_description` varchar(500) DEFAULT NULL,
`dashboard_name` varchar(100) DEFAULT NULL COMMENT 'Hierarchy Level 1',
`metric_group` varchar(100) DEFAULT NULL COMMENT 'Hierarchy Level 2',
`metric_category` varchar(100) DEFAULT NULL COMMENT 'Hierarchy Level 3',
`metric_sub_category` varchar(100) DEFAULT NULL COMMENT 'Additional Classification, such as Product, Line of Business',
`metric_level` varchar(50) DEFAULT NULL COMMENT 'CORE, DEPARTMENT, TEAM, OPERATION, STRATEGIC, TIER1, TIER2',
`metric_source_type` varchar(50) DEFAULT NULL COMMENT 'Table, Cube, File, Web Service',
`metric_source` varchar(300) CHARACTER SET latin1 DEFAULT NULL COMMENT 'Table Name, Cube Name, URL',
`metric_source_dataset_id` int(11) DEFAULT NULL COMMENT 'If metric_source can be matched in dict_dataset',
`metric_ref_id_type` varchar(50) CHARACTER SET latin1 DEFAULT NULL COMMENT 'DWH, ABTEST, FINANCE, SEGMENT, SALESAPP',
`metric_ref_id` varchar(100) CHARACTER SET latin1 DEFAULT NULL COMMENT 'ID in the reference system',
`metric_type` varchar(100) DEFAULT NULL COMMENT 'NUMBER, BOOLEAN, LIST',
`metric_additive_type` varchar(100) DEFAULT NULL COMMENT 'FULL, SEMI, NONE',
`metric_grain` varchar(100) DEFAULT NULL COMMENT 'DAILY, WEEKLY, UNIQUE, ROLLING 7D, ROLLING 30D',
`metric_display_factor` decimal(15,4) DEFAULT NULL,
`metric_display_factor_sym` varchar(20) DEFAULT NULL COMMENT '%, (K), (M), (B), (GB), (TB), (PB)',
`metric_good_direction` varchar(20) DEFAULT NULL COMMENT 'UP, DOWN, ZERO, FLAT',
`metric_formula` text COMMENT 'Expression, Code Snippet or Calculation Logic',
`dimensions` varchar(800) CHARACTER SET latin1 DEFAULT NULL,
`owners` varchar(300) DEFAULT NULL,
`tags` varchar(500) DEFAULT NULL,
`urn` varchar(300) CHARACTER SET latin1 NOT NULL,
`metric_url` varchar(300) CHARACTER SET latin1 DEFAULT NULL,
`wiki_url` varchar(300) CHARACTER SET latin1 DEFAULT NULL,
`scm_url` varchar(300) CHARACTER SET latin1 DEFAULT NULL,
`wh_etl_exec_id` BIGINT COMMENT 'wherehows etl execution id that modified this record',
PRIMARY KEY(`urn`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8
;