yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-09-26 17:34:41 +00:00
Code Issues Packages Projects Releases Wiki Activity

Ingestion: add compute.percentile after publishing usage

Browse Source
This commit is contained in:
Suresh Srinivas 2021-08-14 14:24:01 -07:00
parent 32d32fe755
commit 92c33cf63c
3 changed files with 15 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
ingestion/src/metadata/ingestion/bulksink/elasticsearch.py
View File

@ -124,8 +124,11 @@ class ElasticsearchBulkSink(BulkSink):
column_names=column_names, column_names=column_names,
column_descriptions=column_descriptions, column_descriptions=column_descriptions,
monthly_stats=table.usageSummary.monthlyStats.count, monthly_stats=table.usageSummary.monthlyStats.count,
monthly_percentile_rank=table.usageSummary.monthlyStats.percentileRank,
weekly_stats=table.usageSummary.weeklyStats.count, weekly_stats=table.usageSummary.weeklyStats.count,
weekly_percentile_rank=table.usageSummary.weeklyStats.percentileRank,
daily_stats=table.usageSummary.dailyStats.count, daily_stats=table.usageSummary.dailyStats.count,
daily_percentile_rank=table.usageSummary.dailyStats.percentileRank,
tier=tier, tier=tier,
tags=list(tags), tags=list(tags),
fqdn=fqdn, fqdn=fqdn,

9
ingestion/src/metadata/ingestion/bulksink/elasticsearch_constants.py
View File

@ -79,9 +79,18 @@ TABLE_ELASTICSEARCH_INDEX_MAPPING = textwrap.dedent(
"monthly_stats":{ "monthly_stats":{
"type": "long" "type": "long"
}, },
"monthly_percentile_rank":{
"type": "long"
},
"weekly_stats":{ "weekly_stats":{
"type": "long" "type": "long"
}, },
"weekly_percentile_rank":{
"type": "long"
},
"daily_percentile_rank": {
"type": "long"
},
"daily_stats": { "daily_stats": {
"type": "long" "type": "long"
} }

3
ingestion/src/metadata/ingestion/models/table_metadata.py
View File

@ -179,8 +179,11 @@ class TableESDocument(BaseModel):
column_names: List[str] column_names: List[str]
column_descriptions: List[str] column_descriptions: List[str]
monthly_stats: int monthly_stats: int
monthly_percentile_rank: int
weekly_stats: int weekly_stats: int
weekly_percentile_rank: int
daily_stats: int daily_stats: int
daily_percentile_rank: int
tags: List[str] tags: List[str]
fqdn: str fqdn: str
tier: Optional[str] = None tier: Optional[str] = None