yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-11-03 03:59:12 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add mappings for table ES index ingestion (#5635)

Browse Source
This commit is contained in:
Pere Miquel Brull 2022-06-25 11:41:51 +02:00 committed by GitHub
parent e8c5ddc342
commit b950f0fd06
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 153 additions and 135 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
ingestion/src/metadata/ingestion/sink/elasticsearch.py
View File

@ -603,6 +603,21 @@ class ElasticsearchSink(Sink[Entity]):
for ml_model_hyper_parameter in ml_model.mlHyperParameters:
ml_model_hyper_parameters.append(ml_model_hyper_parameter.name)
service_entity = ESEntityReference(
id=str(ml_model.service.id.__root__),
name=ml_model.service.name,
displayName=ml_model.service.displayName
if ml_model.service.displayName
else "",
description=ml_model.service.description.__root__
if ml_model.service.description
else "",
type=ml_model.service.type,
fullyQualifiedName=ml_model.service.fullyQualifiedName,
deleted=ml_model.service.deleted,
href=ml_model.service.href.__root__,
)
ml_model_doc = MlModelESDocument(
ml_model_id=str(ml_model.id.__root__),
deleted=ml_model.deleted,
@ -618,6 +633,7 @@ class ElasticsearchSink(Sink[Entity]):
fqdn=ml_model_fqn,
owner=ml_model.owner,
followers=ml_model_followers,
service=service_entity,
)
return ml_model_doc

272
ingestion/src/metadata/ingestion/sink/elasticsearch_constants.py
View File

@ -27,162 +27,164 @@ TABLE_ELASTICSEARCH_INDEX_MAPPING = textwrap.dedent(
}
}
},
"properties": {
"name": {
"type": "text"
},
"display_name": {
"type": "text"
},
"owner": {
"mappings": {
"properties": {
"id": {
"type": "keyword",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 128
}
}
},
"type": {
"type": "text"
},
"name": {
"type": "keyword",
"fields": {
"keyword": {
"type": "text"
},
"display_name": {
"type": "text"
},
"owner": {
"properties": {
"id": {
"type": "keyword",
"ignore_above": 256
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 128
}
}
},
"type": {
"type": "text"
},
"name": {
"type": "keyword",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"fullyQualifiedName": {
"type": "text"
},
"description": {
"type": "text"
},
"deleted": {
"type": "boolean"
},
"href": {
"type": "text"
}
}
},
"fullyQualifiedName": {
"type": "text"
},
"description": {
"type": "text"
"fqdn": {
"type": "keyword",
"normalizer": "lowercase_normalizer"
},
"deleted": {
"type": "boolean"
},
"href": {
"type": "text"
}
}
},
"fqdn": {
"type": "keyword",
"normalizer": "lowercase_normalizer"
},
"deleted": {
"type": "boolean"
},
"followers": {
"type": "keyword"
},
"last_updated_timestamp": {
"type": "date",
"format": "epoch_second"
},
"description": {
"type": "text"
},
"tier": {
"type": "keyword"
},
"column_names": {
"type": "text"
},
"column_descriptions": {
"type": "text"
},
"tags": {
"type": "keyword"
},
"service": {
"properties": {
"id": {
"type": "keyword",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"type": {
"followers": {
"type": "keyword"
},
"name": {
"type": "keyword",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"fullyQualifiedName": {
"type": "keyword"
"last_updated_timestamp": {
"type": "date",
"format": "epoch_second"
},
"description": {
"type": "text"
},
"deleted": {
"type": "boolean"
"tier": {
"type": "keyword"
},
"href": {
"column_names": {
"type": "text"
},
"column_descriptions": {
"type": "text"
},
"tags": {
"type": "keyword"
},
"service": {
"properties": {
"id": {
"type": "keyword",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"type": {
"type": "keyword"
},
"name": {
"type": "keyword",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"fullyQualifiedName": {
"type": "keyword"
},
"description": {
"type": "text"
},
"deleted": {
"type": "boolean"
},
"href": {
"type": "text"
}
}
},
"service_type": {
"type": "keyword"
},
"entity_type": {
"type": "keyword"
},
"database": {
"type": "keyword"
},
"database_schema": {
"type": "keyword"
},
"suggest": {
"type": "completion"
},
"column_suggest": {
"type": "completion"
},
"schema_suggest": {
"type": "completion"
},
"database_suggest": {
"type": "completion"
},
"service_suggest": {
"type": "completion"
},
"monthly_stats": {
"type": "long"
},
"monthly_percentile_rank": {
"type": "long"
},
"weekly_stats": {
"type": "long"
},
"weekly_percentile_rank": {
"type": "long"
},
"daily_percentile_rank": {
"type": "long"
},
"daily_stats": {
"type": "long"
}
}
},
"service_type": {
"type": "keyword"
},
"entity_type": {
"type": "keyword"
},
"database": {
"type": "keyword"
},
"database_schema": {
"type": "keyword"
},
"suggest": {
"type": "completion"
},
"column_suggest": {
"type": "completion"
},
"schema_suggest": {
"type": "completion"
},
"database_suggest": {
"type": "completion"
},
"service_suggest": {
"type": "completion"
},
"monthly_stats": {
"type": "long"
},
"monthly_percentile_rank": {
"type": "long"
},
"weekly_stats": {
"type": "long"
},
"weekly_percentile_rank": {
"type": "long"
},
"daily_percentile_rank": {
"type": "long"
},
"daily_stats": {
"type": "long"
}
}
}
"""
)