Add Owner for DBT (#6872)

* Added owner field in DataModel

* Added user in backend

* review comments addressed

Co-authored-by: Onkar Ravgan <onkarravgan@Onkars-MacBook-Pro.local>
This commit is contained in:
Onkar Ravgan 2022-08-26 16:22:27 +05:30 committed by GitHub
parent 87f8bcc4fa
commit 13d433b0d7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 55 additions and 1 deletions

View File

@ -625,6 +625,12 @@ public class TableRepository extends EntityRepository<Table> {
if (nullOrEmpty(table.getDescription())) { if (nullOrEmpty(table.getDescription())) {
table.setDescription(dataModel.getDescription()); table.setDescription(dataModel.getDescription());
} }
// Carry forward the table owner from the model to table entity, if empty
if (table.getOwner() == null) {
storeOwner(table, dataModel.getOwner());
}
// Carry forward the column description from the model to table columns, if empty // Carry forward the column description from the model to table columns, if empty
for (Column modelColumn : listOrEmpty(dataModel.getColumns())) { for (Column modelColumn : listOrEmpty(dataModel.getColumns())) {
Column stored = Column stored =
@ -640,7 +646,9 @@ public class TableRepository extends EntityRepository<Table> {
} }
} }
dao.update(table.getId(), JsonUtils.pojoToJson(table)); dao.update(table.getId(), JsonUtils.pojoToJson(table));
setFields(table, Fields.EMPTY_FIELDS);
setFields(table, new Fields(List.of(FIELD_OWNER), FIELD_OWNER));
return table; return table;
} }

View File

@ -628,6 +628,11 @@
"type": "string" "type": "string"
} }
}, },
"owner": {
"description": "Owner of this Model.",
"$ref": "../../type/entityReference.json",
"default": null
},
"columns": { "columns": {
"description": "Columns from the schema defined during modeling. In case of DBT, the metadata here comes from `schema.yaml`.", "description": "Columns from the schema defined during modeling. In case of DBT, the metadata here comes from `schema.yaml`.",
"type": "array", "type": "array",

View File

@ -21,6 +21,7 @@ from metadata.generated.schema.entity.data.table import (
ModelType, ModelType,
Table, Table,
) )
from metadata.generated.schema.entity.teams.user import User
from metadata.generated.schema.type.entityLineage import EntitiesEdge from metadata.generated.schema.type.entityLineage import EntitiesEdge
from metadata.generated.schema.type.entityReference import EntityReference from metadata.generated.schema.type.entityReference import EntityReference
from metadata.ingestion.ometa.ometa_api import OpenMetadata from metadata.ingestion.ometa.ometa_api import OpenMetadata
@ -77,6 +78,14 @@ class DBTMixin:
schema = mnode["schema"] if mnode["schema"] else "default" schema = mnode["schema"] if mnode["schema"] else "default"
raw_sql = mnode.get("raw_sql", "") raw_sql = mnode.get("raw_sql", "")
description = mnode.get("description") description = mnode.get("description")
dbt_user_name = cnode["metadata"].get("owner")
user_name = f"*{dbt_user_name}*"
user_fqn = fqn.build(
self.metadata, entity_type=User, user_name=user_name
)
owner = self.metadata.get_entity_reference(
entity=User, fqn=user_fqn
)
model = DataModel( model = DataModel(
modelType=ModelType.DBT, modelType=ModelType.DBT,
description=description if description else None, description=description if description else None,
@ -85,6 +94,7 @@ class DBTMixin:
sql=mnode.get("compiled_sql", raw_sql), sql=mnode.get("compiled_sql", raw_sql),
columns=columns, columns=columns,
upstream=upstream_nodes, upstream=upstream_nodes,
owner=owner,
) )
model_fqn = fqn.build( model_fqn = fqn.build(
self.metadata, self.metadata,

View File

@ -33,6 +33,7 @@ from metadata.generated.schema.entity.data.location import Location
from metadata.generated.schema.entity.data.pipeline import Pipeline from metadata.generated.schema.entity.data.pipeline import Pipeline
from metadata.generated.schema.entity.data.table import Column, DataModel, Table from metadata.generated.schema.entity.data.table import Column, DataModel, Table
from metadata.generated.schema.entity.tags.tagCategory import Tag from metadata.generated.schema.entity.tags.tagCategory import Tag
from metadata.generated.schema.entity.teams.user import User
from metadata.ingestion.ometa.ometa_api import OpenMetadata from metadata.ingestion.ometa.ometa_api import OpenMetadata
from metadata.utils.dispatch import class_register from metadata.utils.dispatch import class_register
from metadata.utils.elasticsearch import get_entity_from_es_result from metadata.utils.elasticsearch import get_entity_from_es_result
@ -280,6 +281,36 @@ def _(
return _build(service_name, database_name, schema_name, table_name, column_name) return _build(service_name, database_name, schema_name, table_name, column_name)
@fqn_build_registry.add(User)
def _(
metadata: OpenMetadata,
*,
user_name: str,
fetch_multiple_entities: bool = False,
) -> Union[Optional[str], Optional[List[str]]]:
"""
Building logic for User
:param metadata: OMeta client
:param user_name: User name
:return:
"""
fqn_search_string = _build(user_name)
es_result = metadata.es_search_from_fqn(
entity_type=User,
fqn_search_string=fqn_search_string,
)
entity: Optional[Union[User, List[User]]] = get_entity_from_es_result(
entity_list=es_result, fetch_multiple_entities=fetch_multiple_entities
)
if not entity:
return None
if fetch_multiple_entities:
return [str(user.fullyQualifiedName.__root__) for user in entity]
return str(entity.fullyQualifiedName.__root__)
def split_table_name(table_name: str) -> Dict[str, Optional[str]]: def split_table_name(table_name: str) -> Dict[str, Optional[str]]:
""" """
Given a table name, try to extract database, schema and Given a table name, try to extract database, schema and