yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-09-01 05:03:10 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fixes #18186: Quicksight Ingestion Error handled (#18218)

Browse Source
This commit is contained in:
Ayush Shah 2024-10-16 09:52:07 +05:30 committed by GitHub
parent 831cccf71e
commit 40bd3bd3fa
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 61 additions and 52 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

109
ingestion/src/metadata/ingestion/source/dashboard/quicksight/metadata.py
View File

@ -247,6 +247,7 @@ class QuicksightSource(DashboardServiceSource):
**data_source["RelationalTable"] **data_source["RelationalTable"]
) )
except (KeyError, ValidationError) as err: except (KeyError, ValidationError) as err:
data_source_resp = None
yield Either( yield Either(
left=StackTraceError( left=StackTraceError(
name="Lineage", name="Lineage",
@ -257,62 +258,70 @@ class QuicksightSource(DashboardServiceSource):
stackTrace=traceback.format_exc(), stackTrace=traceback.format_exc(),
) )
) )
if data_source_resp:
schema_name = data_source_resp.schema_name
table_name = data_source_resp.table_name
schema_name = data_source_resp.schema_name list_data_source_func = lambda kwargs: self.client.list_data_sources( # pylint: disable=unnecessary-lambda-assignment
table_name = data_source_resp.table_name **kwargs
)
list_data_source_func = lambda kwargs: self.client.list_data_sources( # pylint: disable=unnecessary-lambda-assignment data_source_summary_list = self._check_pagination(
**kwargs listing_method=list_data_source_func,
) entity_key="DataSources",
)
data_source_summary_list = self._check_pagination( data_source_ids = [
listing_method=list_data_source_func, data_source_arn["DataSourceId"]
entity_key="DataSources", for data_source_arn in data_source_summary_list or []
) if data_source_arn["Arn"] in data_source_resp.datasource_arn
]
data_source_ids = [ for data_source_id in data_source_ids or []:
data_source_arn["DataSourceId"] data_source_resp = DescribeDataSourceResponse(
for data_source_arn in data_source_summary_list or [] **self.client.describe_data_source(
if data_source_arn["Arn"] in data_source_resp.datasource_arn AwsAccountId=self.aws_account_id,
] DataSourceId=data_source_id,
for data_source_id in data_source_ids or []:
data_source_resp = DescribeDataSourceResponse(
**self.client.describe_data_source(
AwsAccountId=self.aws_account_id,
DataSourceId=data_source_id,
)
).DataSource
if data_source_resp and data_source_resp.DataSourceParameters:
data_source_dict = data_source_resp.DataSourceParameters
for db in data_source_dict.keys() or []:
from_fqn = fqn.build(
self.metadata,
entity_type=Table,
service_name=db_service_name,
database_name=data_source_dict[db].get("Database"),
schema_name=schema_name,
table_name=table_name,
skip_es_search=True,
) )
from_entity = self.metadata.get_by_name( ).DataSource
entity=Table, if (
fqn=from_fqn, data_source_resp
) and data_source_resp.DataSourceParameters
to_fqn = fqn.build( ):
self.metadata, data_source_dict = data_source_resp.DataSourceParameters
entity_type=Dashboard, for db in data_source_dict.keys() or []:
service_name=self.config.serviceName, from_fqn = fqn.build(
dashboard_name=dashboard_details.DashboardId, self.metadata,
) entity_type=Table,
to_entity = self.metadata.get_by_name( service_name=db_service_name,
entity=Dashboard, database_name=data_source_dict[db].get(
fqn=to_fqn, "Database"
) ),
if from_entity is not None and to_entity is not None: schema_name=schema_name,
yield self._get_add_lineage_request( table_name=table_name,
to_entity=to_entity, from_entity=from_entity skip_es_search=True,
) )
from_entity = self.metadata.get_by_name(
entity=Table,
fqn=from_fqn,
)
to_fqn = fqn.build(
self.metadata,
entity_type=Dashboard,
service_name=self.config.serviceName,
dashboard_name=dashboard_details.DashboardId,
)
to_entity = self.metadata.get_by_name(
entity=Dashboard,
fqn=to_fqn,
)
if (
from_entity is not None
and to_entity is not None
):
yield self._get_add_lineage_request(
to_entity=to_entity, from_entity=from_entity
)
except Exception as exc: # pylint: disable=broad-except except Exception as exc: # pylint: disable=broad-except
yield Either( yield Either(
left=StackTraceError( left=StackTraceError(

4
ingestion/src/metadata/ingestion/source/dashboard/quicksight/models.py
View File

@ -50,12 +50,12 @@ class DashboardResp(BaseModel):
RequestId: Optional[str] = None RequestId: Optional[str] = None
class DataSource(BaseModel): class DataSourceModel(BaseModel):
DataSourceId: str DataSourceId: str
DataSourceParameters: Optional[dict] = None DataSourceParameters: Optional[dict] = None
class DescribeDataSourceResponse(BaseModel): class DescribeDataSourceResponse(BaseModel):
DataSource: Optional[DataSource] = None DataSource: Optional[DataSourceModel] = None
RequestId: Optional[str] = None RequestId: Optional[str] = None
Status: Optional[int] = None Status: Optional[int] = None