yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-08-30 11:56:01 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fixes #11307 - Handle exceptions if LookML model is invalid (#11320)

Browse Source 
* Fix dynamo docs

* Handle data model fetch exceptions

* Format

* Add example for Private Key format
This commit is contained in:
Pere Miquel Brull 2023-04-27 11:42:16 +02:00 committed by GitHub
parent 7d3df6fd54
commit c53a3413fb
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 52 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

47
ingestion/src/metadata/ingestion/source/dashboard/looker/metadata.py
View File

@ -178,27 +178,47 @@ class LookerSource(DashboardServiceSource):
"""
if self.source_config.includeDataModels:
# First, pick up all the LookML Models
all_lookml_models: Sequence[LookmlModel] = self.client.all_lookml_models()
try:
all_lookml_models: Sequence[
LookmlModel
] = self.client.all_lookml_models()
yield from self.fetch_lookml_explores(all_lookml_models)
except Exception as err:
logger.debug(traceback.format_exc())
logger.error(f"Unexpected error fetching LookML models - {err}")
# Then, fetch the explores for each of them
for lookml_model in all_lookml_models:
# Each LookML model have a list of explores we'll be ingesting
for explore_nav in (
cast(Sequence[LookmlModelNavExplore], lookml_model.explores) or []
def fetch_lookml_explores(
self, all_lookml_models: Sequence[LookmlModel]
) -> Iterable[LookmlModelExplore]:
"""
Based on the LookML models, iterate over the explores
they contain and filter if needed
"""
# Then, fetch the explores for each of them
for lookml_model in all_lookml_models:
# Each LookML model have a list of explores we'll be ingesting
for explore_nav in (
cast(Sequence[LookmlModelNavExplore], lookml_model.explores) or []
):
if filter_by_datamodel(
self.source_config.dataModelFilterPattern, lookml_model.name
):
if filter_by_datamodel(
self.source_config.dataModelFilterPattern, lookml_model.name
):
self.status.filter(
lookml_model.name, "Data model (Explore) filtered out."
)
continue
self.status.filter(
lookml_model.name, "Data model (Explore) filtered out."
)
continue
try:
explore = self.client.lookml_model_explore(
lookml_model_name=lookml_model.name,
explore_name=explore_nav.name,
)
yield explore
except Exception as err:
logger.debug(traceback.format_exc())
logger.warning(
f"Error fetching LookML Explore [{explore_nav.name}] in model [{lookml_model.name}] - {err}"
)
def yield_bulk_datamodel(
self, model: LookmlModelExplore
@ -237,7 +257,6 @@ class LookerSource(DashboardServiceSource):
# We will only try and fetch if we have the credentials
if self.github_credentials:
for view in model.joins:
if filter_by_datamodel(
self.source_config.dataModelFilterPattern, view.name
):

19
openmetadata-ui/src/main/resources/ui/public/locales/en-US/Database/BigQuery.md
View File

@ -84,6 +84,25 @@ $$section
### Private Key $(id="privateKey")
This is the private key associated with the service account that is used to authenticate and authorize access to GCP. To fetch this key, look for the value associated with the `private_key` key in the service account file.
Make sure you are passing the key in a correct format. If your private key looks like this:
```
-----BEGIN ENCRYPTED PRIVATE KEY-----
MII..
MBQ...
CgU..
8Lt..
...
h+4=
-----END ENCRYPTED PRIVATE KEY-----
```
You will have to replace new lines with `\n` and the final private key that you need to pass should look like this:
```
-----BEGIN ENCRYPTED PRIVATE KEY-----\nMII..\nMBQ...\nCgU..\n8Lt..\n...\nh+4=\n-----END ENCRYPTED PRIVATE KEY-----\n
```
$$
$$section