yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-10-29 17:49:14 +00:00
Code Issues Packages Projects Releases Wiki Activity

Looker - Update GitHub creds for UI form (#11019)

Browse Source 
* Update GitHub creds for UI form

* Add filters
This commit is contained in:
Pere Miquel Brull 2023-04-13 06:05:02 +02:00 committed by GitHub
parent 08ecf69978
commit dcbb77f46c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 109 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

157
ingestion/src/metadata/ingestion/source/dashboard/looker/metadata.py
View File

@ -61,6 +61,9 @@ from metadata.generated.schema.entity.services.dashboardService import (
from metadata.generated.schema.metadataIngestion.workflow import ( from metadata.generated.schema.metadataIngestion.workflow import (
Source as WorkflowSource, Source as WorkflowSource,
) )
from metadata.generated.schema.security.credentials.githubCredentials import (
GitHubCredentials,
)
from metadata.generated.schema.type.entityLineage import EntitiesEdge from metadata.generated.schema.type.entityLineage import EntitiesEdge
from metadata.generated.schema.type.entityReference import EntityReference from metadata.generated.schema.type.entityReference import EntityReference
from metadata.generated.schema.type.usageRequest import UsageRequest from metadata.generated.schema.type.usageRequest import UsageRequest
@ -151,30 +154,51 @@ class LookerSource(DashboardServiceSource):
@property @property
def parser(self) -> Optional[LkmlParser]: def parser(self) -> Optional[LkmlParser]:
if not self._parser and self.service_connection.githubCredentials: if not self._parser and self.github_credentials:
self._parser = LkmlParser( self._parser = LkmlParser(reader=GitHubReader(self.github_credentials))
reader=GitHubReader(self.service_connection.githubCredentials)
)
return self._parser return self._parser
@property
def github_credentials(self) -> Optional[GitHubCredentials]:
"""
Check if the credentials are informed and return them.
We either get GitHubCredentials or `NoGitHubCredentials`
"""
if self.service_connection.githubCredentials and isinstance(
self.service_connection.githubCredentials, GitHubCredentials
):
return self.service_connection.githubCredentials
return None
def list_datamodels(self) -> Iterable[LookmlModelExplore]: def list_datamodels(self) -> Iterable[LookmlModelExplore]:
""" """
Fetch explores with the SDK Fetch explores with the SDK
""" """
# First, pick up all the LookML Models if self.source_config.includeDataModels:
all_lookml_models: Sequence[LookmlModel] = self.client.all_lookml_models() # First, pick up all the LookML Models
all_lookml_models: Sequence[LookmlModel] = self.client.all_lookml_models()
# Then, fetch the explores for each of them # Then, fetch the explores for each of them
for lookml_model in all_lookml_models: for lookml_model in all_lookml_models:
# Each LookML model have a list of explores we'll be ingesting # Each LookML model have a list of explores we'll be ingesting
for explore_nav in ( for explore_nav in (
cast(Sequence[LookmlModelNavExplore], lookml_model.explores) or [] cast(Sequence[LookmlModelNavExplore], lookml_model.explores) or []
): ):
explore = self.client.lookml_model_explore( if filter_by_datamodel(
lookml_model_name=lookml_model.name, explore_name=explore_nav.name self.source_config.dataModelFilterPattern, lookml_model.name
) ):
yield explore self.status.filter(
lookml_model.name, "Data model (Explore) filtered out."
)
continue
explore = self.client.lookml_model_explore(
lookml_model_name=lookml_model.name,
explore_name=explore_nav.name,
)
yield explore
def yield_bulk_datamodel( def yield_bulk_datamodel(
self, model: LookmlModelExplore self, model: LookmlModelExplore
@ -183,55 +207,63 @@ class LookerSource(DashboardServiceSource):
Get the Explore and View information and prepare Get the Explore and View information and prepare
the model creation request the model creation request
""" """
if self.source_config.includeDataModels: try:
try: datamodel_name = build_datamodel_name(model.model_name, model.name)
datamodel_name = build_datamodel_name(model.model_name, model.name) if filter_by_datamodel(
if filter_by_datamodel( self.source_config.dataModelFilterPattern, datamodel_name
self.source_config.dataModelFilterPattern, datamodel_name ):
): self.status.filter(datamodel_name, "Data model filtered out.")
self.status.filter(datamodel_name, "Data model filtered out.") else:
else: explore_datamodel = CreateDashboardDataModelRequest(
explore_datamodel = CreateDashboardDataModelRequest( name=datamodel_name,
name=datamodel_name, displayName=model.name,
displayName=model.name, description=model.description,
description=model.description, service=self.context.dashboard_service.fullyQualifiedName.__root__,
service=self.context.dashboard_service.fullyQualifiedName.__root__, dataModelType=DataModelType.LookMlExplore.value,
dataModelType=DataModelType.LookMlExplore.value, serviceType=DashboardServiceType.Looker.value,
serviceType=DashboardServiceType.Looker.value, columns=get_columns_from_model(model),
columns=get_columns_from_model(model), sql=self._get_explore_sql(model),
sql=self._get_explore_sql(model), )
) yield explore_datamodel
yield explore_datamodel self.status.scanned(f"Data Model Scanned: {model.name}")
self.status.scanned(f"Data Model Scanned: {model.name}")
# Maybe use the project_name as key too? # Maybe use the project_name as key too?
# Save the explores for when we create the lineage with the dashboards and views # Save the explores for when we create the lineage with the dashboards and views
self._explores_cache[ self._explores_cache[
explore_datamodel.name.__root__ explore_datamodel.name.__root__
] = self.context.dataModel # This is the newly created explore ] = self.context.dataModel # This is the newly created explore
# We can get VIEWs from the JOINs to know the dependencies # We can get VIEWs from the JOINs to know the dependencies
# We will only try and fetch if we have the credentials # We will only try and fetch if we have the credentials
if self.service_connection.githubCredentials: if self.github_credentials:
for view in model.joins: for view in model.joins:
yield from self._process_view(
view_name=ViewName(view.name), explore=model if filter_by_datamodel(
self.source_config.dataModelFilterPattern, view.name
):
self.status.filter(
view.name, "Data model (View) filtered out."
) )
continue
except ValidationError as err: yield from self._process_view(
error = f"Validation error yielding Data Model [{model.name}]: {err}" view_name=ViewName(view.name), explore=model
logger.debug(traceback.format_exc()) )
logger.error(error)
self.status.failed( except ValidationError as err:
name=model.name, error=error, stack_trace=traceback.format_exc() error = f"Validation error yielding Data Model [{model.name}]: {err}"
) logger.debug(traceback.format_exc())
except Exception as err: logger.error(error)
error = f"Wild error yielding Data Model [{model.name}]: {err}" self.status.failed(
logger.debug(traceback.format_exc()) name=model.name, error=error, stack_trace=traceback.format_exc()
logger.error(error) )
self.status.failed( except Exception as err:
name=model.name, error=error, stack_trace=traceback.format_exc() error = f"Wild error yielding Data Model [{model.name}]: {err}"
) logger.debug(traceback.format_exc())
logger.error(error)
self.status.failed(
name=model.name, error=error, stack_trace=traceback.format_exc()
)
def _get_explore_sql(self, explore: LookmlModelExplore) -> Optional[str]: def _get_explore_sql(self, explore: LookmlModelExplore) -> Optional[str]:
""" """
@ -239,7 +271,7 @@ class LookerSource(DashboardServiceSource):
file definition and add it here file definition and add it here
""" """
# Only look to parse if creds are in # Only look to parse if creds are in
if self.service_connection.githubCredentials: if self.github_credentials:
try: try:
# This will only parse if the file has not been parsed yet # This will only parse if the file has not been parsed yet
self.parser.parse_file(Includes(explore.source_file)) self.parser.parse_file(Includes(explore.source_file))
@ -379,7 +411,6 @@ class LookerSource(DashboardServiceSource):
""" """
Method to Get Dashboard Entity Method to Get Dashboard Entity
""" """
dashboard_request = CreateDashboardRequest( dashboard_request = CreateDashboardRequest(
name=clean_dashboard_name(dashboard_details.id), name=clean_dashboard_name(dashboard_details.id),
displayName=dashboard_details.title, displayName=dashboard_details.title,

2
ingestion/src/metadata/utils/filters.py
View File

@ -229,7 +229,7 @@ def filter_by_datamodel(
datamodel_filter_pattern: Optional[FilterPattern], datamodel_name: str datamodel_filter_pattern: Optional[FilterPattern], datamodel_name: str
) -> bool: ) -> bool:
""" """
Return True if the chart needs to be filtered, False otherwise Return True if the models needs to be filtered, False otherwise
Include takes precedence over exclude Include takes precedence over exclude

15
openmetadata-spec/src/main/resources/json/schema/entity/services/connections/dashboard/lookerConnection.json
View File

@ -11,6 +11,12 @@
"type": "string", "type": "string",
"enum": ["Looker"], "enum": ["Looker"],
"default": "Looker" "default": "Looker"
},
"noGitHubCredentials": {
"title": "No GitHub Credentials",
"description": "Do not set any credentials. Note that credentials are required to extract .lkml views and their lineage.",
"type": "object",
"additionalProperties": false
} }
}, },
"properties": { "properties": {
@ -41,7 +47,14 @@
"githubCredentials": { "githubCredentials": {
"title": "GitHub Credentials", "title": "GitHub Credentials",
"description": "Credentials to extract the .lkml files from a repository. This is required to get all the lineage and definitions.", "description": "Credentials to extract the .lkml files from a repository. This is required to get all the lineage and definitions.",
"$ref": "../../../../security/credentials/githubCredentials.json" "oneOf": [
{
"$ref": "#/definitions/noGitHubCredentials"
},
{
"$ref": "../../../../security/credentials/githubCredentials.json"
}
]
}, },
"supportsMetadataExtraction": { "supportsMetadataExtraction": {
"title": "Supports Metadata Extraction", "title": "Supports Metadata Extraction",