yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-09-18 13:43:19 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(ingest/looker): correct looker/lookml capability reports (#7683)

Browse Source
This commit is contained in:
Harshal Sheth 2023-03-27 05:06:12 +05:30 committed by GitHub
parent 419bee8614
commit 6d04511949
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 14 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
metadata-ingestion/src/datahub/ingestion/source/looker/looker_source.py
View File

@ -24,7 +24,7 @@ from pydantic import Field, validator
import datahub.emitter.mce_builder as builder import datahub.emitter.mce_builder as builder
from datahub.configuration.common import AllowDenyPattern, ConfigurationError from datahub.configuration.common import AllowDenyPattern, ConfigurationError
from datahub.configuration.source_common import DatasetSourceConfigMixin from datahub.configuration.source_common import DatasetSourceConfigMixin, EnvConfigMixin
from datahub.configuration.validate_field_removal import pydantic_removed_field from datahub.configuration.validate_field_removal import pydantic_removed_field
from datahub.emitter.mcp import MetadataChangeProposalWrapper from datahub.emitter.mcp import MetadataChangeProposalWrapper
from datahub.emitter.mcp_builder import create_embed_mcp from datahub.emitter.mcp_builder import create_embed_mcp
@ -106,7 +106,7 @@ class LookerDashboardSourceConfig(
LookerAPIConfig, LookerAPIConfig,
LookerCommonConfig, LookerCommonConfig,
StatefulIngestionConfigBase, StatefulIngestionConfigBase,
DatasetSourceConfigMixin, EnvConfigMixin,
): ):
_removed_github_info = pydantic_removed_field("github_info") _removed_github_info = pydantic_removed_field("github_info")
@ -168,23 +168,18 @@ class LookerDashboardSourceConfig(
) -> Optional[str]: ) -> Optional[str]:
return v or values.get("base_url") return v or values.get("base_url")
@validator("platform_instance")
def platform_instance_not_supported(cls, v: Optional[str]) -> Optional[str]:
if v is not None:
raise ConfigurationError("Looker Source doesn't support platform instances")
return v
@platform_name("Looker") @platform_name("Looker")
@support_status(SupportStatus.CERTIFIED) @support_status(SupportStatus.CERTIFIED)
@config_class(LookerDashboardSourceConfig) @config_class(LookerDashboardSourceConfig)
@capability(SourceCapability.DESCRIPTIONS, "Enabled by default") @capability(SourceCapability.DESCRIPTIONS, "Enabled by default")
@capability(SourceCapability.PLATFORM_INSTANCE, "Enabled by default") @capability(SourceCapability.PLATFORM_INSTANCE, "Not supported", supported=False)
@capability( @capability(
SourceCapability.OWNERSHIP, "Enabled by default, configured using `extract_owners`" SourceCapability.OWNERSHIP, "Enabled by default, configured using `extract_owners`"
) )
@capability( @capability(
SourceCapability.USAGE_STATS, "Can be enabled using `extract_usage_history`" SourceCapability.USAGE_STATS,
"Enabled by default, configured using `extract_usage_history`",
) )
class LookerDashboardSource(TestableSource, StatefulIngestionSourceBase): class LookerDashboardSource(TestableSource, StatefulIngestionSourceBase):
""" """

18
metadata-ingestion/src/datahub/ingestion/source/looker/lookml_source.py
View File

@ -27,11 +27,13 @@ from datahub.emitter.mcp import MetadataChangeProposalWrapper
from datahub.ingestion.api.common import PipelineContext from datahub.ingestion.api.common import PipelineContext
from datahub.ingestion.api.decorators import ( from datahub.ingestion.api.decorators import (
SupportStatus, SupportStatus,
capability,
config_class, config_class,
platform_name, platform_name,
support_status, support_status,
) )
from datahub.ingestion.api.registry import import_path from datahub.ingestion.api.registry import import_path
from datahub.ingestion.api.source import SourceCapability
from datahub.ingestion.api.workunit import MetadataWorkUnit from datahub.ingestion.api.workunit import MetadataWorkUnit
from datahub.ingestion.source.common.subtypes import DatasetSubTypes from datahub.ingestion.source.common.subtypes import DatasetSubTypes
from datahub.ingestion.source.git.git_import import GitClone from datahub.ingestion.source.git.git_import import GitClone
@ -175,7 +177,7 @@ class LookerConnectionDefinition(ConfigModel):
class LookMLSourceConfig( class LookMLSourceConfig(
LookerCommonConfig, StatefulIngestionConfigBase, DatasetSourceConfigMixin LookerCommonConfig, StatefulIngestionConfigBase, EnvConfigMixin
): ):
git_info: Optional[GitInfo] = Field( git_info: Optional[GitInfo] = Field(
None, None,
@ -237,14 +239,6 @@ class LookMLSourceConfig(
default=None, description="" default=None, description=""
) )
@validator("platform_instance")
def platform_instance_not_supported(cls, v: Optional[str]) -> Optional[str]:
if v is not None:
raise ConfigurationError(
"LookML Source doesn't support platform instance at the top level. However connection-specific platform instances are supported for generating lineage edges. Read the documentation to find out more."
)
return v
@validator("connection_to_platform_map", pre=True) @validator("connection_to_platform_map", pre=True)
def convert_string_to_connection_def(cls, conn_map): def convert_string_to_connection_def(cls, conn_map):
# Previous version of config supported strings in connection map. This upconverts strings to ConnectionMap # Previous version of config supported strings in connection map. This upconverts strings to ConnectionMap
@ -1094,6 +1088,12 @@ class LookerManifest:
@platform_name("Looker") @platform_name("Looker")
@config_class(LookMLSourceConfig) @config_class(LookMLSourceConfig)
@support_status(SupportStatus.CERTIFIED) @support_status(SupportStatus.CERTIFIED)
@capability(SourceCapability.PLATFORM_INSTANCE, "Not supported", supported=False)
@capability(SourceCapability.LINEAGE_COARSE, "Supported by default")
@capability(
SourceCapability.LINEAGE_FINE,
"Enabled by default, configured using `extract_column_level_lineage`",
)
class LookMLSource(StatefulIngestionSourceBase): class LookMLSource(StatefulIngestionSourceBase):
""" """
This plugin extracts the following: This plugin extracts the following: