yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-11-08 15:30:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(ingest): fail gracefully when lookml used on old python versions (#2614)

Browse Source
This commit is contained in:
Harshal Sheth 2021-05-26 17:16:17 -07:00 committed by GitHub
parent 8d53924892
commit acef397ece
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 6 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
metadata-ingestion/setup.py
View File

@ -84,14 +84,12 @@ plugins: Dict[str, Set[str]] = {
"oracle": sql_common | {"cx_Oracle"}, "oracle": sql_common | {"cx_Oracle"},
"ldap": {"python-ldap>=2.4"}, "ldap": {"python-ldap>=2.4"},
"looker": {"looker-sdk==21.6.0"}, "looker": {"looker-sdk==21.6.0"},
"lookml": {"lkml>=1.1.0", "sql-metadata==1.12.0"},
"druid": sql_common | {"pydruid>=0.6.2"}, "druid": sql_common | {"pydruid>=0.6.2"},
"mongodb": {"pymongo>=3.11"}, "mongodb": {"pymongo>=3.11"},
"superset": {"requests"}, "superset": {"requests"},
"glue": {"boto3"}, "glue": {"boto3"},
} }
if is_py37_or_newer:
plugins["lookml"] = {"lkml>=1.1.0", "sql-metadata==1.12.0"}
base_dev_requirements = { base_dev_requirements = {
*base_requirements, *base_requirements,
@ -130,6 +128,7 @@ base_dev_requirements = {
} }
if is_py37_or_newer: if is_py37_or_newer:
# The lookml plugin only works on Python 3.7 or newer.
base_dev_requirements = base_dev_requirements.union( base_dev_requirements = base_dev_requirements.union(
{dependency for plugin in ["lookml"] for dependency in plugins[plugin]} {dependency for plugin in ["lookml"] for dependency in plugins[plugin]}
) )
@ -161,6 +160,7 @@ entry_points = {
"kafka-connect = datahub.ingestion.source.kafka_connect:KafkaConnectSource", "kafka-connect = datahub.ingestion.source.kafka_connect:KafkaConnectSource",
"ldap = datahub.ingestion.source.ldap:LDAPSource", "ldap = datahub.ingestion.source.ldap:LDAPSource",
"looker = datahub.ingestion.source.looker:LookerDashboardSource", "looker = datahub.ingestion.source.looker:LookerDashboardSource",
"lookml = datahub.ingestion.source.lookml:LookMLSource",
"mongodb = datahub.ingestion.source.mongodb:MongoDBSource", "mongodb = datahub.ingestion.source.mongodb:MongoDBSource",
"mssql = datahub.ingestion.source.mssql:SQLServerSource", "mssql = datahub.ingestion.source.mssql:SQLServerSource",
"mysql = datahub.ingestion.source.mysql:MySQLSource", "mysql = datahub.ingestion.source.mysql:MySQLSource",
@ -179,11 +179,6 @@ entry_points = {
"apache_airflow_provider": ["provider_info=datahub_provider:get_provider_info"], "apache_airflow_provider": ["provider_info=datahub_provider:get_provider_info"],
} }
if is_py37_or_newer:
entry_points["datahub.ingestion.source.plugins"].append(
"lookml = datahub.ingestion.source.lookml:LookMLSource"
)
setuptools.setup( setuptools.setup(
# Package metadata. # Package metadata.

1
metadata-ingestion/src/datahub/check/check_cli.py
View File

@ -40,6 +40,7 @@ def local_docker() -> None:
@check.command() @check.command()
@click.option( @click.option(
"--verbose", "--verbose",
"-v",
type=bool, type=bool,
is_flag=True, is_flag=True,
default=False, default=False,

4
metadata-ingestion/src/datahub/ingestion/source/lookml.py
View File

@ -10,10 +10,10 @@ from enum import Enum
from pathlib import Path from pathlib import Path
from typing import Dict, Iterable, List, Optional, Tuple from typing import Dict, Iterable, List, Optional, Tuple
if sys.version_info[1] >= 7: if sys.version_info >= (3, 7):
import lkml import lkml
else: else:
logging.warning("This plugin requres Python 3.7 or newer.") raise ModuleNotFoundError("The lookml plugin requires Python 3.7 or newer.")
from sql_metadata import get_query_tables from sql_metadata import get_query_tables
from datahub.configuration import ConfigModel from datahub.configuration import ConfigModel