yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-12-25 00:48:45 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(ingestion/looker): Add sqlglot dependency and remove unused sqlparser (#10874)

Browse Source
This commit is contained in:
sid-acryl 2024-07-10 01:21:08 +05:30 committed by GitHub
parent 3e86192b29
commit 609847fa59
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 3 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
metadata-ingestion/setup.py
View File

@ -163,8 +163,7 @@ looker_common = {
# LookML files with spaces between an item and the following comma.
# See https://github.com/joshtemple/lkml/issues/73.
"lkml>=1.3.4",
"sql-metadata==2.2.2",
*sqllineage_lib,
*sqlglot_lib,
"GitPython>2",
"python-liquid",
}
@ -372,7 +371,7 @@ plugins: Dict[str, Set[str]] = {
"kafka-connect": sql_common | {"requests", "JPype1"},
"ldap": {"python-ldap>=2.4"},
"looker": looker_common,
"lookml": looker_common | sqlglot_lib,
"lookml": looker_common,
"metabase": {"requests"} | sqlglot_lib,
"mlflow": {
"mlflow-skinny>=2.3.0",

13
metadata-ingestion/src/datahub/ingestion/source/looker/lookml_source.py
View File

@ -4,7 +4,7 @@ import tempfile
from collections import OrderedDict
from dataclasses import dataclass
from datetime import datetime, timezone
from typing import Dict, Iterable, List, Optional, Set, Tuple, Type
from typing import Dict, Iterable, List, Optional, Set, Tuple
import lkml
import lkml.simple
@ -22,7 +22,6 @@ from datahub.ingestion.api.decorators import (
platform_name,
support_status,
)
from datahub.ingestion.api.registry import import_path
from datahub.ingestion.api.source import MetadataWorkUnitProcessor, SourceCapability
from datahub.ingestion.api.workunit import MetadataWorkUnit
from datahub.ingestion.source.common.subtypes import (
@ -96,7 +95,6 @@ from datahub.metadata.schema_classes import (
SubTypesClass,
)
from datahub.sql_parsing.sqlglot_lineage import ColumnRef
from datahub.utilities.sql_parser import SQLParser
logger = logging.getLogger(__name__)
@ -111,15 +109,6 @@ class LookerView:
raw_file_content: str
view_details: Optional[ViewProperties] = None
@classmethod
def _import_sql_parser_cls(cls, sql_parser_path: str) -> Type[SQLParser]:
assert "." in sql_parser_path, "sql_parser-path must contain a ."
parser_cls = import_path(sql_parser_path)
if not issubclass(parser_cls, SQLParser):
raise ValueError(f"must be derived from {SQLParser}; got {parser_cls}")
return parser_cls
@classmethod
def determine_view_file_path(
cls, base_folder_path: str, absolute_file_path: str