yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-11-09 16:03:31 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(ingest): add option to specify source platform database in lookml ingestion (#2749)

Browse Source
This commit is contained in:
Remi 2021-06-23 17:16:20 -06:00 committed by GitHub
parent 22a2ed81e4
commit 91f5d4f59a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 16 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
metadata-ingestion/README.md
View File

@ -620,15 +620,15 @@ Extracts:
source: source:
type: "lookml" type: "lookml"
config: config:
base_folder: /path/to/model/files # Where the *.model.lkml and *.view.lkml files are stored. base_folder: /path/to/model/files # where the *.model.lkml and *.view.lkml files are stored
connection_to_platform_map: # mapping between connection names in the model files to platform names. connection_to_platform_map: # mappings between connection names in the model files to platform names
my_snowflake_conn: snowflake connection_name: platform_name (or platform_name.database_name) # for ex. my_snowflake_conn: snowflake.my_database
platform_name: looker_views # Optional, default is "looker_views" platform_name: "looker" # optional, default is "looker"
actor: "urn:li:corpuser:etl" # Optional, "urn:li:corpuser:etl" actor: "urn:li:corpuser:etl" # optional, default is "urn:li:corpuser:etl"
model_pattern: {} model_pattern: {}
view_pattern: {} view_pattern: {}
env: "PROD" # Optional, default is "PROD" env: "PROD" # optional, default is "PROD"
parse_table_names_from_sql: False # See note below. parse_table_names_from_sql: False # see note below
``` ```
Note! The integration can use [`sql-metadata`](https://pypi.org/project/sql-metadata/) to try to parse the tables the Note! The integration can use [`sql-metadata`](https://pypi.org/project/sql-metadata/) to try to parse the tables the

10
metadata-ingestion/src/datahub/ingestion/source/lookml.py
View File

@ -382,7 +382,15 @@ class LookMLSource(Source): # pragma: no cover
def _construct_datalineage_urn(self, sql_table_name: str, connection: str) -> str: def _construct_datalineage_urn(self, sql_table_name: str, connection: str) -> str:
platform = self._get_platform_based_on_connection(connection) platform = self._get_platform_based_on_connection(connection)
return f"urn:li:dataset:(urn:li:dataPlatform:{platform},{sql_table_name},{self.source_config.env})"
if "." in platform:
platform_name, database_name = platform.lower().split(".", maxsplit=1)
sql_table_name = f"{database_name}.{sql_table_name}".lower()
else:
platform_name = platform.lower()
sql_table_name = sql_table_name.lower()
return f"urn:li:dataset:(urn:li:dataPlatform:{platform_name},{sql_table_name},{self.source_config.env})"
def _get_platform_based_on_connection(self, connection: str) -> str: def _get_platform_based_on_connection(self, connection: str) -> str:
if connection in self.source_config.connection_to_platform_map: if connection in self.source_config.connection_to_platform_map: