yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-09-25 08:50:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

Added databricks in lineage model (#9887)

Browse Source
This commit is contained in:
Milan Bariya 2023-01-25 16:06:48 +05:30 committed by GitHub
parent 40111d810c
commit 176111aa27
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 7 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
ingestion/src/metadata/ingestion/lineage/models.py
View File

@ -23,6 +23,9 @@ from metadata.generated.schema.entity.services.connections.database.bigQueryConn
from metadata.generated.schema.entity.services.connections.database.clickhouseConnection import (
ClickhouseType,
)
from metadata.generated.schema.entity.services.connections.database.databricksConnection import (
DatabricksType,
)
from metadata.generated.schema.entity.services.connections.database.db2Connection import (
Db2Type,
)
@ -64,6 +67,7 @@ class Dialect(Enum):
ATHENA = "athena"
BIGQUERY = "bigquery"
CLICKHOUSE = "clickhouse"
DATABRICKS = "databricks"
DB2 = "db2"
DUCKDB = "duckdb"
EXASOL = "exasol"
@ -85,6 +89,7 @@ MAP_CONNECTION_TYPE_DIALECT: Dict[str, Dialect] = {
str(AthenaType.Athena.value): Dialect.ATHENA,
str(BigqueryType.BigQuery.value): Dialect.BIGQUERY,
str(ClickhouseType.Clickhouse.value): Dialect.CLICKHOUSE,
str(DatabricksType.Databricks.value): Dialect.DATABRICKS,
str(Db2Type.Db2.value): Dialect.DB2,
str(HiveType.Hive.value): Dialect.HIVE,
str(MySQLType.Mysql.value): Dialect.MYSQL,

2
ingestion/src/metadata/ingestion/source/database/databricks/connection.py
View File

@ -37,7 +37,7 @@ def get_connection(connection: DatabricksConnection) -> Engine:
if connection.httpPath:
if not connection.connectionArguments:
connection.connectionArguments = init_empty_connection_arguments()
connection.connectionArguments.__root__["httpPath"] = connection.httpPath
connection.connectionArguments.__root__["http_path"] = connection.httpPath
return create_generic_db_connection(
connection=connection,

1
ingestion/src/metadata/ingestion/source/database/databricks/query_parser.py
View File

@ -47,6 +47,7 @@ class DatabricksQueryParserSource(QueryParserSource, ABC):
self.metadata_config = metadata_config
self.metadata = OpenMetadata(metadata_config)
self.connection = self.config.serviceConnection.__root__.config
self.service_connection = self.config.serviceConnection.__root__.config
self.source_config = self.config.sourceConfig.config
self.start, self.end = get_start_and_end(self.source_config.queryLogDuration)
self.report = SQLSourceStatus()