yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-09-26 17:34:41 +00:00
Code Issues Packages Projects Releases Wiki Activity

Added databricks in lineage model (#9887)

Browse Source
This commit is contained in:
Milan Bariya 2023-01-25 16:06:48 +05:30 committed by GitHub
parent 40111d810c
commit 176111aa27
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 7 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
ingestion/src/metadata/ingestion/lineage/models.py
View File

@ -23,6 +23,9 @@ from metadata.generated.schema.entity.services.connections.database.bigQueryConn
from metadata.generated.schema.entity.services.connections.database.clickhouseConnection import ( from metadata.generated.schema.entity.services.connections.database.clickhouseConnection import (
ClickhouseType, ClickhouseType,
) )
from metadata.generated.schema.entity.services.connections.database.databricksConnection import (
DatabricksType,
)
from metadata.generated.schema.entity.services.connections.database.db2Connection import ( from metadata.generated.schema.entity.services.connections.database.db2Connection import (
Db2Type, Db2Type,
) )
@ -64,6 +67,7 @@ class Dialect(Enum):
ATHENA = "athena" ATHENA = "athena"
BIGQUERY = "bigquery" BIGQUERY = "bigquery"
CLICKHOUSE = "clickhouse" CLICKHOUSE = "clickhouse"
DATABRICKS = "databricks"
DB2 = "db2" DB2 = "db2"
DUCKDB = "duckdb" DUCKDB = "duckdb"
EXASOL = "exasol" EXASOL = "exasol"
@ -85,6 +89,7 @@ MAP_CONNECTION_TYPE_DIALECT: Dict[str, Dialect] = {
str(AthenaType.Athena.value): Dialect.ATHENA, str(AthenaType.Athena.value): Dialect.ATHENA,
str(BigqueryType.BigQuery.value): Dialect.BIGQUERY, str(BigqueryType.BigQuery.value): Dialect.BIGQUERY,
str(ClickhouseType.Clickhouse.value): Dialect.CLICKHOUSE, str(ClickhouseType.Clickhouse.value): Dialect.CLICKHOUSE,
str(DatabricksType.Databricks.value): Dialect.DATABRICKS,
str(Db2Type.Db2.value): Dialect.DB2, str(Db2Type.Db2.value): Dialect.DB2,
str(HiveType.Hive.value): Dialect.HIVE, str(HiveType.Hive.value): Dialect.HIVE,
str(MySQLType.Mysql.value): Dialect.MYSQL, str(MySQLType.Mysql.value): Dialect.MYSQL,

2
ingestion/src/metadata/ingestion/source/database/databricks/connection.py
View File

@ -37,7 +37,7 @@ def get_connection(connection: DatabricksConnection) -> Engine:
if connection.httpPath: if connection.httpPath:
if not connection.connectionArguments: if not connection.connectionArguments:
connection.connectionArguments = init_empty_connection_arguments() connection.connectionArguments = init_empty_connection_arguments()
connection.connectionArguments.__root__["httpPath"] = connection.httpPath connection.connectionArguments.__root__["http_path"] = connection.httpPath
return create_generic_db_connection( return create_generic_db_connection(
connection=connection, connection=connection,

1
ingestion/src/metadata/ingestion/source/database/databricks/query_parser.py
View File

@ -47,6 +47,7 @@ class DatabricksQueryParserSource(QueryParserSource, ABC):
self.metadata_config = metadata_config self.metadata_config = metadata_config
self.metadata = OpenMetadata(metadata_config) self.metadata = OpenMetadata(metadata_config)
self.connection = self.config.serviceConnection.__root__.config self.connection = self.config.serviceConnection.__root__.config
self.service_connection = self.config.serviceConnection.__root__.config
self.source_config = self.config.sourceConfig.config self.source_config = self.config.sourceConfig.config
self.start, self.end = get_start_and_end(self.source_config.queryLogDuration) self.start, self.end = get_start_and_end(self.source_config.queryLogDuration)
self.report = SQLSourceStatus() self.report = SQLSourceStatus()