diff --git a/ingestion/setup.py b/ingestion/setup.py index 7afc8071abc..ce607bfccb2 100644 --- a/ingestion/setup.py +++ b/ingestion/setup.py @@ -164,6 +164,7 @@ plugins: Dict[str, Set[str]] = { "thrift>=0.13,<1", "sasl~=0.3", "thrift-sasl~=0.4", + "impyla~=0.18.0", }, "kafka": {*COMMONS["kafka"]}, "kinesis": {VERSIONS["boto3"]}, diff --git a/ingestion/src/metadata/ingestion/source/database/hive/metadata.py b/ingestion/src/metadata/ingestion/source/database/hive/metadata.py index ef45e74c02f..ef22a8fad12 100644 --- a/ingestion/src/metadata/ingestion/source/database/hive/metadata.py +++ b/ingestion/src/metadata/ingestion/source/database/hive/metadata.py @@ -15,6 +15,7 @@ Hive source methods. import re from typing import Tuple +from impala.sqlalchemy import ImpalaDialect from pyhive.sqlalchemy_hive import HiveDialect, _type_map from sqlalchemy import types, util from sqlalchemy.engine import reflection @@ -184,6 +185,7 @@ def get_table_comment( # pylint: disable=unused-argument HiveDialect.get_columns = get_columns HiveDialect.get_table_comment = get_table_comment +ImpalaDialect.get_table_comment = get_table_comment HIVE_VERSION_WITH_VIEW_SUPPORT = "2.2.0" @@ -223,6 +225,10 @@ class HiveSource(CommonDbSourceService): ): HiveDialect.get_table_names = get_table_names HiveDialect.get_view_names = get_view_names + ImpalaDialect.get_table_names = get_table_names + ImpalaDialect.get_table_names = get_view_names else: HiveDialect.get_table_names = get_table_names_older_versions HiveDialect.get_view_names = get_view_names_older_versions + ImpalaDialect.get_table_names = get_table_names_older_versions + ImpalaDialect.get_table_names = get_view_names_older_versions diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/hiveConnection.json b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/hiveConnection.json index 2abf0dc88ad..6527bbf9af1 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/hiveConnection.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/hiveConnection.json @@ -15,7 +15,7 @@ "hiveScheme": { "description": "SQLAlchemy driver scheme options.", "type": "string", - "enum": ["hive", "hive+http", "hive+https"], + "enum": ["hive", "hive+http", "hive+https", "impala", "impala4"], "default": "hive" } },